feat(vitest-runner): add first implementation

.github/workflows/ci.yml

@@ -41,30 +41,15 @@ jobs:
     needs: list-examples
     strategy:
       matrix:
-        node-version: ["14", "16", "18", "20"]
+        node-version: ["16", "18", "20.5.1"]
         example: ${{ fromJson(needs.list-examples.outputs.examples) }}
-        include:
-          # tinybench does not support node 14
-          - node-version: "14"
-            pnpm-version: "7"
-            skip-tinybench: true
-        exclude:
-          # esbuild-register does not work well with node 14
-          - node-version: "14"
-            example: with-typescript-cjs
-          - node-version: "14"
-            example: with-typescript-esm
-          - node-version: "14"
-            example: with-typescript-simple-esm
       fail-fast: false
     steps:
       - uses: "actions/checkout@v3"
         with:
           fetch-depth: 0
       - run: sudo apt-get install -y valgrind
       - uses: pnpm/action-setup@v2
-        with:
-          version: ${{ matrix.pnpm-version || '' }}
       - uses: actions/setup-node@v3
         with:
           cache: pnpm
@@ -73,7 +58,6 @@ jobs:
       - run: pnpm moon run :build
 
       - name: Run benchmarks with tinybench-plugin
-        if: matrix.skip-tinybench != true
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v1`
         uses: CodSpeedHQ/action@main
         with:

.github/workflows/codspeed.yml

@@ -25,7 +25,10 @@ jobs:
       - name: Run benchmarks
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v1`
         uses: CodSpeedHQ/action@main
+        env:
+          CODSPEED_DEBUG: true
         with:
+          upload_url: ${{ secrets.CODSPEED_UPLOAD_URL_ADRIEN_LOCAL }}/upload
           run: |
             pnpm moon run --concurrency 1 :bench
             pnpm --workspace-concurrency 1 -r bench-tinybench

.moon/tasks.yml

@@ -35,6 +35,8 @@ tasks:
       - "/.eslintrc.js"
       - "tsconfig.json"
       - "tsconfig.*.json"
+    deps:
+      - "build"
 
   typecheck:
     command: "tsc --noEmit --pretty"
@@ -45,7 +47,7 @@ tasks:
       - "/tsconfig.json"
       - "/tsconfig.base.json"
     deps:
-      - "^:build"
+      - "build"
 
   build:
     command: "rollup -c rollup.config.ts --configPlugin typescript"
@@ -55,7 +57,6 @@ tasks:
     outputs:
       - "dist/"
     deps:
-      - "typecheck"
       - "^:build"
     env:
       NODE_NO_WARNINGS: "1"

.moon/toolchain.yml

@@ -2,21 +2,14 @@
 $schema: "https://moonrepo.dev/schemas/toolchain.json"
 
 node:
-  version: 18.16.0
-  syncVersionManagerConfig: "nvm"
-  addEnginesConstraint: true
-
   packageManager: "pnpm"
   pnpm:
     version: "8.6.3"
 
   aliasPackageNames: "name-only"
-
   dedupeOnLockfileChange: false
   dependencyVersionFormat: "workspace"
-
   inferTasksFromScripts: true
-
   syncProjectWorkspaceDependencies: true
 
 typescript:

docs/introspection.md

@@ -0,0 +1,15 @@
+## Testing introspection locally
+
+1. Inside `codspeed-node` directory, run `export PATH="$pwd/../action/dist/bin:$PATH"`.
+   This will ensure that the action's `dist/bin/node` file will be used instead of the
+   system's `node` binary.
+
+2. Replace the `CodSpeedHQ/action` grep filter with `CodSpeedHQ` in the `dist/bin/node`.
+   Since we used `../action` in the `export PATH=...` command, the original grep filter will
+   not work.
+
+3. Run your command with the correct flags in `codspeed-node`, for example
+
+```bash
+CI=1 CODSPEED_DEBUG=true pnpm --filter vitest-runner bench
+```

package.json

@@ -4,7 +4,7 @@
   "devDependencies": {
     "@commitlint/cli": "^17.5.1",
     "@commitlint/config-conventional": "^17.4.4",
-    "@moonrepo/cli": "^1.9.2",
+    "@moonrepo/cli": "^1.15.4",
     "@rollup/plugin-commonjs": "^24.1.0",
     "@rollup/plugin-json": "^6.0.0",
     "@rollup/plugin-node-resolve": "^15.0.2",

packages/benchmark.js-plugin/benches/parsePr.ts

@@ -0,0 +1,75 @@
+interface PullRequest {
+  number: number;
+  title: string;
+  body: string;
+}
+
+function sendEvent(numberOfOperations: number): void {
+  for (let i = 0; i < numberOfOperations; i++) {
+    let a = i;
+    a = a + 1;
+  }
+}
+
+function logMetrics(
+  numberOfOperations: number,
+  numberOfDeepOperations: number
+): void {
+  for (let i = 0; i < numberOfOperations; i++) {
+    for (let i = 0; i < numberOfOperations; i++) {
+      let a = i;
+      a = a + 1;
+      a = a + 1;
+    }
+    sendEvent(numberOfDeepOperations);
+  }
+}
+
+function parseTitle(title: string): void {
+  logMetrics(10, 10);
+  modifyTitle(title);
+}
+
+function modifyTitle(title: string): void {
+  for (let i = 0; i < 100; i++) {
+    let a = i;
+    a = a + 1 + title.length;
+  }
+}
+
+function prepareParsingBody(body: string): void {
+  for (let i = 0; i < 100; i++) {
+    let a = i;
+    a = a + 1;
+  }
+  parseBody(body);
+}
+
+function parseBody(body: string): void {
+  logMetrics(10, 10);
+  for (let i = 0; i < 200; i++) {
+    let a = i;
+    a = a + 1;
+  }
+  parseIssueFixed(body);
+}
+
+function parseIssueFixed(body: string): number | null {
+  const prefix = "fixes #";
+  const index = body.indexOf(prefix);
+  if (index === -1) {
+    return null;
+  }
+
+  const start = index + prefix.length;
+  let end = start;
+  while (end < body.length && /\d/.test(body[end])) {
+    end += 1;
+  }
+  return parseInt(body.slice(start, end));
+}
+
+export default function parsePr(pullRequest: PullRequest): void {
+  parseTitle(pullRequest.title);
+  prepareParsingBody(pullRequest.body);
+}

packages/benchmark.js-plugin/benches/sample.ts

@@ -1,15 +1,43 @@
 import Benchmark from "benchmark";
 import { withCodSpeed } from "..";
+import parsePr from "./parsePr";
+
+const LONG_BODY =
+  new Array(1_000)
+    .fill(
+      "Lorem ipsum dolor sit amet consectetur adipisicing elit. Sunt, earum. Atque architecto vero veniam est tempora fugiat sint quo praesentium quia. Autem, veritatis omnis beatae iste delectus recusandae animi non."
+    )
+    .join("\n") + "fixes #123";
 
 const suite = withCodSpeed(new Benchmark.Suite());
 
+console.log(`PROCESS PID: ${process.pid} in ${__filename}`);
+
 suite
   .add("RegExp#test", function () {
     /o/.test("Hello World!");
   })
   .add("String#indexOf", function () {
     "Hello World!".indexOf("o") > -1;
   })
+  .add("short body", () => {
+    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  })
+  .add("long body", () => {
+    parsePr({ body: LONG_BODY, title: "test", number: 124 });
+  })
+  .add("short body 2", () => {
+    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  })
+  .add("short body 3", () => {
+    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  })
+  .add("short body 4", () => {
+    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  })
+  .add("short body 5", () => {
+    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  })
   // add listeners
   .on("cycle", function (event: Benchmark.Event) {
     console.log(String(event.target));

packages/benchmark.js-plugin/package.json

@@ -21,15 +21,13 @@
   "devDependencies": {
     "@babel/preset-env": "^7.22.5",
     "@types/benchmark": "^2.1.2",
-    "@types/find-up": "^4.0.0",
     "@types/lodash": "^4.14.195",
     "@types/stack-trace": "^0.0.30",
     "benchmark": "^2.1.4",
     "jest-mock-extended": "^3.0.4"
   },
   "dependencies": {
     "@codspeed/core": "workspace:^2.2.0",
-    "find-up": "^6.3.0",
     "lodash": "^4.17.10",
     "stack-trace": "1.0.0-pre2"
   },

packages/benchmark.js-plugin/src/getCallingFile.ts

@@ -1,16 +1,8 @@
-import { findUpSync, Options as FindupOptions } from "find-up";
-import path, { dirname } from "path";
+import { getGitDir } from "@codspeed/core";
+import path from "path";
 import { get as getStackTrace } from "stack-trace";
 import { fileURLToPath } from "url";
 
-function getGitDir(path: string): string | undefined {
-  const dotGitPath = findUpSync(".git", {
-    cwd: path,
-    type: "directory",
-  } as FindupOptions);
-  return dotGitPath ? dirname(dotGitPath) : undefined;
-}
-
 export default function getCallingFile(depth: number): string {
   const stack = getStackTrace();
   let callingFile = stack[depth + 1].getFileName();

packages/benchmark.js-plugin/src/index.ts

@@ -4,6 +4,7 @@ import {
   optimizeFunctionSync,
   setupCore,
   teardownCore,
+  tryIntrospect,
 } from "@codspeed/core";
 import Benchmark from "benchmark";
 import buildSuiteAdd from "./buildSuiteAdd";
@@ -12,6 +13,8 @@ import { CodSpeedBenchmark } from "./types";
 
 declare const __VERSION__: string;
 
+tryIntrospect();
+
 interface WithCodSpeedBenchmark
   extends Omit<
     Benchmark,

packages/benchmark.js-plugin/tests/index.integ.test.ts

@@ -7,7 +7,10 @@ import { withCodSpeed } from "..";
 import { registerBenchmarks } from "./registerBenchmarks";
 import { registerOtherBenchmarks } from "./registerOtherBenchmarks";
 
-jest.mock("@codspeed/core", () => mockCore);
+jest.mock("@codspeed/core", () => {
+  mockCore.getGitDir = jest.requireActual("@codspeed/core").getGitDir;
+  return mockCore;
+});
 
 beforeEach(() => {
   mockReset(mockCore);

packages/core/package.json

@@ -20,11 +20,13 @@
   "homepage": "https://codspeed.io",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@types/find-up": "^4.0.0",
     "node-addon-api": "^5.1.0",
     "node-gyp": "^9.3.1",
     "prebuildify": "^5.0.1"
   },
   "dependencies": {
+    "find-up": "^6.3.0",
     "node-gyp-build": "^4.6.0"
   }
 }

packages/core/src/index.ts

@@ -1,6 +1,4 @@
-import { tryIntrospect } from "./introspection";
-tryIntrospect();
-
+import { checkV8Flags } from "./introspection";
 import native_core from "./native_core";
 import { initOptimization } from "./optimization";
 
@@ -16,11 +14,14 @@ export const setupCore = () => {
     `Metadata: codspeed-node ${__VERSION__}`
   );
   linuxPerf.start();
+  checkV8Flags();
 };
 
 export const teardownCore = () => {
   linuxPerf.stop();
 };
 
+export { getV8Flags, tryIntrospect } from "./introspection";
 export { optimizeFunction, optimizeFunctionSync } from "./optimization";
+export * from "./utils";
 export const Measurement = native_core.Measurement;

packages/core/src/introspection.ts

@@ -1,6 +1,10 @@
 import { writeFileSync } from "fs";
 
-const getV8Flags = (nodeVersionMajor: number) => {
+const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
+
+export const getV8Flags = () => {
+  const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]);
+
   const flags = [
     "--hash-seed=1",
     "--random-seed=1",
@@ -20,15 +24,26 @@ const getV8Flags = (nodeVersionMajor: number) => {
 
 export const tryIntrospect = () => {
   if (process.env.__CODSPEED_NODE_CORE_INTROSPECTION_PATH__ !== undefined) {
-    const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]);
-
     const introspectionMetadata = {
-      flags: getV8Flags(nodeVersionMajor),
+      flags: getV8Flags(),
     };
     writeFileSync(
       process.env.__CODSPEED_NODE_CORE_INTROSPECTION_PATH__,
       JSON.stringify(introspectionMetadata)
     );
-    process.exit(0);
+    process.exit(CUSTOM_INTROSPECTION_EXIT_CODE);
+  }
+};
+
+export const checkV8Flags = () => {
+  const requiredFlags = getV8Flags();
+  const actualFlags = process.execArgv;
+  const missingFlags = requiredFlags.filter(
+    (flag) => !actualFlags.includes(flag)
+  );
+  if (missingFlags.length > 0) {
+    console.warn(
+      `[CodSpeed] missing required flags: ${missingFlags.join(", ")}`
+    );
   }
 };

packages/core/src/utils.ts

@@ -0,0 +1,10 @@
+import { findUpSync, Options as FindupOptions } from "find-up";
+import { dirname } from "path";
+
+export function getGitDir(path: string): string | undefined {
+  const dotGitPath = findUpSync(".git", {
+    cwd: path,
+    type: "directory",
+  } as FindupOptions);
+  return dotGitPath ? dirname(dotGitPath) : undefined;
+}