feat: dynamic sampling time

src/benchmark/convergence/coefficientOfVariance.ts

@@ -0,0 +1,73 @@
+import Debug from "debug";
+import {BenchmarkOpts, ConvergenceCheckFn} from "../../types.js";
+import {calcMean, calcMedian, calcVariance, filterOutliers, OutlierSensitivity, sortData} from "../../utils/math.js";
+
+const debug = Debug("@chainsafe/benchmark/convergence");
+
+export function createCVConvergenceCriteria(
+  startMs: number,
+  {maxMs, maxRuns, minRuns, minMs, convergeFactor}: Required<BenchmarkOpts>
+): ConvergenceCheckFn {
+  let lastConvergenceSample = startMs;
+  let sampleEveryMs = Math.min(100, minMs);
+  const minSamples = Math.max(5, minRuns);
+  const maxSamplesForCV = 1000;
+
+  return function canTerminate(runIdx: number, totalNs: bigint, runsNs: bigint[]): boolean {
+    const currentMs = Date.now();
+    const elapsedMs = currentMs - startMs;
+    const timeSinceLastCheck = currentMs - lastConvergenceSample;
+    const mustStop = elapsedMs >= maxMs || runIdx >= maxRuns;
+    const mayStop = elapsedMs >= minMs && runIdx >= minRuns && runIdx >= minSamples;
+
+    debug(
+      "trying to converge benchmark via cv mustStop=%o, mayStop=%o, timeSinceLastCheck=%o",
+      mustStop,
+      mayStop,
+      timeSinceLastCheck
+    );
+
+    // Must stop
+    if (mustStop) return true;
+    if (!mayStop) return false;
+
+    // Only attempt to compute the confidence interval every sampleEveryMs
+    if (timeSinceLastCheck < sampleEveryMs) {
+      // If last call was wade 50% faster than the sampleEveryMs, let's reduce the sample interval to 10%
+      if (sampleEveryMs > 2 && sampleEveryMs / 2 > timeSinceLastCheck) {
+        sampleEveryMs -= sampleEveryMs * 0.1;
+      }
+      return false;
+    }
+
+    if (timeSinceLastCheck < sampleEveryMs) return false;
+    lastConvergenceSample = currentMs;
+    // For all statistical calculations we don't want to loose the precision so have to convert to numbers first
+    const samples = filterOutliers(sortData(runsNs.map((n) => Number(n))), true, OutlierSensitivity.Mild);
+
+    // If CV does not stabilize we fallback to the median approach
+    if (runsNs.length > maxSamplesForCV) {
+      const median = calcMedian(samples, true);
+      const mean = calcMean(samples);
+      const medianFactor = Math.abs(Number(mean - median)) / Number(median);
+
+      debug("checking convergence median convergeFactor=%o, medianFactor=%o", convergeFactor, medianFactor);
+
+      return medianFactor < convergeFactor;
+    }
+
+    const mean = calcMean(samples);
+    const variance = calcVariance(samples, mean);
+    const cv = Math.sqrt(variance) / mean;
+
+    debug(
+      "checking convergence via cv convergeFactor=%o, cv=%o, samples=%o, outliers=%o",
+      convergeFactor,
+      cv,
+      runsNs.length,
+      runsNs.length - samples.length
+    );
+
+    return cv < convergeFactor;
+  };
+}

src/benchmark/convergence/linearAverage.ts

@@ -0,0 +1,66 @@
+import Debug from "debug";
+import {BenchmarkOpts, ConvergenceCheckFn} from "../../types.js";
+
+const debug = Debug("@chainsafe/benchmark/convergence");
+
+export function createLinearConvergenceCriteria(
+  startMs: number,
+  {maxMs, maxRuns, minRuns, minMs, convergeFactor}: Required<BenchmarkOpts>
+): ConvergenceCheckFn {
+  let prevAvg0 = 0;
+  let prevAvg1 = 0;
+  let lastConvergenceSample = startMs;
+  const sampleEveryMs = 100;
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  return function canTerminate(runIdx: number, totalNs: bigint, _runNs: bigint[]): boolean {
+    const currentMs = Date.now();
+    const elapsedMs = currentMs - startMs;
+    const timeSinceLastCheck = currentMs - lastConvergenceSample;
+    const mustStop = elapsedMs >= maxMs || runIdx >= maxRuns;
+    const mayStop = elapsedMs >= minMs && runIdx >= minRuns;
+
+    debug(
+      "trying to converge benchmark via confidence-interval mustStop=%o, mayStop=%o, timeSinceLastCheck=%o",
+      mustStop,
+      mayStop,
+      timeSinceLastCheck
+    );
+
+    // Must stop
+    if (mustStop) return true;
+
+    // When is a good time to stop a benchmark? A naive answer is after N milliseconds or M runs.
+    // This code aims to stop the benchmark when the average fn run time has converged at a value
+    // within a given convergence factor. To prevent doing expensive math to often for fast fn,
+    // it only takes samples every `sampleEveryMs`. It stores two past values to be able to compute
+    // a very rough linear and quadratic convergence.a
+    if (timeSinceLastCheck <= sampleEveryMs) return false;
+
+    lastConvergenceSample = currentMs;
+    const avg = Number(totalNs / BigInt(runIdx));
+
+    // Compute convergence (1st order + 2nd order)
+    const a = prevAvg0;
+    const b = prevAvg1;
+    const c = avg;
+
+    if (mayStop) {
+      // Approx linear convergence
+      const convergence1 = Math.abs(c - a);
+      // Approx quadratic convergence
+      const convergence2 = Math.abs(b - (a + c) / 2);
+      // Take the greater of both to enforce linear and quadratic are below convergeFactor
+      const convergence = Math.max(convergence1, convergence2) / a;
+
+      debug("checking convergence convergeFactor=%o, convergence=%o", convergeFactor, convergence);
+
+      // Okay to stop + has converged, stop now
+      if (convergence < convergeFactor) return true;
+    }
+
+    prevAvg0 = prevAvg1;
+    prevAvg1 = avg;
+    return false;
+  };
+}

src/benchmark/options.ts

@@ -1,4 +1,4 @@
-import {BenchmarkOpts} from "../types.js";
+import {AverageCalculation, BenchmarkOpts, Convergence} from "../types.js";
 
 export const defaultBenchmarkOptions: Required<BenchmarkOpts> = {
   minRuns: 1,
@@ -17,8 +17,8 @@ export const defaultBenchmarkOptions: Required<BenchmarkOpts> = {
   skip: false,
   only: false,
   threshold: 2,
-  convergence: "linear",
-  averageCalculation: "simple",
+  convergence: Convergence.Linear,
+  averageCalculation: AverageCalculation.Simple,
 };
 
 export function getBenchmarkOptionsWithDefaults(opts: BenchmarkOpts): Required<BenchmarkOpts> {

src/benchmark/runBenchmarkFn.ts

@@ -1,12 +1,17 @@
-import {BenchmarkResult, BenchmarkOpts} from "../types.js";
+import Debug from "debug";
+import {BenchmarkResult, BenchmarkOpts, Convergence, ConvergenceCheckFn} from "../types.js";
 import {calcSum, filterOutliers, OutlierSensitivity} from "../utils/math.js";
 import {getBenchmarkOptionsWithDefaults} from "./options.js";
-import {createCVConvergenceCriteria, createLinearConvergenceCriteria} from "./termination.js";
+import {createLinearConvergenceCriteria} from "./convergence/linearAverage.js";
+import {createCVConvergenceCriteria} from "./convergence/coefficientOfVariance.js";
 
-const convergenceCriteria = {
-  ["linear"]: createLinearConvergenceCriteria,
-  ["cv"]: createCVConvergenceCriteria,
-};
+const debug = Debug("@chainsafe/benchmark/run");
+
+const convergenceCriteria: Record<Convergence, (startMs: number, opts: Required<BenchmarkOpts>) => ConvergenceCheckFn> =
+  {
+    [Convergence.Linear]: createLinearConvergenceCriteria,
+    [Convergence.CV]: createCVConvergenceCriteria,
+  };
 
 export type BenchmarkRunOpts = BenchmarkOpts & {
   id: string;
@@ -23,6 +28,7 @@ export async function runBenchFn<T, T2>(
   opts: BenchmarkRunOptsWithFn<T, T2>
 ): Promise<{result: BenchmarkResult; runsNs: bigint[]}> {
   const {id, before, beforeEach, fn, ...rest} = opts;
+  debug("running %o", id);
   const benchOptions = getBenchmarkOptionsWithDefaults(rest);
   const {maxMs, maxRuns, maxWarmUpMs, maxWarmUpRuns, runsFactor, threshold, convergence, averageCalculation} =
     benchOptions;
@@ -39,8 +45,8 @@ export async function runBenchFn<T, T2>(
     throw new Error(`Average calculation logic is not defined. ${averageCalculation}`);
   }
 
-  if (convergence !== "linear" && convergence !== "cv") {
-    throw new Error(`Unknown convergence value ${convergence}`);
+  if (!Object.values(Convergence).includes(convergence)) {
+    throw new Error(`Unknown convergence value ${convergence}. Valid values are ${Object.values(Convergence)}`);
   }
 
   // Ratio of maxMs that the warmup is allow to take from elapsedMs
@@ -59,12 +65,17 @@ export async function runBenchFn<T, T2>(
   let totalWarmUpRuns = 0;
   let isWarmUpPhase = maxWarmUpNs > 0 && maxWarmUpRuns > 0;
 
+  debug("starting before");
   const inputAll = before ? await before() : (undefined as unknown as T2);
+  debug("finished before");
 
   while (true) {
+    debug("executing individual run isWarmUpPhase=%o", isWarmUpPhase);
     const elapsedMs = Date.now() - startRunMs;
 
+    debug("starting beforeEach");
     const input = beforeEach ? await beforeEach(inputAll, runIdx) : (undefined as unknown as T);
+    debug("finished beforeEach");
 
     const startNs = process.hrtime.bigint();
     await fn(input);

src/cli/options.ts

@@ -1,5 +1,5 @@
 import {Options} from "yargs";
-import {StorageOptions, BenchmarkOpts, FileCollectionOptions} from "../types.js";
+import {StorageOptions, BenchmarkOpts, FileCollectionOptions, Convergence, AverageCalculation} from "../types.js";
 import {defaultBenchmarkOptions} from "../benchmark/options.js";
 
 export const optionsDefault = {
@@ -209,14 +209,14 @@ export const benchmarkOptions: ICliCommandOptions<CLIBenchmarkOptions> = {
     type: "string",
     description: "The algorithm used to detect the convergence to stop benchmark runs",
     default: defaultBenchmarkOptions.convergence,
-    choices: ["linear", "cv"],
+    choices: Object.values(Convergence),
     group: benchmarkGroup,
   },
   averageCalculation: {
     type: "string",
     description: "Use simple average of all runs or clean the outliers before calculating average",
     default: defaultBenchmarkOptions.averageCalculation,
-    choices: ["simple", "clean-outliers"],
+    choices: Object.values(AverageCalculation),
     group: benchmarkGroup,
   },
 };