From 73b61179bdd581c96993fc3ff09d90a00141f0da Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Wed, 31 Jul 2024 18:35:03 -0300 Subject: [PATCH 1/3] benchmark: support benchmark coverage --- benchmark/_cli.js | 13 ++++- benchmark/coverage.js | 115 ++++++++++++++++++++++++++++++++++++++++++ benchmark/run.js | 92 +++++++++++++++++++++++++++++++-- 3 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 benchmark/coverage.js diff --git a/benchmark/_cli.js b/benchmark/_cli.js index 583da8f6b20622..fab9ca02f4dd50 100644 --- a/benchmark/_cli.js +++ b/benchmark/_cli.js @@ -24,6 +24,7 @@ function CLI(usage, settings) { this.optional = {}; this.items = []; this.test = false; + this.coverage = false; for (const argName of settings.arrayArgs) { this.optional[argName] = []; @@ -66,6 +67,14 @@ function CLI(usage, settings) { mode = 'both'; } else if (arg === 'test') { this.test = true; + } else if (arg === 'coverage') { + this.coverage = true; + // TODO: add support to those benchmarks + const excludedBenchmarks = ['napi', 'http']; + this.items = Object.keys(benchmarks) + .filter(b => !excludedBenchmarks.includes(b)); + // Run once + this.optional.set = ['n=1']; } else if (['both', 'item'].includes(mode)) { // item arguments this.items.push(arg); @@ -140,8 +149,8 @@ CLI.prototype.getCpuCoreSetting = function() { const isValid = /^(\d+(-\d+)?)(,\d+(-\d+)?)*$/.test(value); if (!isValid) { throw new Error(` - Invalid CPUSET format: "${value}". Please use a single core number (e.g., "0"), - a range of cores (e.g., "0-3"), or a list of cores/ranges + Invalid CPUSET format: "${value}". Please use a single core number (e.g., "0"), + a range of cores (e.g., "0-3"), or a list of cores/ranges (e.g., "0,2,4" or "0-2,4").\n\n${this.usage} `); } diff --git a/benchmark/coverage.js b/benchmark/coverage.js new file mode 100644 index 00000000000000..1fc66055f60673 --- /dev/null +++ b/benchmark/coverage.js @@ -0,0 +1,115 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const Mod = require('node:module'); + +const benchmarkFolder = __dirname; +const dir = fs.readdirSync(path.join(__dirname, '../lib')); + +const allModuleExports = {}; + +function getCallSite() { + const originalStackFormatter = Error.prepareStackTrace; + Error.prepareStackTrace = (err, stack) => { + // Some benchmarks change the stackTraceLimit if so, get the last line + // TODO: check if it matches the benchmark folder + if (stack.length >= 2) { + return `${stack[2].getFileName()}:${stack[2].getLineNumber()}`; + } + return stack; + } + + const err = new Error(); + err.stack; + Error.prepareStackTrace = originalStackFormatter; + return err.stack; +} + +const skippedFunctionClasses = [ + 'EventEmitter', + 'Worker', + 'ClientRequest', + 'Readable', + 'StringDecoder', + 'TLSSocket', + 'MessageChannel', +]; + +const skippedModules = [ + 'node:cluster', + 'node:trace_events', + 'node:stream/promises', +]; + +function fetchModules (allModuleExports) { + for (const f of dir) { + if (f.endsWith('.js') && !f.startsWith('_')) { + const moduleName = `node:${f.slice(0, f.length - 3)}` + if (skippedModules.includes(moduleName)) { + continue; + } + const exports = require(moduleName); + allModuleExports[moduleName] = Object.assign({}, exports); + + for (const fnKey of Object.keys(exports)) { + if (typeof exports[fnKey] === 'function' && !fnKey.startsWith('_')) { + if ( + exports[fnKey].toString().match(/^class/) || + skippedFunctionClasses.includes(fnKey) + ) { + // Skip classes for now + continue; + } + const originalFn = exports[fnKey]; + allModuleExports[moduleName][fnKey] = function () { + const callerStr = getCallSite(); + if (typeof callerStr === 'string' && callerStr.startsWith(benchmarkFolder) && + callerStr.replace(benchmarkFolder, '').match(/^\/.+\/.+/)) { + if (!allModuleExports[moduleName][fnKey]._called) { + allModuleExports[moduleName][fnKey]._called = 0; + } + allModuleExports[moduleName][fnKey]._called++; + + + if (!allModuleExports[moduleName][fnKey]._calls) { + allModuleExports[moduleName][fnKey]._calls = []; + } + allModuleExports[moduleName][fnKey]._calls.push(callerStr); + } + return originalFn.apply(exports, arguments); + } + } + } + } + } +} + +fetchModules(allModuleExports); + +const req = Mod.prototype.require; +Mod.prototype.require = function (id) { + let newId = id; + if (!id.startsWith('node:')) { + newId = `node:${id}`; + } + const data = allModuleExports[newId] + if (!data) { + return req.apply(this, arguments) + } + return data; +}; + +process.on('beforeExit', () => { + for (const module of Object.keys(allModuleExports)) { + for (const fn of Object.keys(allModuleExports[module])) { + if (allModuleExports[module][fn]?._called) { + const _fn = allModuleExports[module][fn]; + process.send({ + type: 'coverage', + module, + fn, + times: _fn._called + }); + } + } + } +}) diff --git a/benchmark/run.js b/benchmark/run.js index 6a61df71221710..62e7e95e26e048 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -1,7 +1,8 @@ 'use strict'; -const path = require('path'); +const path = require('node:path'); const { spawn, fork } = require('node:child_process'); +const fs = require('node:fs'); const CLI = require('./_cli.js'); const cli = new CLI(`usage: ./node run.js [options] [--] ... @@ -16,6 +17,8 @@ const cli = new CLI(`usage: ./node run.js [options] [--] ... --format [simple|csv] optional value that specifies the output format test only run a single configuration from the options matrix + coverage generate a coverage report for the nodejs + benchmark suite all each benchmark category is run one after the other Examples: @@ -41,15 +44,47 @@ if (!validFormats.includes(format)) { return; } -if (format === 'csv') { +if (format === 'csv' && !cli.coverage) { console.log('"filename", "configuration", "rate", "time"'); } +function fetchModules () { + const dir = fs.readdirSync(path.join(__dirname, '../lib')); + const allModuleExports = {}; + for (const f of dir) { + if (f.endsWith('.js') && !f.startsWith('_')) { + const moduleName = `node:${f.slice(0, f.length - 3)}` + const exports = require(moduleName); + allModuleExports[moduleName] = {} + for (const fnKey of Object.keys(exports)) { + if (typeof exports[fnKey] === 'function' && !fnKey.startsWith('_')) { + allModuleExports[moduleName] = { + ...allModuleExports[moduleName], + [fnKey]: 0, + }; + } + } + } + } + return allModuleExports; +} + +let allModuleExports = {}; +if (cli.coverage) { + allModuleExports = fetchModules(); +} + (function recursive(i) { const filename = benchmarks[i]; const scriptPath = path.resolve(__dirname, filename); - const args = cli.test ? ['--test'] : cli.optional.set; + const args = cli.test ? ['--test'] : [...cli.optional.set]; + + let execArgv = []; + if (cli.coverage) { + execArgv = ['-r', path.join(__dirname, './coverage.js'), '--experimental-sqlite', '--no-warnings']; + } + const cpuCore = cli.getCpuCoreSetting(); let child; if (cpuCore !== null) { @@ -60,6 +95,7 @@ if (format === 'csv') { child = fork( scriptPath, args, + { execArgv }, ); } @@ -69,6 +105,15 @@ if (format === 'csv') { } child.on('message', (data) => { + if (cli.coverage) { + if (data.type === 'coverage') { + if (allModuleExports[data.module][data.fn] !== undefined) { + delete allModuleExports[data.module][data.fn]; + } + } + return; + } + if (data.type !== 'report') { return; } @@ -102,3 +147,44 @@ if (format === 'csv') { } }); })(0); + +const skippedFunctionClasses = [ + 'EventEmitter', + 'Worker', + 'ClientRequest', + 'Readable', + 'StringDecoder', + 'TLSSocket', + 'MessageChannel', +]; + +const skippedModules = [ + 'node:cluster', + 'node:trace_events', + 'node:stream/promises', +]; + +if (cli.coverage) { + process.on('beforeExit', () => { + for (const key in allModuleExports) { + if (skippedModules.includes(key)) continue; + const tableData = []; + for (const innerKey in allModuleExports[key]) { + if ( + allModuleExports[key][innerKey].toString().match(/^class/) || + skippedFunctionClasses.includes(innerKey) + ) { + continue; + } + + tableData.push({ + [key]: innerKey, + Values: allModuleExports[key][innerKey] + }); + } + if (tableData.length) { + console.table(tableData) + } + } + }) +} From f8f263e8a2168a2001cacf2be9254e6c594ca696 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Thu, 8 Aug 2024 19:06:05 -0300 Subject: [PATCH 2/3] benchmark: use assert.ok searchparams --- benchmark/url/url-searchparams-update.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/url/url-searchparams-update.js b/benchmark/url/url-searchparams-update.js index 082d476a5d2250..3c42de61110ef3 100644 --- a/benchmark/url/url-searchparams-update.js +++ b/benchmark/url/url-searchparams-update.js @@ -17,7 +17,7 @@ function getMethod(url, property) { function main({ searchParams, property, n }) { const url = new URL('https://nodejs.org'); - if (searchParams === 'true') assert(url.searchParams); + if (searchParams === 'true') assert.ok(url.searchParams); const method = getMethod(url, property); From 821a8d0f424850b71189d3b1b3bd34eda863c781 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Mon, 12 Aug 2024 12:03:52 -0300 Subject: [PATCH 3/3] fixup! lint fix --- benchmark/_cli.js | 4 ++-- benchmark/coverage.js | 24 +++++++++++++----------- benchmark/run.js | 24 ++++++++++++------------ 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/benchmark/_cli.js b/benchmark/_cli.js index fab9ca02f4dd50..2becd7368dc610 100644 --- a/benchmark/_cli.js +++ b/benchmark/_cli.js @@ -70,9 +70,9 @@ function CLI(usage, settings) { } else if (arg === 'coverage') { this.coverage = true; // TODO: add support to those benchmarks - const excludedBenchmarks = ['napi', 'http']; + const excludedBenchmarks = ['napi', 'http']; this.items = Object.keys(benchmarks) - .filter(b => !excludedBenchmarks.includes(b)); + .filter((b) => !excludedBenchmarks.includes(b)); // Run once this.optional.set = ['n=1']; } else if (['both', 'item'].includes(mode)) { diff --git a/benchmark/coverage.js b/benchmark/coverage.js index 1fc66055f60673..582ccd0a0f2ac3 100644 --- a/benchmark/coverage.js +++ b/benchmark/coverage.js @@ -1,3 +1,5 @@ +'use strict'; + const fs = require('node:fs'); const path = require('node:path'); const Mod = require('node:module'); @@ -16,10 +18,10 @@ function getCallSite() { return `${stack[2].getFileName()}:${stack[2].getLineNumber()}`; } return stack; - } + }; const err = new Error(); - err.stack; + err.stack; // eslint-disable-line no-unused-expressions Error.prepareStackTrace = originalStackFormatter; return err.stack; } @@ -40,10 +42,10 @@ const skippedModules = [ 'node:stream/promises', ]; -function fetchModules (allModuleExports) { +function fetchModules(allModuleExports) { for (const f of dir) { if (f.endsWith('.js') && !f.startsWith('_')) { - const moduleName = `node:${f.slice(0, f.length - 3)}` + const moduleName = `node:${f.slice(0, f.length - 3)}`; if (skippedModules.includes(moduleName)) { continue; } @@ -60,7 +62,7 @@ function fetchModules (allModuleExports) { continue; } const originalFn = exports[fnKey]; - allModuleExports[moduleName][fnKey] = function () { + allModuleExports[moduleName][fnKey] = function() { const callerStr = getCallSite(); if (typeof callerStr === 'string' && callerStr.startsWith(benchmarkFolder) && callerStr.replace(benchmarkFolder, '').match(/^\/.+\/.+/)) { @@ -76,7 +78,7 @@ function fetchModules (allModuleExports) { allModuleExports[moduleName][fnKey]._calls.push(callerStr); } return originalFn.apply(exports, arguments); - } + }; } } } @@ -86,14 +88,14 @@ function fetchModules (allModuleExports) { fetchModules(allModuleExports); const req = Mod.prototype.require; -Mod.prototype.require = function (id) { +Mod.prototype.require = function(id) { let newId = id; if (!id.startsWith('node:')) { newId = `node:${id}`; } - const data = allModuleExports[newId] + const data = allModuleExports[newId]; if (!data) { - return req.apply(this, arguments) + return req.apply(this, arguments); } return data; }; @@ -107,9 +109,9 @@ process.on('beforeExit', () => { type: 'coverage', module, fn, - times: _fn._called + times: _fn._called, }); } } } -}) +}); diff --git a/benchmark/run.js b/benchmark/run.js index 62e7e95e26e048..0e8c33dea24bb3 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -48,14 +48,14 @@ if (format === 'csv' && !cli.coverage) { console.log('"filename", "configuration", "rate", "time"'); } -function fetchModules () { +function fetchModules() { const dir = fs.readdirSync(path.join(__dirname, '../lib')); const allModuleExports = {}; for (const f of dir) { if (f.endsWith('.js') && !f.startsWith('_')) { - const moduleName = `node:${f.slice(0, f.length - 3)}` + const moduleName = `node:${f.slice(0, f.length - 3)}`; const exports = require(moduleName); - allModuleExports[moduleName] = {} + allModuleExports[moduleName] = {}; for (const fnKey of Object.keys(exports)) { if (typeof exports[fnKey] === 'function' && !fnKey.startsWith('_')) { allModuleExports[moduleName] = { @@ -170,21 +170,21 @@ if (cli.coverage) { if (skippedModules.includes(key)) continue; const tableData = []; for (const innerKey in allModuleExports[key]) { - if ( - allModuleExports[key][innerKey].toString().match(/^class/) || + if ( + allModuleExports[key][innerKey].toString().match(/^class/) || skippedFunctionClasses.includes(innerKey) - ) { - continue; - } + ) { + continue; + } tableData.push({ [key]: innerKey, - Values: allModuleExports[key][innerKey] + Values: allModuleExports[key][innerKey], }); } - if (tableData.length) { - console.table(tableData) + if (tableData.length) { + console.table(tableData); } } - }) + }); }