From 63c61f08d5f42738546a95ed307059a93743abe9 Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Fri, 30 Dec 2022 03:01:04 +0100 Subject: [PATCH 01/10] Add helper scripts - update-headers.js: Fetch latest headers from nodejs/node - write-symbols.js: Use clang to process headers to create symbols.js --- .npmignore | 1 + package.json | 2 + scripts/update-headers.js | 31 ++++++++ scripts/write-symbols.js | 144 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 .npmignore create mode 100644 scripts/update-headers.js create mode 100644 scripts/write-symbols.js diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..5093a29 --- /dev/null +++ b/.npmignore @@ -0,0 +1 @@ +scripts/ diff --git a/package.json b/package.json index 1680474..877d7b8 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,8 @@ "url": "git://github.com/nodejs/node-api-headers.git" }, "scripts": { + "update-headers": "node --no-warnings scripts/update-headers.js", + "write-symbols": "node --no-warnings scripts/write-symbols.js" }, "version": "0.0.2", "support": true diff --git a/scripts/update-headers.js b/scripts/update-headers.js new file mode 100644 index 0000000..56d04a3 --- /dev/null +++ b/scripts/update-headers.js @@ -0,0 +1,31 @@ +const { createWriteStream } = require('fs'); +const { Readable } = require('stream'); +const { finished } = require('stream/promises'); +const { resolve } = require('path'); + +const files = ['js_native_api_types.h', 'js_native_api.h', 'node_api_types.h', 'node_api.h']; + +const commit = process.argv[2] ?? 'main'; + +console.log(`Using commit ${commit}:`); + +async function main() { + for (const filename of files) { + const url = `https://raw.githubusercontent.com/nodejs/node/${commit}/src/${filename}`; + const path = resolve(__dirname, '..', 'include', filename); + console.log(` ${url} -> ${path}`); + + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Fetch of ${url} returned ${response.status} ${response.statusText}`); + } + + const stream = createWriteStream(path); + await finished(Readable.fromWeb(response.body).pipe(stream)); + } +} + +main().catch(e => { + console.error(e); + process.exitCode = 1; +}); diff --git a/scripts/write-symbols.js b/scripts/write-symbols.js new file mode 100644 index 0000000..687335d --- /dev/null +++ b/scripts/write-symbols.js @@ -0,0 +1,144 @@ +const { spawnSync } = require('child_process'); +const { resolve } = require('path'); +const { writeFileSync } = require('fs'); + +function getSymbolsForVersion(version) { + const spawned = spawnSync('clang', + ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', version ? `-DNAPI_VERSION=${version}` : '-DNAPI_EXPERIMENTAL', resolve(__dirname, '..', 'include', 'node_api.h')], + { maxBuffer: 2_000_000 } + ); + + if (spawned.error) { + if (spawned.error.code === 'ENOENT') { + throw new Error('This tool requires clang to be installed.'); + } + throw spawned.error; + } else if (spawned.stderr.length > 0) { + throw new Error(spawned.stderr.toString('utf-8')); + } + + const ast = JSON.parse(spawned.stdout.toString('utf-8')); + const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; + + for (const statement of ast.inner) { + if (statement.kind !== 'FunctionDecl') { + continue; + } + + const name = statement.name; + const file = statement.loc.includedFrom?.file; + + if (file) { + symbols.js_native_api_symbols.push(name); + } else { + symbols.node_api_symbols.push(name); + } + } + + symbols.js_native_api_symbols.sort(); + symbols.node_api_symbols.sort(); + + return symbols; +} + + +function getAllSymbols() { + const allSymbols = {}; + let version = 1; + + console.log('Processing symbols from clang:') + while (true) { + const symbols = getSymbolsForVersion(version); + + if (version > 1) { + const previousSymbols = allSymbols[`v${version - 1}`]; + if (previousSymbols.js_native_api_symbols.length == symbols.js_native_api_symbols.length && previousSymbols.node_api_symbols.length === symbols.node_api_symbols.length) { + --version; + break; + } + } + allSymbols[`v${version}`] = symbols; + console.log(` v${version}: ${symbols.js_native_api_symbols.length} js_native_api_symbols, ${symbols.node_api_symbols.length} node_api_symbols`); + ++version; + } + + const symbols = allSymbols[`experimental`] = getSymbolsForVersion(); + console.log(` Experimental: ${symbols.js_native_api_symbols.length} js_native_api_symbols, ${symbols.node_api_symbols.length} node_api_symbols`); + return { + maxVersion: version, + symbols: allSymbols + }; +} + +function getUniqueSymbols(previousSymbols, currentSymbols) { + const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; + for (const symbol of currentSymbols.js_native_api_symbols) { + if (!previousSymbols.js_native_api_symbols.includes(symbol)) { + symbols.js_native_api_symbols.push(symbol); + } + } + for (const symbol of currentSymbols.node_api_symbols) { + if (!previousSymbols.node_api_symbols.includes(symbol)) { + symbols.node_api_symbols.push(symbol); + } + } + return symbols; +} + +function joinSymbols(symbols, prependNewLine) { + if (symbols.length === 0) return ''; + return `${prependNewLine ? ',\n ' : ''}'${symbols.join("',\n '")}'`; +} + +function getSymbolData() { + const { maxVersion, symbols } = getAllSymbols(); + + let data = `'use strict' + +const v1 = { + js_native_api_symbols: [ + ${joinSymbols(symbols.v1.js_native_api_symbols)} + ], + node_api_symbols: [ + ${joinSymbols(symbols.v1.node_api_symbols)} + ] +} +`; + + for (let version = 2; version <= maxVersion + 1; ++version) { + const newSymbols = getUniqueSymbols(symbols[`v${version - 1}`], symbols[version === maxVersion + 1 ? 'experimental' : `v${version}`]); + + data += ` +const ${version === maxVersion + 1 ? 'experimental' : `v${version}`} = { + js_native_api_symbols: [ + ...v${version - 1}.js_native_api_symbols${joinSymbols(newSymbols.js_native_api_symbols, true)} + ], + node_api_symbols: [ + ...v${version - 1}.node_api_symbols${joinSymbols(newSymbols.node_api_symbols, true)} + ] +} +`; + } + + data += ` +module.exports = { + ${new Array(maxVersion).fill(undefined).map((_, i) => `v${i + 1}`).join(',\n ')}, + experimental +} +` + return data; +} + +function main() { + const path = resolve(__dirname, '../symbols.js'); + const data = getSymbolData(); + console.log(`Writing symbols to ${path}`) + writeFileSync(path, data); +} + +try { + main(); +} catch (e) { + console.error(e); + process.exitCode = 1; +} From 8e693f75d5dbf726bf559b04834efa0a0f12112b Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Fri, 30 Dec 2022 03:28:45 +0100 Subject: [PATCH 02/10] Use current node major version branch for headers --- scripts/update-headers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update-headers.js b/scripts/update-headers.js index 56d04a3..61c3838 100644 --- a/scripts/update-headers.js +++ b/scripts/update-headers.js @@ -5,7 +5,7 @@ const { resolve } = require('path'); const files = ['js_native_api_types.h', 'js_native_api.h', 'node_api_types.h', 'node_api.h']; -const commit = process.argv[2] ?? 'main'; +const commit = process.argv[2] ?? `${process.version.substring(0,3)}.x`; console.log(`Using commit ${commit}:`); From 0834c30961cec82d25cb0736ec1216b18a1d3cfd Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Fri, 30 Dec 2022 23:47:57 +0100 Subject: [PATCH 03/10] Use latest release from nodejs.org releases --- scripts/update-headers.js | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/scripts/update-headers.js b/scripts/update-headers.js index 61c3838..6b939b2 100644 --- a/scripts/update-headers.js +++ b/scripts/update-headers.js @@ -2,18 +2,40 @@ const { createWriteStream } = require('fs'); const { Readable } = require('stream'); const { finished } = require('stream/promises'); const { resolve } = require('path'); +const { parseArgs } = require('util') -const files = ['js_native_api_types.h', 'js_native_api.h', 'node_api_types.h', 'node_api.h']; +async function getLatestReleaseVersion() { + const response = await fetch('https://nodejs.org/download/release/index.json'); + const json = await response.json(); + return json[0].version; +} -const commit = process.argv[2] ?? `${process.version.substring(0,3)}.x`; +async function main() { + const { values: { tag, verbose } } = parseArgs({ + options: { + tag: { + type: "string", + short: "t", + default: await getLatestReleaseVersion() + }, + verbose: { + type: "boolean", + short: "v", + }, + }, + }); -console.log(`Using commit ${commit}:`); + console.log(`Update headers from nodejs/node tag ${tag}`); + + const files = ['js_native_api_types.h', 'js_native_api.h', 'node_api_types.h', 'node_api.h']; -async function main() { for (const filename of files) { - const url = `https://raw.githubusercontent.com/nodejs/node/${commit}/src/${filename}`; + const url = `https://raw.githubusercontent.com/nodejs/node/${tag}/src/${filename}`; const path = resolve(__dirname, '..', 'include', filename); - console.log(` ${url} -> ${path}`); + + if (verbose) { + console.log(` ${url} -> ${path}`); + } const response = await fetch(url); if (!response.ok) { From eb2f62898f4d6f3eb34ec4f5e133f972d1bb0806 Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Sat, 31 Dec 2022 04:28:19 +0100 Subject: [PATCH 04/10] Add GitHub Actions sync workflow --- .github/workflows/sync-headers.yml | 52 ++++++++++++++++++++++++++++++ .npmignore | 1 + 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/sync-headers.yml diff --git a/.github/workflows/sync-headers.yml b/.github/workflows/sync-headers.yml new file mode 100644 index 0000000..5a9459f --- /dev/null +++ b/.github/workflows/sync-headers.yml @@ -0,0 +1,52 @@ +name: Header Sync + +on: + workflow_dispatch: null + schedule: + - cron: "0 0 * * *" + +permissions: + contents: write + pull-requests: write + +jobs: + build: + runs-on: ubuntu-latest + name: Update headers from nodejs/node + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - shell: bash + id: check-changes + name: Check Changes + run: | + COMMIT_MESSAGE=$(npm run --silent update-headers) + VERSION=${COMMIT_MESSAGE##* } + echo $COMMIT_MESSAGE + npm run --silent write-symbols + CHANGED_FILES=$(git diff --name-only) + BRANCH_NAME="update-headers/${VERSION}" + if [ -z "$CHANGED_FILES" ]; then + echo "No changes exist. Nothing to do." + else + echo "Changes exist. Checking if branch exists: $BRANCH_NAME" + if git ls-remote --exit-code --heads $GITHUB_SERVER_URL/$GITHUB_REPOSITORY $BRANCH_NAME >/dev/null; then + echo "Branch exists. Nothing to do." + else + echo "Branch does not exists." + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT + echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> $GITHUB_OUTPUT + fi + fi + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + if: ${{ steps.check-changes.outputs.BRANCH_NAME }} + with: + branch: ${{ steps.check-changes.outputs.BRANCH_NAME }} + commit-message: ${{ steps.check-changes.outputs.COMMIT_MESSAGE }} + title: ${{ steps.check-changes.outputs.COMMIT_MESSAGE }} + author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + body: null + delete-branch: true diff --git a/.npmignore b/.npmignore index 5093a29..127db70 100644 --- a/.npmignore +++ b/.npmignore @@ -1 +1,2 @@ scripts/ +.github/ From b9a57c3721c8be26c76a5e2777e05ed3886a938b Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:18:03 +0100 Subject: [PATCH 05/10] Address review comments - Add 'use strict' - Make write-symbols main() async --- scripts/update-headers.js | 2 + scripts/write-symbols.js | 148 ++++++++++++++++++++++++-------------- 2 files changed, 96 insertions(+), 54 deletions(-) diff --git a/scripts/update-headers.js b/scripts/update-headers.js index 6b939b2..13a1687 100644 --- a/scripts/update-headers.js +++ b/scripts/update-headers.js @@ -1,3 +1,5 @@ +'use strict'; + const { createWriteStream } = require('fs'); const { Readable } = require('stream'); const { finished } = require('stream/promises'); diff --git a/scripts/write-symbols.js b/scripts/write-symbols.js index 687335d..3cad531 100644 --- a/scripts/write-symbols.js +++ b/scripts/write-symbols.js @@ -1,54 +1,87 @@ -const { spawnSync } = require('child_process'); -const { resolve } = require('path'); -const { writeFileSync } = require('fs'); - -function getSymbolsForVersion(version) { - const spawned = spawnSync('clang', - ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', version ? `-DNAPI_VERSION=${version}` : '-DNAPI_EXPERIMENTAL', resolve(__dirname, '..', 'include', 'node_api.h')], - { maxBuffer: 2_000_000 } - ); - - if (spawned.error) { - if (spawned.error.code === 'ENOENT') { - throw new Error('This tool requires clang to be installed.'); +// @ts-check +'use strict'; + +const { spawn } = require('child_process'); +const { resolve: resolvePath } = require('path'); +const { writeFile } = require('fs/promises'); + +/** @typedef {{ js_native_api_symbols: string[]; node_api_symbols: string[]; }} SymbolInfo */ + +/** + * @param {number | undefined} [version] + * @returns {Promise} + */ +async function getSymbolsForVersion(version) { + try { + const { exitCode, stdout, stderr } = await new Promise((resolve, reject) => { + const spawned = spawn('clang', + ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', version ? `-DNAPI_VERSION=${version}` : '-DNAPI_EXPERIMENTAL', resolvePath(__dirname, '..', 'include', 'node_api.h')] + ); + + let stdout = ''; + let stderr = ''; + + spawned.stdout?.on('data', (data) => { + stdout += data.toString('utf-8'); + }); + spawned.stderr?.on('data', (data) => { + stderr += data.toString('utf-8'); + }); + + spawned.on('exit', function (exitCode) { + resolve({ exitCode, stdout, stderr }); + }); + + spawned.on('error', function (err) { + reject(err); + }); + }); + + if (exitCode !== 0) { + throw new Error(`clang exited with non-zero exit code ${exitCode}. stderr: ${stderr ? stderr : ''}`); } - throw spawned.error; - } else if (spawned.stderr.length > 0) { - throw new Error(spawned.stderr.toString('utf-8')); - } - const ast = JSON.parse(spawned.stdout.toString('utf-8')); - const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; + const ast = JSON.parse(stdout); - for (const statement of ast.inner) { - if (statement.kind !== 'FunctionDecl') { - continue; - } + /** @type {{js_native_api_symbols: string[], node_api_symbols: string[]}} */ + const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; - const name = statement.name; - const file = statement.loc.includedFrom?.file; + for (const statement of ast.inner) { + if (statement.kind !== 'FunctionDecl') { + continue; + } + + const name = statement.name; + const file = statement.loc.includedFrom?.file; - if (file) { - symbols.js_native_api_symbols.push(name); - } else { - symbols.node_api_symbols.push(name); + if (file) { + symbols.js_native_api_symbols.push(name); + } else { + symbols.node_api_symbols.push(name); + } } - } - symbols.js_native_api_symbols.sort(); - symbols.node_api_symbols.sort(); + symbols.js_native_api_symbols.sort(); + symbols.node_api_symbols.sort(); - return symbols; + return symbols; + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error('This tool requires clang to be installed.'); + } + throw err; + } } - -function getAllSymbols() { +/** @returns {Promise<{maxVersion: number, symbols: {[x: string]: SymbolInfo}}>} */ +async function getAllSymbols() { + /** @type {{[x: string]: SymbolInfo}} */ const allSymbols = {}; let version = 1; console.log('Processing symbols from clang:') while (true) { - const symbols = getSymbolsForVersion(version); + const symbols = await getSymbolsForVersion(version); if (version > 1) { const previousSymbols = allSymbols[`v${version - 1}`]; @@ -62,7 +95,7 @@ function getAllSymbols() { ++version; } - const symbols = allSymbols[`experimental`] = getSymbolsForVersion(); + const symbols = allSymbols[`experimental`] = await getSymbolsForVersion(); console.log(` Experimental: ${symbols.js_native_api_symbols.length} js_native_api_symbols, ${symbols.node_api_symbols.length} node_api_symbols`); return { maxVersion: version, @@ -70,7 +103,13 @@ function getAllSymbols() { }; } +/** + * @param {SymbolInfo} previousSymbols + * @param {SymbolInfo} currentSymbols + * @returns {SymbolInfo} + */ function getUniqueSymbols(previousSymbols, currentSymbols) { + /** @type {SymbolInfo} */ const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; for (const symbol of currentSymbols.js_native_api_symbols) { if (!previousSymbols.js_native_api_symbols.includes(symbol)) { @@ -85,22 +124,25 @@ function getUniqueSymbols(previousSymbols, currentSymbols) { return symbols; } -function joinSymbols(symbols, prependNewLine) { - if (symbols.length === 0) return ''; - return `${prependNewLine ? ',\n ' : ''}'${symbols.join("',\n '")}'`; +/** + * @param {string[]} strings + */ +function joinStrings(strings, prependNewLine = false) { + if (strings.length === 0) return ''; + return `${prependNewLine ? ',\n ' : ''}'${strings.join("',\n '")}'`; } -function getSymbolData() { - const { maxVersion, symbols } = getAllSymbols(); +async function getSymbolData() { + const { maxVersion, symbols } = await getAllSymbols(); let data = `'use strict' const v1 = { js_native_api_symbols: [ - ${joinSymbols(symbols.v1.js_native_api_symbols)} + ${joinStrings(symbols.v1.js_native_api_symbols)} ], node_api_symbols: [ - ${joinSymbols(symbols.v1.node_api_symbols)} + ${joinStrings(symbols.v1.node_api_symbols)} ] } `; @@ -111,10 +153,10 @@ const v1 = { data += ` const ${version === maxVersion + 1 ? 'experimental' : `v${version}`} = { js_native_api_symbols: [ - ...v${version - 1}.js_native_api_symbols${joinSymbols(newSymbols.js_native_api_symbols, true)} + ...v${version - 1}.js_native_api_symbols${joinStrings(newSymbols.js_native_api_symbols, true)} ], node_api_symbols: [ - ...v${version - 1}.node_api_symbols${joinSymbols(newSymbols.node_api_symbols, true)} + ...v${version - 1}.node_api_symbols${joinStrings(newSymbols.node_api_symbols, true)} ] } `; @@ -129,16 +171,14 @@ module.exports = { return data; } -function main() { - const path = resolve(__dirname, '../symbols.js'); - const data = getSymbolData(); +async function main() { + const path = resolvePath(__dirname, '../symbols.js'); + const data = await getSymbolData(); console.log(`Writing symbols to ${path}`) - writeFileSync(path, data); + return writeFile(path, data); } -try { - main(); -} catch (e) { +main().catch(e => { console.error(e); process.exitCode = 1; -} +}); From 4d032603be833dd47bdbfa7e0136df66cc763314 Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Tue, 3 Jan 2023 16:01:40 +0100 Subject: [PATCH 06/10] Remove ts-check comment, clean up jsdoc --- scripts/write-symbols.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/write-symbols.js b/scripts/write-symbols.js index 3cad531..2c8f51f 100644 --- a/scripts/write-symbols.js +++ b/scripts/write-symbols.js @@ -1,4 +1,3 @@ -// @ts-check 'use strict'; const { spawn } = require('child_process'); @@ -43,7 +42,7 @@ async function getSymbolsForVersion(version) { const ast = JSON.parse(stdout); - /** @type {{js_native_api_symbols: string[], node_api_symbols: string[]}} */ + /** @type {SymbolInfo} */ const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; for (const statement of ast.inner) { From 219ad9f098f1888b0f9c6dd9189c8baa6770eee4 Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Fri, 6 Jan 2023 19:01:56 +0100 Subject: [PATCH 07/10] Remove experimental from symbols.js --- scripts/write-symbols.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/write-symbols.js b/scripts/write-symbols.js index 2c8f51f..a881d99 100644 --- a/scripts/write-symbols.js +++ b/scripts/write-symbols.js @@ -7,14 +7,14 @@ const { writeFile } = require('fs/promises'); /** @typedef {{ js_native_api_symbols: string[]; node_api_symbols: string[]; }} SymbolInfo */ /** - * @param {number | undefined} [version] + * @param {number} [version] * @returns {Promise} */ async function getSymbolsForVersion(version) { try { const { exitCode, stdout, stderr } = await new Promise((resolve, reject) => { const spawned = spawn('clang', - ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', version ? `-DNAPI_VERSION=${version}` : '-DNAPI_EXPERIMENTAL', resolvePath(__dirname, '..', 'include', 'node_api.h')] + ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', `-DNAPI_VERSION=${version}`, resolvePath(__dirname, '..', 'include', 'node_api.h')] ); let stdout = ''; @@ -94,8 +94,6 @@ async function getAllSymbols() { ++version; } - const symbols = allSymbols[`experimental`] = await getSymbolsForVersion(); - console.log(` Experimental: ${symbols.js_native_api_symbols.length} js_native_api_symbols, ${symbols.node_api_symbols.length} node_api_symbols`); return { maxVersion: version, symbols: allSymbols @@ -146,11 +144,11 @@ const v1 = { } `; - for (let version = 2; version <= maxVersion + 1; ++version) { - const newSymbols = getUniqueSymbols(symbols[`v${version - 1}`], symbols[version === maxVersion + 1 ? 'experimental' : `v${version}`]); + for (let version = 2; version <= maxVersion; ++version) { + const newSymbols = getUniqueSymbols(symbols[`v${version - 1}`], symbols[`v${version}`]); data += ` -const ${version === maxVersion + 1 ? 'experimental' : `v${version}`} = { +const ${`v${version}`} = { js_native_api_symbols: [ ...v${version - 1}.js_native_api_symbols${joinStrings(newSymbols.js_native_api_symbols, true)} ], @@ -163,8 +161,7 @@ const ${version === maxVersion + 1 ? 'experimental' : `v${version}`} = { data += ` module.exports = { - ${new Array(maxVersion).fill(undefined).map((_, i) => `v${i + 1}`).join(',\n ')}, - experimental + ${new Array(maxVersion).fill(undefined).map((_, i) => `v${i + 1}`).join(',\n ')} } ` return data; From 41f07678bc62ea8acceeecff62f9306a3056103c Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Wed, 11 Jan 2023 00:38:12 +0100 Subject: [PATCH 08/10] Address review comments - Remove double string interpolation Co-authored-by: Chengzhong Wu --- scripts/write-symbols.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/write-symbols.js b/scripts/write-symbols.js index a881d99..a716c70 100644 --- a/scripts/write-symbols.js +++ b/scripts/write-symbols.js @@ -148,7 +148,7 @@ const v1 = { const newSymbols = getUniqueSymbols(symbols[`v${version - 1}`], symbols[`v${version}`]); data += ` -const ${`v${version}`} = { +const v${version} = { js_native_api_symbols: [ ...v${version - 1}.js_native_api_symbols${joinStrings(newSymbols.js_native_api_symbols, true)} ], From 807e927e457b8477e62aad4496b9c7c8a1408d52 Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Sat, 11 Feb 2023 22:59:35 +0530 Subject: [PATCH 09/10] Remove experimental APIs from headers --- scripts/update-headers.js | 119 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 4 deletions(-) diff --git a/scripts/update-headers.js b/scripts/update-headers.js index 13a1687..b4561f2 100644 --- a/scripts/update-headers.js +++ b/scripts/update-headers.js @@ -1,10 +1,12 @@ 'use strict'; -const { createWriteStream } = require('fs'); +const { writeFile } = require('fs/promises'); const { Readable } = require('stream'); -const { finished } = require('stream/promises'); const { resolve } = require('path'); const { parseArgs } = require('util') +const { createInterface } = require('readline'); +const { inspect } = require('util'); + async function getLatestReleaseVersion() { const response = await fetch('https://nodejs.org/download/release/index.json'); @@ -12,6 +14,116 @@ async function getLatestReleaseVersion() { return json[0].version; } +/** + * @param {NodeJS.ReadableStream} stream + * @param {string} destination + * @param {boolean} verbose + */ +function removeExperimentals(stream, destination, verbose) { + return new Promise((resolve, reject) => { + const debug = (...args) => { + if (verbose) { + console.log(...args); + } + }; + const rl = createInterface(stream); + + /** @type {Array<'write' | 'ignore'>} */ + let mode = ['write']; + + /** @type {Array} */ + const macroStack = []; + + /** @type {RegExpMatchArray | null} */ + let matches; + + let lineNumber = 0; + let toWrite = ""; + + rl.on('line', function lineHandler(line) { + ++lineNumber; + if (matches = line.match(/^\s*#if(n)?def\s+([A-Za-z_][A-Za-z0-9_]*)/)) { + const negated = Boolean(matches[1]); + const identifier = matches[2]; + macroStack.push(identifier); + + debug(`Line ${lineNumber} Pushed ${identifier}`); + + if (identifier === 'NAPI_EXPERIMENTAL') { + if (negated) { + mode.push('write'); + } else { + mode.push('ignore'); + } + return; + } else { + mode.push('write'); + } + + } + else if (matches = line.match(/^\s*#if\s+(.+)$/)) { + const identifier = matches[1]; + macroStack.push(identifier); + mode.push('write'); + + debug(`Line ${lineNumber} Pushed ${identifier}`); + } + else if (line.match(/^#else(?:\s+|$)/)) { + const identifier = macroStack[macroStack.length - 1]; + + debug(`Line ${lineNumber} Peeked ${identifier}`); + + if (!identifier) { + rl.off('line', lineHandler); + reject(new Error(`Macro stack is empty handling #else on line ${lineNumber}`)); + } + + if (identifier === 'NAPI_EXPERIMENTAL') { + const lastMode = mode[mode.length - 1]; + mode[mode.length - 1] = (lastMode === 'ignore') ? 'write' : 'ignore'; + return; + } + } + else if (line.match(/^\s*#endif(?:\s+|$)/)) { + const identifier = macroStack.pop(); + mode.pop(); + + debug(`Line ${lineNumber} Popped ${identifier}`); + + if (!identifier) { + rl.off('line', lineHandler); + reject(new Error(`Macro stack is empty handling #endif on line ${lineNumber}`)); + } + + if (identifier === 'NAPI_EXPERIMENTAL') { + return; + } + } + + if (mode.length === 0) { + rl.off('line', lineHandler); + reject(new Error(`Write mode empty handling #endif on line ${lineNumber}`)); + } + + if (mode[mode.length - 1] === 'write') { + toWrite += `${line}\n`; + } + }); + + rl.on('close', () => { + if (macroStack.length > 0) { + reject(new Error(`Macro stack is not empty at EOF: ${inspect(macroStack)}`)); + } + else if (mode.length > 1) { + reject(new Error(`Write mode greater than 1 at EOF: ${inspect(mode)}`)); + } + else { + resolve(writeFile(destination, toWrite)); + } + }); + }); +} + async function main() { const { values: { tag, verbose } } = parseArgs({ options: { @@ -44,8 +156,7 @@ async function main() { throw new Error(`Fetch of ${url} returned ${response.status} ${response.statusText}`); } - const stream = createWriteStream(path); - await finished(Readable.fromWeb(response.body).pipe(stream)); + await removeExperimentals(Readable.fromWeb(response.body), path, verbose) } } From 36a1e92b19642782e459d129657d5dbe761aa1ce Mon Sep 17 00:00:00 2001 From: Kevin Eady <8634912+KevinEady@users.noreply.github.com> Date: Mon, 13 Feb 2023 00:42:14 +0530 Subject: [PATCH 10/10] Some refactoring and cleanup --- scripts/clang-utils.js | 50 +++++++++++++++++++++++++++++++++++++++ scripts/update-headers.js | 39 +++++++++++++++++++++++------- scripts/write-symbols.js | 30 ++--------------------- 3 files changed, 83 insertions(+), 36 deletions(-) create mode 100644 scripts/clang-utils.js diff --git a/scripts/clang-utils.js b/scripts/clang-utils.js new file mode 100644 index 0000000..2f2b032 --- /dev/null +++ b/scripts/clang-utils.js @@ -0,0 +1,50 @@ +'use strict'; + +const { spawn } = require('child_process'); + +/** + * @param {Array} [args] + * @returns {Promise<{exitCode: number | null, stdout: string, stderr: string}>} + */ +async function runClang(args = []) { + try { + const { exitCode, stdout, stderr } = await new Promise((resolve, reject) => { + const spawned = spawn('clang', + ['-Xclang', ...args] + ); + + let stdout = ''; + let stderr = ''; + + spawned.stdout?.on('data', (data) => { + stdout += data.toString('utf-8'); + }); + spawned.stderr?.on('data', (data) => { + stderr += data.toString('utf-8'); + }); + + spawned.on('exit', function (exitCode) { + resolve({ exitCode, stdout, stderr }); + }); + + spawned.on('error', function (err) { + reject(err); + }); + }); + + if (exitCode !== 0) { + throw new Error(`clang exited with non-zero exit code ${exitCode}. stderr: ${stderr ? stderr : ''}`); + } + + return { exitCode, stdout, stderr }; + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error('This tool requires clang to be installed.'); + } + throw err; + } +} + +module.exports = { + runClang +}; diff --git a/scripts/update-headers.js b/scripts/update-headers.js index b4561f2..03e7c1f 100644 --- a/scripts/update-headers.js +++ b/scripts/update-headers.js @@ -6,8 +6,11 @@ const { resolve } = require('path'); const { parseArgs } = require('util') const { createInterface } = require('readline'); const { inspect } = require('util'); +const { runClang } = require('./clang-utils'); - +/** + * @returns {Promise} Version string, eg. `'v19.6.0'`. + */ async function getLatestReleaseVersion() { const response = await fetch('https://nodejs.org/download/release/index.json'); const json = await response.json(); @@ -18,8 +21,9 @@ async function getLatestReleaseVersion() { * @param {NodeJS.ReadableStream} stream * @param {string} destination * @param {boolean} verbose + * @returns {Promise} The `writeFile` Promise. */ -function removeExperimentals(stream, destination, verbose) { +function removeExperimentals(stream, destination, verbose = false) { return new Promise((resolve, reject) => { const debug = (...args) => { if (verbose) { @@ -38,7 +42,7 @@ function removeExperimentals(stream, destination, verbose) { let matches; let lineNumber = 0; - let toWrite = ""; + let toWrite = ''; rl.on('line', function lineHandler(line) { ++lineNumber; @@ -76,6 +80,7 @@ function removeExperimentals(stream, destination, verbose) { if (!identifier) { rl.off('line', lineHandler); reject(new Error(`Macro stack is empty handling #else on line ${lineNumber}`)); + return; } if (identifier === 'NAPI_EXPERIMENTAL') { @@ -103,6 +108,7 @@ function removeExperimentals(stream, destination, verbose) { if (mode.length === 0) { rl.off('line', lineHandler); reject(new Error(`Write mode empty handling #endif on line ${lineNumber}`)); + return; } if (mode[mode.length - 1] === 'write') { @@ -117,6 +123,9 @@ function removeExperimentals(stream, destination, verbose) { else if (mode.length > 1) { reject(new Error(`Write mode greater than 1 at EOF: ${inspect(mode)}`)); } + else if (toWrite.match(/^\s*#if(?:n)?def\s+NAPI_EXPERIMENTAL/m)) { + reject(new Error(`Output has match for NAPI_EXPERIMENTAL`)); + } else { resolve(writeFile(destination, toWrite)); } @@ -124,17 +133,29 @@ function removeExperimentals(stream, destination, verbose) { }); } +/** + * Validate syntax for a file using clang. + * @param {string} path Path for file to validate with clang. + */ +async function validateSyntax(path) { + try { + await runClang(['-fsyntax-only', path]); + } catch (e) { + throw new Error(`Syntax validation failed for ${path}: ${e}`); + } +} + async function main() { const { values: { tag, verbose } } = parseArgs({ options: { tag: { - type: "string", - short: "t", + type: 'string', + short: 't', default: await getLatestReleaseVersion() }, verbose: { - type: "boolean", - short: "v", + type: 'boolean', + short: 'v', }, }, }); @@ -156,7 +177,9 @@ async function main() { throw new Error(`Fetch of ${url} returned ${response.status} ${response.statusText}`); } - await removeExperimentals(Readable.fromWeb(response.body), path, verbose) + await removeExperimentals(Readable.fromWeb(response.body), path, verbose); + + await validateSyntax(path); } } diff --git a/scripts/write-symbols.js b/scripts/write-symbols.js index a716c70..d3621c1 100644 --- a/scripts/write-symbols.js +++ b/scripts/write-symbols.js @@ -1,8 +1,8 @@ 'use strict'; -const { spawn } = require('child_process'); const { resolve: resolvePath } = require('path'); const { writeFile } = require('fs/promises'); +const { runClang } = require('./clang-utils'); /** @typedef {{ js_native_api_symbols: string[]; node_api_symbols: string[]; }} SymbolInfo */ @@ -12,33 +12,7 @@ const { writeFile } = require('fs/promises'); */ async function getSymbolsForVersion(version) { try { - const { exitCode, stdout, stderr } = await new Promise((resolve, reject) => { - const spawned = spawn('clang', - ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', `-DNAPI_VERSION=${version}`, resolvePath(__dirname, '..', 'include', 'node_api.h')] - ); - - let stdout = ''; - let stderr = ''; - - spawned.stdout?.on('data', (data) => { - stdout += data.toString('utf-8'); - }); - spawned.stderr?.on('data', (data) => { - stderr += data.toString('utf-8'); - }); - - spawned.on('exit', function (exitCode) { - resolve({ exitCode, stdout, stderr }); - }); - - spawned.on('error', function (err) { - reject(err); - }); - }); - - if (exitCode !== 0) { - throw new Error(`clang exited with non-zero exit code ${exitCode}. stderr: ${stderr ? stderr : ''}`); - } + const { stdout } = await runClang([ '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', `-DNAPI_VERSION=${version}`, resolvePath(__dirname, '..', 'include', 'node_api.h')]) const ast = JSON.parse(stdout);