From b0a4d45e8532d8269e1a70378b44077279442a98 Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Sun, 12 May 2019 13:08:37 -0400 Subject: [PATCH 1/6] doc,tools: get altDocs versions from CHANGELOG.md Parse `CHANGELOG.md` for versions of Node.js used by the documentation feature `View another version` so that we don't have to manually update the list when we cut a new version or transition a release to LTS. Backport-PR-URL: https://github.com/nodejs/node/pull/32642 PR-URL: https://github.com/nodejs/node/pull/27661 Reviewed-By: Rich Trott --- test/internet/test-doctool-versions.js | 59 ++++++++++++++++++++++++++ tools/doc/html.js | 21 +++------ tools/doc/versions.js | 45 ++++++++++++++++++++ 3 files changed, 109 insertions(+), 16 deletions(-) create mode 100644 test/internet/test-doctool-versions.js create mode 100644 tools/doc/versions.js diff --git a/test/internet/test-doctool-versions.js b/test/internet/test-doctool-versions.js new file mode 100644 index 00000000000000..8bb4f81c795d95 --- /dev/null +++ b/test/internet/test-doctool-versions.js @@ -0,0 +1,59 @@ +'use strict'; + +require('../common'); +const assert = require('assert'); +const util = require('util'); +const { versions } = require('../../tools/doc/versions.js'); + +// At the time of writing these are the minimum expected versions. +// New versions of Node.js do not have to be explicitly added here. +const expected = [ + '12.x', + '11.x', + '10.x', + '9.x', + '8.x', + '7.x', + '6.x', + '5.x', + '4.x', + '0.12.x', + '0.10.x', +]; + +async function test() { + const vers = await versions(); + // Coherence checks for each returned version. + for (const version of vers) { + const tested = util.inspect(version); + const parts = version.num.split('.'); + const expectedLength = parts[0] === '0' ? 3 : 2; + assert.strictEqual(parts.length, expectedLength, + `'num' from ${tested} should be '.x'.`); + assert.strictEqual(parts[parts.length - 1], 'x', + `'num' from ${tested} doesn't end in '.x'.`); + const isEvenRelease = Number.parseInt(parts[expectedLength - 2]) % 2 === 0; + const hasLtsProperty = version.hasOwnProperty('lts'); + if (hasLtsProperty) { + // Odd-numbered versions of Node.js are never LTS. + assert.ok(isEvenRelease, `${tested} should not be an 'lts' release.`); + assert.ok(version.lts, `'lts' from ${tested} should 'true'.`); + } + } + + // Check that the minimum number of versions were returned. + // Later versions are allowed, but not checked for here (they were checked + // above). + // Also check for the previous semver major -- From master this will be the + // most recent major release. + const thisMajor = Number.parseInt(process.versions.node.split('.')[0]); + const prevMajorString = `${thisMajor - 1}.x`; + if (!expected.includes(prevMajorString)) { + expected.unshift(prevMajorString); + } + for (const version of expected) { + assert.ok(vers.find((x) => x.num === version), + `Did not find entry for '${version}' in ${util.inspect(vers)}`); + } +} +test(); diff --git a/tools/doc/html.js b/tools/doc/html.js index c9abae3818e771..4c969b14c52f0b 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -23,6 +23,7 @@ const common = require('./common.js'); const fs = require('fs'); +const getVersions = require('./versions.js'); const unified = require('unified'); const find = require('unist-util-find'); const visit = require('unist-util-visit'); @@ -62,7 +63,7 @@ const gtocHTML = unified() const templatePath = path.join(docPath, 'template.html'); const template = fs.readFileSync(templatePath, 'utf8'); -function toHTML({ input, content, filename, nodeVersion }, cb) { +async function toHTML({ input, content, filename, nodeVersion }, cb) { filename = path.basename(filename, '.md'); const id = filename.replace(/\W+/g, '-'); @@ -80,7 +81,7 @@ function toHTML({ input, content, filename, nodeVersion }, cb) { const docCreated = input.match( //); if (docCreated) { - HTML = HTML.replace('__ALTDOCS__', altDocs(filename, docCreated)); + HTML = HTML.replace('__ALTDOCS__', await altDocs(filename, docCreated)); } else { console.error(`Failed to add alternative version links to ${filename}`); HTML = HTML.replace('__ALTDOCS__', ''); @@ -380,22 +381,10 @@ function getId(text, idCounters) { return text; } -function altDocs(filename, docCreated) { +async function altDocs(filename, docCreated) { const [, docCreatedMajor, docCreatedMinor] = docCreated.map(Number); const host = 'https://nodejs.org'; - const versions = [ - { num: '12.x' }, - { num: '11.x' }, - { num: '10.x', lts: true }, - { num: '9.x' }, - { num: '8.x', lts: true }, - { num: '7.x' }, - { num: '6.x' }, - { num: '5.x' }, - { num: '4.x' }, - { num: '0.12.x' }, - { num: '0.10.x' } - ]; + const versions = await getVersions.versions(); const getHref = (versionNum) => `${host}/docs/latest-v${versionNum}/api/${filename}.html`; diff --git a/tools/doc/versions.js b/tools/doc/versions.js new file mode 100644 index 00000000000000..854329bd9a8a02 --- /dev/null +++ b/tools/doc/versions.js @@ -0,0 +1,45 @@ +'use strict'; + +let _versions; + +const getUrl = (url) => { + return new Promise((resolve, reject) => { + const https = require('https'); + const request = https.get(url, (response) => { + if (response.statusCode !== 200) { + reject(new Error( + `Failed to get ${url}, status code ${response.statusCode}`)); + } + response.setEncoding('utf8'); + let body = ''; + response.on('data', (data) => body += data); + response.on('end', () => resolve(body)); + }); + request.on('error', (err) => reject(err)); + }); +}; + +module.exports = { + async versions() { + if (_versions) { + return _versions; + } + + // The CHANGELOG.md on release branches may not reference newer semver + // majors of Node.js so fetch and parse the version from the master branch. + const githubContentUrl = 'https://raw.githubusercontent.com/nodejs/node/'; + const changelog = await getUrl(`${githubContentUrl}/master/CHANGELOG.md`); + const ltsRE = /Long Term Support/i; + const versionRE = /\* \[Node\.js ([0-9.]+)\][^-—]+[-—]\s*(.*)\n/g; + _versions = []; + let match; + while ((match = versionRE.exec(changelog)) != null) { + const entry = { num: `${match[1]}.x` }; + if (ltsRE.test(match[2])) { + entry.lts = true; + } + _versions.push(entry); + } + return _versions; + } +}; From ed69a5f9887d6b671aa0d15bab188623ccf94ee9 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Mon, 14 Oct 2019 23:15:28 -0700 Subject: [PATCH 2/6] test: fix flaky doctool and test Doctool tests have been failing a lot in CI on Win2008 R2. It appears async functions and callback-based functions are being used in combination such that the callback-based function cannot guarantee that it will invoke its callback. Convert the callback-based functions to async functions so we have one paradigm and reliable results. Backport-PR-URL: https://github.com/nodejs/node/pull/32642 PR-URL: https://github.com/nodejs/node/pull/29979 Reviewed-By: Joyee Cheung Reviewed-By: Gireesh Punathil Reviewed-By: Jeremiah Senkpiel --- test/doctool/test-doctool-html.js | 32 +++++++++++-------------------- tools/doc/generate.js | 17 ++++++---------- tools/doc/html.js | 4 ++-- 3 files changed, 19 insertions(+), 34 deletions(-) diff --git a/test/doctool/test-doctool-html.js b/test/doctool/test-doctool-html.js index 703a7dcd21acb0..11b28dc8a2a717 100644 --- a/test/doctool/test-doctool-html.js +++ b/test/doctool/test-doctool-html.js @@ -22,7 +22,7 @@ const remark2rehype = require('remark-rehype'); const raw = require('rehype-raw'); const htmlStringify = require('rehype-stringify'); -function toHTML({ input, filename, nodeVersion }, cb) { +async function toHTML({ input, filename, nodeVersion }) { const content = unified() .use(markdown) .use(html.firstHeader) @@ -34,10 +34,7 @@ function toHTML({ input, filename, nodeVersion }, cb) { .use(htmlStringify) .processSync(input); - html.toHTML( - { input, content, filename, nodeVersion }, - cb - ); + return html.toHTML({ input, content, filename, nodeVersion }); } // Test data is a list of objects with two properties. @@ -107,23 +104,16 @@ testData.forEach(({ file, html }) => { // Normalize expected data by stripping whitespace. const expected = html.replace(spaces, ''); - readFile(file, 'utf8', common.mustCall((err, input) => { + readFile(file, 'utf8', common.mustCall(async (err, input) => { assert.ifError(err); - toHTML( - { - input: input, - filename: 'foo', - nodeVersion: process.version, - }, - common.mustCall((err, output) => { - assert.ifError(err); + const output = await toHTML({ input: input, + filename: 'foo', + nodeVersion: process.version }); - const actual = output.replace(spaces, ''); - // Assert that the input stripped of all whitespace contains the - // expected markup. - assert(actual.includes(expected), - `ACTUAL: ${actual}\nEXPECTED: ${expected}`); - }) - ); + const actual = output.replace(spaces, ''); + // Assert that the input stripped of all whitespace contains the + // expected markup. + assert(actual.includes(expected), + `ACTUAL: ${actual}\nEXPECTED: ${expected}`); })); }); diff --git a/tools/doc/generate.js b/tools/doc/generate.js index dd213a35a6bbc4..9c0bc027ac531f 100644 --- a/tools/doc/generate.js +++ b/tools/doc/generate.js @@ -67,7 +67,7 @@ if (!filename) { } -fs.readFile(filename, 'utf8', (er, input) => { +fs.readFile(filename, 'utf8', async (er, input) => { if (er) throw er; const content = unified() @@ -84,15 +84,10 @@ fs.readFile(filename, 'utf8', (er, input) => { const basename = path.basename(filename, '.md'); - html.toHTML( - { input, content, filename, nodeVersion }, - (err, html) => { - const target = path.join(outputDir, `${basename}.html`); - if (err) throw err; - fs.writeFileSync(target, html); - } - ); + const myHtml = await html.toHTML({ input, content, filename, nodeVersion }); + const htmlTarget = path.join(outputDir, `${basename}.html`); + fs.writeFileSync(htmlTarget, myHtml); - const target = path.join(outputDir, `${basename}.json`); - fs.writeFileSync(target, JSON.stringify(content.json, null, 2)); + const jsonTarget = path.join(outputDir, `${basename}.json`); + fs.writeFileSync(jsonTarget, JSON.stringify(content.json, null, 2)); }); diff --git a/tools/doc/html.js b/tools/doc/html.js index 4c969b14c52f0b..12d9a541e17187 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -63,7 +63,7 @@ const gtocHTML = unified() const templatePath = path.join(docPath, 'template.html'); const template = fs.readFileSync(templatePath, 'utf8'); -async function toHTML({ input, content, filename, nodeVersion }, cb) { +async function toHTML({ input, content, filename, nodeVersion }) { filename = path.basename(filename, '.md'); const id = filename.replace(/\W+/g, '-'); @@ -87,7 +87,7 @@ async function toHTML({ input, content, filename, nodeVersion }, cb) { HTML = HTML.replace('__ALTDOCS__', ''); } - cb(null, HTML); + return HTML; } // Set the section name based on the first header. Default to 'Index'. From ac9127e0234a7db611b8987fa6182d18ce045b7a Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Sat, 2 Nov 2019 03:48:47 -0400 Subject: [PATCH 3/6] tools: make doctool work if no internet available Allow doctool to fallback to use local files if not building a release build. Backport-PR-URL: https://github.com/nodejs/node/pull/32642 PR-URL: https://github.com/nodejs/node/pull/30214 Fixes: https://github.com/nodejs/node/issues/29918 Reviewed-By: Rich Trott Reviewed-By: Gus Caplan Reviewed-By: Chengzhong Wu Reviewed-By: Joyee Cheung Reviewed-By: Ruben Bridgewater Reviewed-By: Jiawen Geng --- .../test-doctool-versions.js | 0 tools/doc/versions.js | 33 ++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) rename test/{internet => doctool}/test-doctool-versions.js (100%) diff --git a/test/internet/test-doctool-versions.js b/test/doctool/test-doctool-versions.js similarity index 100% rename from test/internet/test-doctool-versions.js rename to test/doctool/test-doctool-versions.js diff --git a/tools/doc/versions.js b/tools/doc/versions.js index 854329bd9a8a02..7a4e2c3ff76b1a 100644 --- a/tools/doc/versions.js +++ b/tools/doc/versions.js @@ -1,21 +1,33 @@ 'use strict'; +const { readFileSync } = require('fs'); +const path = require('path'); +const srcRoot = path.join(__dirname, '..', '..'); + let _versions; +const isRelease = () => { + const re = /#define NODE_VERSION_IS_RELEASE 0/; + const file = path.join(srcRoot, 'src', 'node_version.h'); + return !re.test(readFileSync(file, { encoding: 'utf8' })); +}; + const getUrl = (url) => { return new Promise((resolve, reject) => { const https = require('https'); - const request = https.get(url, (response) => { + const request = https.get(url, { timeout: 5000 }, (response) => { if (response.statusCode !== 200) { reject(new Error( `Failed to get ${url}, status code ${response.statusCode}`)); } response.setEncoding('utf8'); let body = ''; + response.on('aborted', () => reject()); response.on('data', (data) => body += data); response.on('end', () => resolve(body)); }); request.on('error', (err) => reject(err)); + request.on('timeout', () => request.abort()); }); }; @@ -27,10 +39,23 @@ module.exports = { // The CHANGELOG.md on release branches may not reference newer semver // majors of Node.js so fetch and parse the version from the master branch. - const githubContentUrl = 'https://raw.githubusercontent.com/nodejs/node/'; - const changelog = await getUrl(`${githubContentUrl}/master/CHANGELOG.md`); + const url = + 'https://raw.githubusercontent.com/nodejs/node/master/CHANGELOG.md'; + let changelog; + try { + changelog = await getUrl(url); + } catch (e) { + // Fail if this is a release build, otherwise fallback to local files. + if (isRelease()) { + throw e; + } else { + const file = path.join(srcRoot, 'CHANGELOG.md'); + console.warn(`Unable to retrieve ${url}. Falling back to ${file}.`); + changelog = readFileSync(file, { encoding: 'utf8' }); + } + } const ltsRE = /Long Term Support/i; - const versionRE = /\* \[Node\.js ([0-9.]+)\][^-—]+[-—]\s*(.*)\n/g; + const versionRE = /\* \[Node\.js ([0-9.]+)\][^-—]+[-—]\s*(.*)\r?\n/g; _versions = []; let match; while ((match = versionRE.exec(changelog)) != null) { From 2c3c6a090ec75c7dd60e5d2ad5ae83af12a604bb Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 18 Feb 2020 23:01:21 +0800 Subject: [PATCH 4/6] tools: add NODE_TEST_NO_INTERNET to the doc builder At the moment the doc builder tries to access the internet for CHANGELOG information and only falls back to local sources after the connection fails or a 5 second timeout. This means that the doc building could take at least 7 minutes on a machine with hijacked connection to Github for useless network attempts. This patch adds a NODE_TEST_NO_INTERNET environment variable to directly bypass these attempts so that docs can be built in reasonable time on a machine like that. Backport-PR-URL: https://github.com/nodejs/node/pull/32642 PR-URL: https://github.com/nodejs/node/pull/31849 Fixes: https://github.com/nodejs/node/issues/29918 Reviewed-By: Matheus Marchini Reviewed-By: Richard Lau Reviewed-By: Ruben Bridgewater Reviewed-By: Colin Ihrig Reviewed-By: Luigi Pinca --- tools/doc/versions.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/doc/versions.js b/tools/doc/versions.js index 7a4e2c3ff76b1a..782ce90ee2c616 100644 --- a/tools/doc/versions.js +++ b/tools/doc/versions.js @@ -31,6 +31,8 @@ const getUrl = (url) => { }); }; +const kNoInternet = !!process.env.NODE_TEST_NO_INTERNET; + module.exports = { async versions() { if (_versions) { @@ -42,16 +44,20 @@ module.exports = { const url = 'https://raw.githubusercontent.com/nodejs/node/master/CHANGELOG.md'; let changelog; - try { - changelog = await getUrl(url); - } catch (e) { - // Fail if this is a release build, otherwise fallback to local files. - if (isRelease()) { - throw e; - } else { - const file = path.join(srcRoot, 'CHANGELOG.md'); - console.warn(`Unable to retrieve ${url}. Falling back to ${file}.`); - changelog = readFileSync(file, { encoding: 'utf8' }); + const file = path.join(srcRoot, 'CHANGELOG.md'); + if (kNoInternet) { + changelog = readFileSync(file, { encoding: 'utf8' }); + } else { + try { + changelog = await getUrl(url); + } catch (e) { + // Fail if this is a release build, otherwise fallback to local files. + if (isRelease()) { + throw e; + } else { + console.warn(`Unable to retrieve ${url}. Falling back to ${file}.`); + changelog = readFileSync(file, { encoding: 'utf8' }); + } } } const ltsRE = /Long Term Support/i; From df5d115a615a8f4c4618fb542502de0ef5d44d27 Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Fri, 27 Mar 2020 10:04:40 -0400 Subject: [PATCH 5/6] tools: only fetch previous versions when necessary Refactor the logic for working out the previous versions of Node.js for the API documentation so that the parsing (including the potential https get) happens at most once per build (as opposed to the current once per generated API doc). Signed-off-by: Richard Lau Backport-PR-URL: https://github.com/nodejs/node/pull/32642 PR-URL: https://github.com/nodejs/node/pull/32518 Fixes: https://github.com/nodejs/node/issues/32512 Reviewed-By: Joyee Cheung Reviewed-By: Myles Borins --- Makefile | 11 +++- test/doctool/test-doctool-html.js | 21 +++++-- test/doctool/test-doctool-versions.js | 83 +++++++++++++++----------- tools/doc/generate.js | 11 +++- tools/doc/html.js | 8 +-- tools/doc/versions.js | 86 ++++++++++++++------------- 6 files changed, 132 insertions(+), 88 deletions(-) diff --git a/Makefile b/Makefile index 82eb9435a2af02..3fd6d974ed74c1 100644 --- a/Makefile +++ b/Makefile @@ -649,15 +649,22 @@ out/doc/api/assets/%: doc/api_assets/% out/doc/api/assets run-npm-ci = $(PWD)/$(NPM) ci LINK_DATA = out/doc/apilinks.json +VERSIONS_DATA = out/doc/previous-versions.json gen-api = tools/doc/generate.js --node-version=$(FULLVERSION) \ - --apilinks=$(LINK_DATA) $< --output-directory=out/doc/api + --apilinks=$(LINK_DATA) $< --output-directory=out/doc/api \ + --versions-file=$(VERSIONS_DATA) gen-apilink = tools/doc/apilinks.js $(LINK_DATA) $(wildcard lib/*.js) $(LINK_DATA): $(wildcard lib/*.js) tools/doc/apilinks.js $(call available-node, $(gen-apilink)) +# Regenerate previous versions data if the current version changes +$(VERSIONS_DATA): CHANGELOG.md src/node_version.h tools/doc/versions.js + $(call available-node, tools/doc/versions.js $@) + out/doc/api/%.json out/doc/api/%.html: doc/api/%.md tools/doc/generate.js \ - tools/doc/html.js tools/doc/json.js tools/doc/apilinks.js | $(LINK_DATA) + tools/doc/html.js tools/doc/json.js tools/doc/apilinks.js \ + $(VERSIONS_DATA) | $(LINK_DATA) $(call available-node, $(gen-api)) out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js \ diff --git a/test/doctool/test-doctool-html.js b/test/doctool/test-doctool-html.js index 11b28dc8a2a717..b128a379d9e27c 100644 --- a/test/doctool/test-doctool-html.js +++ b/test/doctool/test-doctool-html.js @@ -22,7 +22,7 @@ const remark2rehype = require('remark-rehype'); const raw = require('rehype-raw'); const htmlStringify = require('rehype-stringify'); -async function toHTML({ input, filename, nodeVersion }) { +function toHTML({ input, filename, nodeVersion, versions }) { const content = unified() .use(markdown) .use(html.firstHeader) @@ -34,7 +34,7 @@ async function toHTML({ input, filename, nodeVersion }) { .use(htmlStringify) .processSync(input); - return html.toHTML({ input, content, filename, nodeVersion }); + return html.toHTML({ input, content, filename, nodeVersion, versions }); } // Test data is a list of objects with two properties. @@ -99,6 +99,16 @@ const testData = [ ]; const spaces = /\s/g; +const versions = [ + { num: '10.x', lts: true }, + { num: '9.x' }, + { num: '8.x' }, + { num: '7.x' }, + { num: '6.x' }, + { num: '5.x' }, + { num: '4.x' }, + { num: '0.12.x' }, + { num: '0.10.x' }]; testData.forEach(({ file, html }) => { // Normalize expected data by stripping whitespace. @@ -106,9 +116,10 @@ testData.forEach(({ file, html }) => { readFile(file, 'utf8', common.mustCall(async (err, input) => { assert.ifError(err); - const output = await toHTML({ input: input, - filename: 'foo', - nodeVersion: process.version }); + const output = toHTML({ input: input, + filename: 'foo', + nodeVersion: process.version, + versions: versions }); const actual = output.replace(spaces, ''); // Assert that the input stripped of all whitespace contains the diff --git a/test/doctool/test-doctool-versions.js b/test/doctool/test-doctool-versions.js index 8bb4f81c795d95..a37ead7c0adb5b 100644 --- a/test/doctool/test-doctool-versions.js +++ b/test/doctool/test-doctool-versions.js @@ -2,8 +2,14 @@ require('../common'); const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const tmpdir = require('../common/tmpdir'); const util = require('util'); -const { versions } = require('../../tools/doc/versions.js'); + +const debuglog = util.debuglog('test'); +const versionsTool = path.join('../../tools/doc/versions.js'); // At the time of writing these are the minimum expected versions. // New versions of Node.js do not have to be explicitly added here. @@ -21,39 +27,48 @@ const expected = [ '0.10.x', ]; -async function test() { - const vers = await versions(); - // Coherence checks for each returned version. - for (const version of vers) { - const tested = util.inspect(version); - const parts = version.num.split('.'); - const expectedLength = parts[0] === '0' ? 3 : 2; - assert.strictEqual(parts.length, expectedLength, - `'num' from ${tested} should be '.x'.`); - assert.strictEqual(parts[parts.length - 1], 'x', - `'num' from ${tested} doesn't end in '.x'.`); - const isEvenRelease = Number.parseInt(parts[expectedLength - 2]) % 2 === 0; - const hasLtsProperty = version.hasOwnProperty('lts'); - if (hasLtsProperty) { - // Odd-numbered versions of Node.js are never LTS. - assert.ok(isEvenRelease, `${tested} should not be an 'lts' release.`); - assert.ok(version.lts, `'lts' from ${tested} should 'true'.`); - } - } +tmpdir.refresh(); +const versionsFile = path.join(tmpdir.path, 'versions.json'); +debuglog(versionsFile); +const opts = { cwd: tmpdir.path, encoding: 'utf8' }; +const cp = spawnSync(process.execPath, [ versionsTool, versionsFile ], opts); +debuglog(cp.stderr); +debuglog(cp.stdout); +assert.strictEqual(cp.stdout, ''); +assert.strictEqual(cp.signal, null); +assert.strictEqual(cp.status, 0); +const versions = JSON.parse(fs.readFileSync(versionsFile)); +debuglog(versions); - // Check that the minimum number of versions were returned. - // Later versions are allowed, but not checked for here (they were checked - // above). - // Also check for the previous semver major -- From master this will be the - // most recent major release. - const thisMajor = Number.parseInt(process.versions.node.split('.')[0]); - const prevMajorString = `${thisMajor - 1}.x`; - if (!expected.includes(prevMajorString)) { - expected.unshift(prevMajorString); - } - for (const version of expected) { - assert.ok(vers.find((x) => x.num === version), - `Did not find entry for '${version}' in ${util.inspect(vers)}`); +// Coherence checks for each returned version. +for (const version of versions) { + const tested = util.inspect(version); + const parts = version.num.split('.'); + const expectedLength = parts[0] === '0' ? 3 : 2; + assert.strictEqual(parts.length, expectedLength, + `'num' from ${tested} should be '.x'.`); + assert.strictEqual(parts[parts.length - 1], 'x', + `'num' from ${tested} doesn't end in '.x'.`); + const isEvenRelease = Number.parseInt(parts[expectedLength - 2]) % 2 === 0; + const hasLtsProperty = version.hasOwnProperty('lts'); + if (hasLtsProperty) { + // Odd-numbered versions of Node.js are never LTS. + assert.ok(isEvenRelease, `${tested} should not be an 'lts' release.`); + assert.ok(version.lts, `'lts' from ${tested} should 'true'.`); } } -test(); + +// Check that the minimum number of versions were returned. +// Later versions are allowed, but not checked for here (they were checked +// above). +// Also check for the previous semver major -- From master this will be the +// most recent major release. +const thisMajor = Number.parseInt(process.versions.node.split('.')[0]); +const prevMajorString = `${thisMajor - 1}.x`; +if (!expected.includes(prevMajorString)) { + expected.unshift(prevMajorString); +} +for (const version of expected) { + assert.ok(versions.find((x) => x.num === version), + `Did not find entry for '${version}' in ${util.inspect(versions)}`); +} diff --git a/tools/doc/generate.js b/tools/doc/generate.js index 9c0bc027ac531f..3127023511e248 100644 --- a/tools/doc/generate.js +++ b/tools/doc/generate.js @@ -40,6 +40,7 @@ let filename = null; let nodeVersion = null; let outputDir = null; let apilinks = {}; +let versions = {}; args.forEach(function(arg) { if (!arg.startsWith('--')) { @@ -55,6 +56,13 @@ args.forEach(function(arg) { throw new Error(`${linkFile} is empty`); } apilinks = JSON.parse(data); + } else if (arg.startsWith('--versions-file=')) { + const versionsFile = arg.replace(/^--versions-file=/, ''); + const data = fs.readFileSync(versionsFile, 'utf8'); + if (!data.trim()) { + throw new Error(`${versionsFile} is empty`); + } + versions = JSON.parse(data); } }); @@ -84,7 +92,8 @@ fs.readFile(filename, 'utf8', async (er, input) => { const basename = path.basename(filename, '.md'); - const myHtml = await html.toHTML({ input, content, filename, nodeVersion }); + const myHtml = html.toHTML({ input, content, filename, nodeVersion, + versions }); const htmlTarget = path.join(outputDir, `${basename}.html`); fs.writeFileSync(htmlTarget, myHtml); diff --git a/tools/doc/html.js b/tools/doc/html.js index 12d9a541e17187..c357d3a408ea96 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -23,7 +23,6 @@ const common = require('./common.js'); const fs = require('fs'); -const getVersions = require('./versions.js'); const unified = require('unified'); const find = require('unist-util-find'); const visit = require('unist-util-visit'); @@ -63,7 +62,7 @@ const gtocHTML = unified() const templatePath = path.join(docPath, 'template.html'); const template = fs.readFileSync(templatePath, 'utf8'); -async function toHTML({ input, content, filename, nodeVersion }) { +function toHTML({ input, content, filename, nodeVersion, versions }) { filename = path.basename(filename, '.md'); const id = filename.replace(/\W+/g, '-'); @@ -81,7 +80,7 @@ async function toHTML({ input, content, filename, nodeVersion }) { const docCreated = input.match( //); if (docCreated) { - HTML = HTML.replace('__ALTDOCS__', await altDocs(filename, docCreated)); + HTML = HTML.replace('__ALTDOCS__', altDocs(filename, docCreated, versions)); } else { console.error(`Failed to add alternative version links to ${filename}`); HTML = HTML.replace('__ALTDOCS__', ''); @@ -381,10 +380,9 @@ function getId(text, idCounters) { return text; } -async function altDocs(filename, docCreated) { +function altDocs(filename, docCreated, versions) { const [, docCreatedMajor, docCreatedMinor] = docCreated.map(Number); const host = 'https://nodejs.org'; - const versions = await getVersions.versions(); const getHref = (versionNum) => `${host}/docs/latest-v${versionNum}/api/${filename}.html`; diff --git a/tools/doc/versions.js b/tools/doc/versions.js index 782ce90ee2c616..52f5648ecae92f 100644 --- a/tools/doc/versions.js +++ b/tools/doc/versions.js @@ -1,11 +1,9 @@ 'use strict'; -const { readFileSync } = require('fs'); +const { readFileSync, writeFileSync } = require('fs'); const path = require('path'); const srcRoot = path.join(__dirname, '..', '..'); -let _versions; - const isRelease = () => { const re = /#define NODE_VERSION_IS_RELEASE 0/; const file = path.join(srcRoot, 'src', 'node_version.h'); @@ -15,7 +13,7 @@ const isRelease = () => { const getUrl = (url) => { return new Promise((resolve, reject) => { const https = require('https'); - const request = https.get(url, { timeout: 5000 }, (response) => { + const request = https.get(url, { timeout: 30000 }, (response) => { if (response.statusCode !== 200) { reject(new Error( `Failed to get ${url}, status code ${response.statusCode}`)); @@ -32,45 +30,51 @@ const getUrl = (url) => { }; const kNoInternet = !!process.env.NODE_TEST_NO_INTERNET; +const outFile = (process.argv.length > 2 ? process.argv[2] : undefined); -module.exports = { - async versions() { - if (_versions) { - return _versions; - } - - // The CHANGELOG.md on release branches may not reference newer semver - // majors of Node.js so fetch and parse the version from the master branch. - const url = - 'https://raw.githubusercontent.com/nodejs/node/master/CHANGELOG.md'; - let changelog; - const file = path.join(srcRoot, 'CHANGELOG.md'); - if (kNoInternet) { - changelog = readFileSync(file, { encoding: 'utf8' }); - } else { - try { - changelog = await getUrl(url); - } catch (e) { - // Fail if this is a release build, otherwise fallback to local files. - if (isRelease()) { - throw e; - } else { - console.warn(`Unable to retrieve ${url}. Falling back to ${file}.`); - changelog = readFileSync(file, { encoding: 'utf8' }); - } +async function versions() { + // The CHANGELOG.md on release branches may not reference newer semver + // majors of Node.js so fetch and parse the version from the master branch. + const url = + 'https://raw.githubusercontent.com/nodejs/node/master/CHANGELOG.md'; + let changelog; + const file = path.join(srcRoot, 'CHANGELOG.md'); + if (kNoInternet) { + changelog = readFileSync(file, { encoding: 'utf8' }); + } else { + try { + changelog = await getUrl(url); + } catch (e) { + // Fail if this is a release build, otherwise fallback to local files. + if (isRelease()) { + throw e; + } else { + console.warn(`Unable to retrieve ${url}. Falling back to ${file}.`); + changelog = readFileSync(file, { encoding: 'utf8' }); } } - const ltsRE = /Long Term Support/i; - const versionRE = /\* \[Node\.js ([0-9.]+)\][^-—]+[-—]\s*(.*)\r?\n/g; - _versions = []; - let match; - while ((match = versionRE.exec(changelog)) != null) { - const entry = { num: `${match[1]}.x` }; - if (ltsRE.test(match[2])) { - entry.lts = true; - } - _versions.push(entry); + } + const ltsRE = /Long Term Support/i; + const versionRE = /\* \[Node\.js ([0-9.]+)\]\S+ (.*)\r?\n/g; + const _versions = []; + let match; + while ((match = versionRE.exec(changelog)) != null) { + const entry = { num: `${match[1]}.x` }; + if (ltsRE.test(match[2])) { + entry.lts = true; } - return _versions; + _versions.push(entry); } -}; + return _versions; +} + +versions().then((v) => { + if (outFile) { + writeFileSync(outFile, JSON.stringify(v)); + } else { + console.log(JSON.stringify(v)); + } +}).catch((err) => { + console.error(err); + process.exit(1); +}); From 91ea203144731e525bd63c9c76b19ea73ca9243b Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Fri, 3 Apr 2020 14:05:29 -0400 Subject: [PATCH 6/6] test: fix tool path in test-doctool-versions.js Path to the versions tool tested by test-doctool-versions.js would be incorrect if the test temporary directory was redirected (e.g. via NODE_TEST_DIR) outside of `test/`. Signed-off-by: Richard Lau Backport-PR-URL: https://github.com/nodejs/node/pull/32642 PR-URL: https://github.com/nodejs/node/pull/32645 Refs: https://github.com/nodejs/node/pull/32518 Reviewed-By: Colin Ihrig Reviewed-By: Shelley Vohr Reviewed-By: Anna Henningsen --- test/doctool/test-doctool-versions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/doctool/test-doctool-versions.js b/test/doctool/test-doctool-versions.js index a37ead7c0adb5b..5673cc79402df7 100644 --- a/test/doctool/test-doctool-versions.js +++ b/test/doctool/test-doctool-versions.js @@ -9,7 +9,7 @@ const tmpdir = require('../common/tmpdir'); const util = require('util'); const debuglog = util.debuglog('test'); -const versionsTool = path.join('../../tools/doc/versions.js'); +const versionsTool = path.resolve(__dirname, '../../tools/doc/versions.js'); // At the time of writing these are the minimum expected versions. // New versions of Node.js do not have to be explicitly added here. @@ -29,7 +29,7 @@ const expected = [ tmpdir.refresh(); const versionsFile = path.join(tmpdir.path, 'versions.json'); -debuglog(versionsFile); +debuglog(`${process.execPath} ${versionsTool} ${versionsFile}`); const opts = { cwd: tmpdir.path, encoding: 'utf8' }; const cp = spawnSync(process.execPath, [ versionsTool, versionsFile ], opts); debuglog(cp.stderr);