From 86bee2a843485e8c65447cb3c00bde24e7a92011 Mon Sep 17 00:00:00 2001 From: Ron Buckton Date: Tue, 23 Jun 2020 12:59:09 -0700 Subject: [PATCH 1/2] Update failed test tracking to support Mocha 6+ --- .dockerignore | 1 + .gitignore | 1 + .npmignore | 1 + Gulpfile.js | 10 ++-- scripts/build/tests.js | 5 +- scripts/failed-tests.js | 21 +++---- scripts/run-failed-tests.js | 97 --------------------------------- src/testRunner/parallel/host.ts | 2 +- 8 files changed, 24 insertions(+), 114 deletions(-) delete mode 100644 scripts/run-failed-tests.js diff --git a/.dockerignore b/.dockerignore index cc908c0036a3c..4c123cfd1898c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,6 +42,7 @@ yarn.lock yarn-error.log .parallelperf.* .failed-tests +.failed-tests.json TEST-results.xml package-lock.json tests diff --git a/.gitignore b/.gitignore index d2bfafe567fb5..b3e12d3d89666 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ tests/cases/user/*/**/*.d.ts !tests/cases/user/discord.js/ tests/baselines/reference/dt .failed-tests +.failed-tests.json TEST-results.xml package-lock.json tests/cases/user/TypeScript-React-Starter/TypeScript-React-Starter diff --git a/.npmignore b/.npmignore index 7a8f8d5129113..29026e7e52805 100644 --- a/.npmignore +++ b/.npmignore @@ -14,6 +14,7 @@ Jakefile.js .eslintignore .editorconfig .failed-tests +.failed-tests.json .git .git/ .gitattributes diff --git a/Gulpfile.js b/Gulpfile.js index 095ea5025c4d0..bf1a259511813 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -454,11 +454,11 @@ task("runtests", series(preBuild, preTest, runTests, postTest)); task("runtests").description = "Runs the tests using the built run.js file."; task("runtests").flags = { "-t --tests=": "Pattern for tests to run.", - " --failed": "Runs tests listed in '.failed-tests'.", + " --failed": "Runs tests listed in '.failed-tests.json'.", "-r --reporter=": "The mocha reporter to use.", "-d --debug": "Runs tests in debug mode (NodeJS 6 and earlier)", "-i --inspect": "Runs tests in inspector mode (NodeJS 8 and later)", - " --keepFailed": "Keep tests in .failed-tests even if they pass", + " --keepFailed": "Keep tests in '.failed-tests.json even if they pass", " --light": "Run tests in light mode (fewer verifications, but tests run faster)", " --dirty": "Run tests without first cleaning test output directories", " --stackTraceLimit=": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.", @@ -476,7 +476,7 @@ task("runtests-parallel").description = "Runs all the tests in parallel using th task("runtests-parallel").flags = { " --no-lint": "disables lint.", " --light": "Run tests in light mode (fewer verifications, but tests run faster).", - " --keepFailed": "Keep tests in .failed-tests even if they pass.", + " --keepFailed": "Keep tests in '.failed-tests.json' even if they pass.", " --dirty": "Run tests without first cleaning test output directories.", " --stackTraceLimit=": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.", " --workers=": "The number of parallel workers to use.", @@ -634,9 +634,9 @@ task("watch", series(preBuild, preTest, parallel(watchLib, watchDiagnostics, wat task("watch").description = "Watches for changes and rebuilds and runs tests in parallel."; task("watch").flags = { "-t --tests=": "Pattern for tests to run. Forces tests to be run in a single worker.", - " --failed": "Runs tests listed in '.failed-tests'. Forces tests to be run in a single worker.", + " --failed": "Runs tests listed in '.failed-tests.json'. Forces tests to be run in a single worker.", "-r --reporter=": "The mocha reporter to use.", - " --keepFailed": "Keep tests in .failed-tests even if they pass", + " --keepFailed": "Keep tests in '.failed-tests.json' even if they pass", " --light": "Run tests in light mode (fewer verifications, but tests run faster)", " --dirty": "Run tests without first cleaning test output directories", " --stackTraceLimit=": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.", diff --git a/scripts/build/tests.js b/scripts/build/tests.js index 5c4bcfb07dbfc..63972d260eb12 100644 --- a/scripts/build/tests.js +++ b/scripts/build/tests.js @@ -77,12 +77,15 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode, // timeout normally isn"t necessary but Travis-CI has been timing out on compiler baselines occasionally // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer if (!runInParallel) { - args.push(failed ? "scripts/run-failed-tests.js" : mochaJs); + args.push(mochaJs); args.push("-R", "scripts/failed-tests"); args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"'); if (tests) { args.push("-g", `"${tests}"`); } + if (failed) { + args.push("--config", ".failed-tests.json"); + } if (colors) { args.push("--colors"); } diff --git a/scripts/failed-tests.js b/scripts/failed-tests.js index 66463e56c7c2d..63a2513cc78bb 100644 --- a/scripts/failed-tests.js +++ b/scripts/failed-tests.js @@ -7,7 +7,7 @@ const os = require("os"); const failingHookRegExp = /^(.*) "(before|after) (all|each)" hook$/; /** - * .failed-tests reporter + * .failed-tests.json reporter * * @typedef {Object} ReporterOptions * @property {string} [file] @@ -25,7 +25,7 @@ class FailedTestsReporter extends Mocha.reporters.Base { if (!runner) return; const reporterOptions = this.reporterOptions = options.reporterOptions || {}; - if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests"; + if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests.json"; if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false; if (reporterOptions.reporter) { /** @type {Mocha.ReporterConstructor} */ @@ -70,7 +70,7 @@ class FailedTestsReporter extends Mocha.reporters.Base { * @param {(err?: NodeJS.ErrnoException) => void} done */ static writeFailures(file, passes, failures, keepFailed, done) { - const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined); + const failingTests = new Set(readTests()); const possiblyPassingSuites = /**@type {Set}*/(new Set()); // Remove tests that are now passing and track suites that are now @@ -104,9 +104,8 @@ class FailedTestsReporter extends Mocha.reporters.Base { if (failingTests.size > 0) { const failed = Array .from(failingTests) - .sort() - .join(os.EOL); - fs.writeFile(file, failed, "utf8", done); + .sort(); + fs.writeFile(file, JSON.stringify({ grep: failed }, undefined, " "), "utf8", done); } else if (!keepFailed && fs.existsSync(file)) { fs.unlink(file, done); @@ -116,10 +115,12 @@ class FailedTestsReporter extends Mocha.reporters.Base { } function readTests() { - return fs.readFileSync(file, "utf8") - .split(/\r?\n/g) - .map(line => line.trim()) - .filter(line => line.length > 0); + try { + return JSON.parse(fs.readFileSync(file, "utf8")).grep; + } + catch { + return undefined; + } } } diff --git a/scripts/run-failed-tests.js b/scripts/run-failed-tests.js deleted file mode 100644 index bde06df11cb0e..0000000000000 --- a/scripts/run-failed-tests.js +++ /dev/null @@ -1,97 +0,0 @@ -const spawn = require('child_process').spawn; -const os = require("os"); -const fs = require("fs"); -const path = require("path"); - -let grep; -try { - const failedTests = fs.readFileSync(".failed-tests", "utf8"); - grep = failedTests - .split(/\r?\n/g) - .map(test => test.trim()) - .filter(test => test.length > 0) - .map(escapeRegExp); -} -catch (e) { - grep = []; -} - -let args = []; -let waitForGrepValue = false; -let grepIndex = -1; -process.argv.slice(2).forEach((arg, index) => { - const [flag, value] = arg.split('='); - if (flag === "g" || flag === "grep") { - grepIndex = index - 1; - waitForGrepValue = arg !== flag; - if (!waitForGrepValue) grep.push(value.replace(/^"|"$/g, "")); - return; - } - if (waitForGrepValue) { - grep.push(arg.replace(/^"|"$/g, "")); - waitForGrepValue = false; - return; - } - args.push(arg); -}); - -let mocha = "./node_modules/mocha/bin/mocha"; -let grepOption; -let grepOptionValue; -let grepFile; -if (grep.length) { - grepOption = "--grep"; - grepOptionValue = grep.join("|"); - if (grepOptionValue.length > 20) { - grepFile = path.resolve(os.tmpdir(), ".failed-tests.opts"); - fs.writeFileSync(grepFile, `--grep ${grepOptionValue}`, "utf8"); - grepOption = "--opts"; - grepOptionValue = grepFile; - mocha = "./node_modules/mocha/bin/_mocha"; - } -} - -if (grepOption) { - if (grepIndex >= 0) { - args.splice(grepIndex, 0, grepOption, grepOptionValue); - } - else { - args.push(grepOption, grepOptionValue); - } -} - -args.unshift(path.resolve(mocha)); - -console.log(args.join(" ")); -const proc = spawn(process.execPath, args, { - stdio: 'inherit' -}); -proc.on('exit', (code, signal) => { - process.on('exit', () => { - if (grepFile) { - try { - fs.unlinkSync(grepFile); - } - catch (e) { - if (e.code !== "ENOENT") throw e; - } - } - - if (signal) { - process.kill(process.pid, signal); - } else { - process.exit(code); - } - }); -}); - -process.on('SIGINT', () => { - proc.kill('SIGINT'); - proc.kill('SIGTERM'); -}); - -function escapeRegExp(pattern) { - return pattern - .replace(/[^-\w\d\s]/g, match => "\\" + match) - .replace(/\s/g, "\\s"); -} \ No newline at end of file diff --git a/src/testRunner/parallel/host.ts b/src/testRunner/parallel/host.ts index bb4b31b9f058d..eda1e5e296f86 100644 --- a/src/testRunner/parallel/host.ts +++ b/src/testRunner/parallel/host.ts @@ -548,7 +548,7 @@ namespace Harness.Parallel.Host { else { failedTestReporter = new FailedTestReporter(replayRunner, { reporterOptions: { - file: path.resolve(".failed-tests"), + file: path.resolve(".failed-tests.json"), keepFailed: Harness.keepFailed // eslint-disable-line @typescript-eslint/no-unnecessary-qualifier } }); From 62af525a98473cd5747c79856fc04abc6465af97 Mon Sep 17 00:00:00 2001 From: Ron Buckton Date: Tue, 23 Jun 2020 13:54:41 -0700 Subject: [PATCH 2/2] grep should not be an array --- .dockerignore | 1 - .gitignore | 1 - .npmignore | 1 - Gulpfile.js | 10 +++++----- scripts/build/tests.js | 14 +++++++++++++- scripts/failed-tests.js | 21 ++++++++++----------- src/testRunner/parallel/host.ts | 2 +- 7 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4c123cfd1898c..cc908c0036a3c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,7 +42,6 @@ yarn.lock yarn-error.log .parallelperf.* .failed-tests -.failed-tests.json TEST-results.xml package-lock.json tests diff --git a/.gitignore b/.gitignore index b3e12d3d89666..d2bfafe567fb5 100644 --- a/.gitignore +++ b/.gitignore @@ -83,7 +83,6 @@ tests/cases/user/*/**/*.d.ts !tests/cases/user/discord.js/ tests/baselines/reference/dt .failed-tests -.failed-tests.json TEST-results.xml package-lock.json tests/cases/user/TypeScript-React-Starter/TypeScript-React-Starter diff --git a/.npmignore b/.npmignore index 29026e7e52805..7a8f8d5129113 100644 --- a/.npmignore +++ b/.npmignore @@ -14,7 +14,6 @@ Jakefile.js .eslintignore .editorconfig .failed-tests -.failed-tests.json .git .git/ .gitattributes diff --git a/Gulpfile.js b/Gulpfile.js index bf1a259511813..095ea5025c4d0 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -454,11 +454,11 @@ task("runtests", series(preBuild, preTest, runTests, postTest)); task("runtests").description = "Runs the tests using the built run.js file."; task("runtests").flags = { "-t --tests=": "Pattern for tests to run.", - " --failed": "Runs tests listed in '.failed-tests.json'.", + " --failed": "Runs tests listed in '.failed-tests'.", "-r --reporter=": "The mocha reporter to use.", "-d --debug": "Runs tests in debug mode (NodeJS 6 and earlier)", "-i --inspect": "Runs tests in inspector mode (NodeJS 8 and later)", - " --keepFailed": "Keep tests in '.failed-tests.json even if they pass", + " --keepFailed": "Keep tests in .failed-tests even if they pass", " --light": "Run tests in light mode (fewer verifications, but tests run faster)", " --dirty": "Run tests without first cleaning test output directories", " --stackTraceLimit=": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.", @@ -476,7 +476,7 @@ task("runtests-parallel").description = "Runs all the tests in parallel using th task("runtests-parallel").flags = { " --no-lint": "disables lint.", " --light": "Run tests in light mode (fewer verifications, but tests run faster).", - " --keepFailed": "Keep tests in '.failed-tests.json' even if they pass.", + " --keepFailed": "Keep tests in .failed-tests even if they pass.", " --dirty": "Run tests without first cleaning test output directories.", " --stackTraceLimit=": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.", " --workers=": "The number of parallel workers to use.", @@ -634,9 +634,9 @@ task("watch", series(preBuild, preTest, parallel(watchLib, watchDiagnostics, wat task("watch").description = "Watches for changes and rebuilds and runs tests in parallel."; task("watch").flags = { "-t --tests=": "Pattern for tests to run. Forces tests to be run in a single worker.", - " --failed": "Runs tests listed in '.failed-tests.json'. Forces tests to be run in a single worker.", + " --failed": "Runs tests listed in '.failed-tests'. Forces tests to be run in a single worker.", "-r --reporter=": "The mocha reporter to use.", - " --keepFailed": "Keep tests in '.failed-tests.json' even if they pass", + " --keepFailed": "Keep tests in .failed-tests even if they pass", " --light": "Run tests in light mode (fewer verifications, but tests run faster)", " --dirty": "Run tests without first cleaning test output directories", " --stackTraceLimit=": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.", diff --git a/scripts/build/tests.js b/scripts/build/tests.js index 63972d260eb12..5a6297768ee57 100644 --- a/scripts/build/tests.js +++ b/scripts/build/tests.js @@ -84,7 +84,15 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode, args.push("-g", `"${tests}"`); } if (failed) { - args.push("--config", ".failed-tests.json"); + const grep = fs.readFileSync(".failed-tests", "utf8") + .split(/\r?\n/g) + .map(test => test.trim()) + .filter(test => test.length > 0) + .map(regExpEscape) + .join("|"); + const file = path.join(os.tmpdir(), ".failed-tests.json"); + fs.writeFileSync(file, JSON.stringify({ grep }), "utf8"); + args.push("--config", file); } if (colors) { args.push("--colors"); @@ -206,3 +214,7 @@ function restoreSavedNodeEnv() { function deleteTemporaryProjectOutput() { return del(path.join(exports.localBaseline, "projectOutput/")); } + +function regExpEscape(text) { + return text.replace(/[.*+?^${}()|\[\]\\]/g, '\\$&'); +} diff --git a/scripts/failed-tests.js b/scripts/failed-tests.js index 63a2513cc78bb..66463e56c7c2d 100644 --- a/scripts/failed-tests.js +++ b/scripts/failed-tests.js @@ -7,7 +7,7 @@ const os = require("os"); const failingHookRegExp = /^(.*) "(before|after) (all|each)" hook$/; /** - * .failed-tests.json reporter + * .failed-tests reporter * * @typedef {Object} ReporterOptions * @property {string} [file] @@ -25,7 +25,7 @@ class FailedTestsReporter extends Mocha.reporters.Base { if (!runner) return; const reporterOptions = this.reporterOptions = options.reporterOptions || {}; - if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests.json"; + if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests"; if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false; if (reporterOptions.reporter) { /** @type {Mocha.ReporterConstructor} */ @@ -70,7 +70,7 @@ class FailedTestsReporter extends Mocha.reporters.Base { * @param {(err?: NodeJS.ErrnoException) => void} done */ static writeFailures(file, passes, failures, keepFailed, done) { - const failingTests = new Set(readTests()); + const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined); const possiblyPassingSuites = /**@type {Set}*/(new Set()); // Remove tests that are now passing and track suites that are now @@ -104,8 +104,9 @@ class FailedTestsReporter extends Mocha.reporters.Base { if (failingTests.size > 0) { const failed = Array .from(failingTests) - .sort(); - fs.writeFile(file, JSON.stringify({ grep: failed }, undefined, " "), "utf8", done); + .sort() + .join(os.EOL); + fs.writeFile(file, failed, "utf8", done); } else if (!keepFailed && fs.existsSync(file)) { fs.unlink(file, done); @@ -115,12 +116,10 @@ class FailedTestsReporter extends Mocha.reporters.Base { } function readTests() { - try { - return JSON.parse(fs.readFileSync(file, "utf8")).grep; - } - catch { - return undefined; - } + return fs.readFileSync(file, "utf8") + .split(/\r?\n/g) + .map(line => line.trim()) + .filter(line => line.length > 0); } } diff --git a/src/testRunner/parallel/host.ts b/src/testRunner/parallel/host.ts index eda1e5e296f86..bb4b31b9f058d 100644 --- a/src/testRunner/parallel/host.ts +++ b/src/testRunner/parallel/host.ts @@ -548,7 +548,7 @@ namespace Harness.Parallel.Host { else { failedTestReporter = new FailedTestReporter(replayRunner, { reporterOptions: { - file: path.resolve(".failed-tests.json"), + file: path.resolve(".failed-tests"), keepFailed: Harness.keepFailed // eslint-disable-line @typescript-eslint/no-unnecessary-qualifier } });