test(cli): rework snapshots to support parallel test execution

.vscode/launch.json

@@ -11,6 +11,7 @@
       "program": "${workspaceRoot}/packages/build/node_modules/mocha/bin/_mocha",
       "runtimeArgs": ["-r", "${workspaceRoot}/packages/build/node_modules/source-map-support/register"],
       "cwd": "${workspaceRoot}",
+      "autoAttachChildProcesses": true,
       "args": [
         "--config",
         "${workspaceRoot}/packages/build/config/.mocharc.json",
@@ -26,6 +27,7 @@
       "program": "${workspaceRoot}/bin/mocha-current-file",
       "runtimeArgs": ["-r", "${workspaceRoot}/packages/build/node_modules/source-map-support/register"],
       "cwd": "${workspaceRoot}",
+      "autoAttachChildProcesses": true,
       "args": [
         "--config",
         "${workspaceRoot}/packages/build/config/.mocharc.json",

package.json

@@ -69,7 +69,7 @@
     "build:site": "./bin/build-docs-site.sh",
     "docs:prepare": "./docs/bin/build-preview-site.sh",
     "docs:start": "cd docs/_preview && bundle exec jekyll serve --no-w --i",
-    "mocha": "node packages/build/bin/run-mocha --lang en_US.UTF-8 --timeout 5000 \"packages/*/dist/__tests__/**/*.js\" \"extensions/*/dist/__tests__/**/*.js\" \"examples/*/dist/__tests__/**/*.js\" \"packages/cli/test/**/*.js\" \"packages/build/test/*/*.js\"",
+    "mocha": "node packages/build/bin/run-mocha --lang en_US.UTF-8 --timeout 15000 --parallel \"packages/*/dist/__tests__/**/*.js\" \"extensions/*/dist/__tests__/**/*.js\" \"examples/*/dist/__tests__/**/*.js\" \"packages/cli/test/**/*.js\" \"packages/build/test/*/*.js\"",
     "posttest": "npm run lint"
   },
   "config": {

packages/build/bin/run-mocha.js

@@ -24,7 +24,6 @@ function run(argv, options) {
     !utils.isOptionSet(
       mochaOpts,
       '--config', // mocha 6.x
-      '--opts', // legacy
       '--package', // mocha 6.x
       '--no-config', // mocha 6.x
     ) && !utils.mochaConfiguredForProject();

packages/build/bin/utils.js

@@ -228,7 +228,6 @@ function mochaConfiguredForProject() {
     '.mocharc.json',
     '.mocharc.yaml',
     '.mocharc.yml',
-    'test/mocha.opts',
   ];
   return configFiles.some(f => {
     const configFile = path.join(getPackageDir(), f);

packages/build/config/.mocharc.json

@@ -1,5 +1,5 @@
 {
-  "require": "source-map-support/register",
+  "require": ["source-map-support/register"],
   "recursive": true,
   "exit": true,
   "reporter": "dot"

packages/cli/snapshots/integration/dummy/common.integration.snapshots.js

@@ -0,0 +1,12 @@
+// IMPORTANT
+// This snapshot file is auto-generated, but designed for humans.
+// It should be checked into source control and tracked carefully.
+// Re-generate by setting UPDATE_SNAPSHOTS=1 and running tests.
+// Make sure to inspect the changes in the snapshots below.
+// Do not ignore changes!
+
+'use strict';
+
+exports[`common matches snapshot 1`] = `
+common result
+`;

packages/cli/snapshots/integration/dummy/test1.integration.snapshots.js

@@ -0,0 +1,12 @@
+// IMPORTANT
+// This snapshot file is auto-generated, but designed for humans.
+// It should be checked into source control and tracked carefully.
+// Re-generate by setting UPDATE_SNAPSHOTS=1 and running tests.
+// Make sure to inspect the changes in the snapshots below.
+// Do not ignore changes!
+
+'use strict';
+
+exports[`test1 matches snapshot 1 1`] = `
+test1 result
+`;

packages/cli/snapshots/integration/dummy/test2.integration.snapshots.js

@@ -0,0 +1,12 @@
+// IMPORTANT
+// This snapshot file is auto-generated, but designed for humans.
+// It should be checked into source control and tracked carefully.
+// Re-generate by setting UPDATE_SNAPSHOTS=1 and running tests.
+// Make sure to inspect the changes in the snapshots below.
+// Do not ignore changes!
+
+'use strict';
+
+exports[`test2 matches snapshot 2 1`] = `
+test2 result
+`;

packages/cli/test/integration/cli/cli.integration.js

@@ -12,7 +12,7 @@ const main = require('../../../lib/cli');
 const {
   expectToMatchSnapshot,
   expectFileToMatchSnapshot,
-} = require('../../snapshots');
+} = require('../../snapshots')();
 
 function getLog(buffer) {
   buffer = buffer || [];

packages/cli/test/integration/dummy/common.integration.js

@@ -0,0 +1,16 @@
+// Copyright IBM Corp. 2018,2020. All Rights Reserved.
+// Node module: @loopback/cli
+// This file is licensed under the MIT License.
+// License text available at https://opensource.org/licenses/MIT
+
+'use strict';
+
+exports.commonTest = function () {
+  describe('common', () => {
+    const {expectToMatchSnapshot} = require('../../snapshots')();
+
+    it('matches snapshot 1', function () {
+      expectToMatchSnapshot('common result');
+    });
+  });
+};

packages/cli/test/integration/dummy/test1.integration.js

@@ -0,0 +1,16 @@
+// Copyright IBM Corp. 2018,2020. All Rights Reserved.
+// Node module: @loopback/cli
+// This file is licensed under the MIT License.
+// License text available at https://opensource.org/licenses/MIT
+
+'use strict';
+
+const {expectToMatchSnapshot} = require('../../snapshots')();
+
+describe('test1', () => {
+  it('matches snapshot 1', () => {
+    expectToMatchSnapshot('test1 result');
+  });
+});
+
+require('./common.integration').commonTest();

packages/cli/test/integration/dummy/test2.integration.js

@@ -0,0 +1,16 @@
+// Copyright IBM Corp. 2018,2020. All Rights Reserved.
+// Node module: @loopback/cli
+// This file is licensed under the MIT License.
+// License text available at https://opensource.org/licenses/MIT
+
+'use strict';
+
+const {expectToMatchSnapshot} = require('../../snapshots')();
+
+describe('test2', () => {
+  it('matches snapshot 2', () => {
+    expectToMatchSnapshot('test2 result');
+  });
+});
+
+require('./common.integration').commonTest();

packages/cli/test/integration/generators/app.integration.js

@@ -21,7 +21,7 @@ const props = {
 };
 const {expect} = require('@loopback/testlab');
 
-const {assertFilesToMatchSnapshot} = require('../../snapshots');
+const {assertFilesToMatchSnapshot} = require('../../snapshots')();
 
 const tests = require('../lib/project-generator')(
   generator,

packages/cli/test/integration/generators/controller.integration.js

@@ -18,7 +18,7 @@ const tests = require('../lib/artifact-generator')(generator);
 const baseTests = require('../lib/base-generator')(generator);
 const testUtils = require('../../test-utils');
 
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 // Test Sandbox
 const sandbox = new TestSandbox(path.resolve(__dirname, '../.sandbox'));

packages/cli/test/integration/generators/datasource.integration.js

@@ -17,7 +17,7 @@ const generator = path.join(__dirname, '../../../generators/datasource');
 const tests = require('../lib/artifact-generator')(generator);
 const baseTests = require('../lib/base-generator')(generator);
 const testUtils = require('../../test-utils');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 // Test Sandbox
 const sandbox = new TestSandbox(path.resolve(__dirname, '../.sandbox'));

packages/cli/test/integration/generators/discover.integration.js

@@ -9,7 +9,7 @@
 const path = require('path');
 const assert = require('yeoman-assert');
 const {expect, TestSandbox} = require('@loopback/testlab');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/discover');
 require('../lib/artifact-generator')(generator);

packages/cli/test/integration/generators/import-lb3-models.integration.js

@@ -9,7 +9,7 @@ const assert = require('yeoman-assert');
 const {expect, TestSandbox} = require('@loopback/testlab');
 const path = require('path');
 const generator = path.join(__dirname, '../../../generators/import-lb3-models');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 const testUtils = require('../../test-utils');
 
 const sandbox = new TestSandbox(path.resolve(__dirname, '../.sandbox'));

packages/cli/test/integration/generators/interceptor.integration.js

@@ -12,7 +12,7 @@ const generator = path.join(__dirname, '../../../generators/interceptor');
 const SANDBOX_FILES = require('../../fixtures/interceptor').SANDBOX_FILES;
 const testUtils = require('../../test-utils');
 
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 // Test Sandbox
 const sandbox = new TestSandbox(path.resolve(__dirname, '../.sandbox'));

packages/cli/test/integration/generators/model.integration.js

@@ -9,7 +9,7 @@
 const path = require('path');
 const assert = require('yeoman-assert');
 const {expect, TestSandbox} = require('@loopback/testlab');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 const {remove} = require('fs-extra');
 
 const generator = path.join(__dirname, '../../../generators/model');

packages/cli/test/integration/generators/openapi-client.integration.js

@@ -7,7 +7,7 @@
 
 const path = require('path');
 const {TestSandbox} = require('@loopback/testlab');
-const {assertFilesToMatchSnapshot} = require('../../snapshots');
+const {assertFilesToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/openapi');
 const specPath = path.resolve(

packages/cli/test/integration/generators/openapi-petstore.integration.js

@@ -7,7 +7,7 @@
 
 const path = require('path');
 const {TestSandbox} = require('@loopback/testlab');
-const {assertFilesToMatchSnapshot} = require('../../snapshots');
+const {assertFilesToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/openapi');
 const specPath = path.join(

packages/cli/test/integration/generators/openapi-uspto-with-anonymous.integration.js

@@ -7,7 +7,7 @@
 
 const path = require('path');
 const {TestSandbox} = require('@loopback/testlab');
-const {assertFilesToMatchSnapshot} = require('../../snapshots');
+const {assertFilesToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/openapi');
 const specPath = path.join(__dirname, '../../fixtures/openapi/3.0/uspto.yaml');

packages/cli/test/integration/generators/openapi-uspto.integration.js

@@ -7,7 +7,7 @@
 
 const path = require('path');
 const {TestSandbox} = require('@loopback/testlab');
-const {assertFilesToMatchSnapshot} = require('../../snapshots');
+const {assertFilesToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/openapi');
 const specPath = path.join(__dirname, '../../fixtures/openapi/3.0/uspto.yaml');

packages/cli/test/integration/generators/relation.belongs-to.integration.js

@@ -8,7 +8,7 @@
 const path = require('path');
 const assert = require('yeoman-assert');
 const {expect, TestSandbox} = require('@loopback/testlab');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/relation');
 const {SANDBOX_FILES, SourceEntries} = require('../../fixtures/relation');

packages/cli/test/integration/generators/relation.has-many.integration.js

@@ -8,7 +8,7 @@
 const path = require('path');
 const assert = require('yeoman-assert');
 const {expect, TestSandbox} = require('@loopback/testlab');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/relation');
 const {SANDBOX_FILES, SourceEntries} = require('../../fixtures/relation');

packages/cli/test/integration/generators/relation.has-one.integration.js

@@ -8,7 +8,7 @@
 const path = require('path');
 const assert = require('yeoman-assert');
 const {expect, TestSandbox} = require('@loopback/testlab');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 const generator = path.join(__dirname, '../../../generators/relation');
 const {SANDBOX_FILES, SourceEntries} = require('../../fixtures/relation');

packages/cli/test/integration/generators/remote-service.integration.js

@@ -15,7 +15,7 @@ const TestSandbox = testlab.TestSandbox;
 const generator = path.join(__dirname, '../../../generators/service');
 const SANDBOX_FILES = require('../../fixtures/service').SANDBOX_FILES;
 const testUtils = require('../../test-utils');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 // Test Sandbox
 const sandbox = new TestSandbox(path.resolve(__dirname, '../.sandbox'));

packages/cli/test/integration/generators/rest-crud.integration.js

@@ -15,7 +15,7 @@ const TestSandbox = testlab.TestSandbox;
 const generator = path.join(__dirname, '../../../generators/rest-crud');
 const SANDBOX_FILES = require('../../fixtures/rest-crud').SANDBOX_FILES;
 const testUtils = require('../../test-utils');
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 // Test Sandbox
 const sandbox = new TestSandbox(path.resolve(__dirname, '../.sandbox'));

packages/cli/test/integration/generators/swagger-petstore.integration.js

@@ -12,7 +12,7 @@ const specPath = path.join(
   __dirname,
   '../../fixtures/openapi/2.0/petstore-expanded-swagger.json',
 );
-const {expectFileToMatchSnapshot} = require('../../snapshots');
+const {expectFileToMatchSnapshot} = require('../../snapshots')();
 
 const testlab = require('@loopback/testlab');
 const TestSandbox = testlab.TestSandbox;

packages/cli/test/snapshot-matcher.js

@@ -17,6 +17,7 @@ move this file to a standalone package so that all Mocha users can use it.
 const chalk = require('chalk');
 const assert = require('assert');
 const path = require('path');
+const debug = require('debug')('test:snapshot-matcher');
 
 module.exports = {
   initializeSnapshots,
@@ -43,6 +44,22 @@ module.exports = {
  * ```
  */
 function initializeSnapshots(snapshotDir) {
+  if (debug.enabled) {
+    const stack = new Error().stack
+      .split(/\n/g)
+      // Remove the error message and the top stack frame pointing to ourselves
+      // and pick three frames (max), that should be enough to identify
+      // which test file called us.
+      .slice(2, 5)
+      .map(f => `\n${f}`)
+      .join();
+    debug(
+      'Initializing snapshot matcher, storing snapshots in %s%s',
+      snapshotDir,
+      stack,
+    );
+  }
+
   let currentTest;
   let snapshotErrors = false;