shard tests

.github/workflows/test.yml

@@ -1,32 +1,78 @@
 name: Test
-
 on:
   pull_request:
     branches:
       - main
+  push:
+    branches:
+      - main
 
-concurrency: ${{ github.workflow }}-${{ github.ref }}
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
 
 jobs:
-  test:
-    name: Test
-    timeout-minutes: 20
+  determine-packages:
+    runs-on: ubuntu-latest
+    outputs:
+      packages: ${{ steps.set-packages.outputs.packages }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get package names
+        id: set-packages
+        run: |
+          SEARCH_DIRS="plugins utils validate convert enrich import export"
+
+          PACKAGES=$(
+            for dir in $SEARCH_DIRS; do
+              if [ -d "$dir" ]; then
+                find "$dir" -maxdepth 2 -name "package.json" -exec sh -c '
+                  PACKAGE_NAME=$(jq -r .name {})
+                  if [ "$PACKAGE_NAME" != "null" ]; then
+                    echo "$PACKAGE_NAME"
+                  fi
+                ' \;
+              fi
+            done | jq -R -s -c 'split("\n")[:-1]'
+          )
+
+          echo "packages=$PACKAGES" >> $GITHUB_OUTPUT
+          echo "Found packages: $PACKAGES"
+
+  test-packages:
+    needs: determine-packages
+    name: Test (${{ matrix.package }})
     runs-on: ubuntu-latest
     if: ${{ github.head_ref != 'changeset-release/main' }}
+    strategy:
+      matrix:
+        package: ${{fromJson(needs.determine-packages.outputs.packages)}}
+      fail-fast: false
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
+        uses: actions/checkout@v4
 
       - name: Setup Node.js environment
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 16
+          node-version: 18
           cache: 'npm'
 
       - name: Install dependencies
         run: npm ci
 
+      - name: Build
+        run: npx turbo run build
+
       - name: Test
-        run: npm run test
+        run: npx turbo run test --filter=${{ matrix.package }}
+
+  test:
+    needs: test-packages
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check test results
+        run: |
+          echo "All package tests completed successfully"
+          exit 0

convert/what3words/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

convert/what3words/package.json

@@ -37,7 +37,7 @@
     "build:watch": "rollup -c --watch",
     "build:prod": "NODE_ENV=production rollup -c",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [
     "flatfile-plugins",
@@ -60,4 +60,4 @@
   "devDependencies": {
     "@flatfile/rollup-config": "0.1.1"
   }
-}
+}

jest.config.js

@@ -1,6 +1,8 @@
 module.exports = {
   testEnvironment: 'node',
-  testRegex: '.*\\.(e2e-)?spec\\.ts$',
+  // testRegex: '(?<!\\.\\./)src/.*\\.(e2e-)?spec\\.ts$',
+  testMatch: ['<rootDir>/**/**/src/*.spec.ts'],
+
   transform: {
     '^.+\\.tsx?$': 'ts-jest',
   },
@@ -12,4 +14,5 @@ module.exports = {
   testTimeout: 60_000,
   globalSetup: '<rootDir>/test/setup-global.js',
   forceExit: true,
+  passWithNoTests: true,
 }

package.json

@@ -16,8 +16,9 @@
   ],
   "scripts": {
     "clean": "find ./ '(' -name 'node_modules' -o -name 'dist' -o -name '.turbo' -o -name '.parcel-cache' ')' -type d -exec rm -rf {} +",
-    "test": "turbo build && turbo test --concurrency=1",
+    "test": "jest --config=./jest.config.js --runInBand",
     "build": "turbo build",
+    "turbo:test": "turbo test",
     "build:prod": "turbo build:prod",
     "build:clean": "npm run clean && npm i && turbo build",
     "build:prod:clean": "npm run clean && npm i && turbo build:prod",

plugins/autocast/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/autocast/package.json

@@ -37,7 +37,7 @@
     "build:watch": "rollup -c --watch",
     "build:prod": "NODE_ENV=production rollup -c",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [
     "flatfile-plugins",

plugins/automap/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 90_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/automap/package.json

@@ -18,7 +18,7 @@
     "build:watch": "parcel watch",
     "build:prod": "NODE_ENV=production parcel build",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [
     "flatfile-plugins",

plugins/constraints/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/constraints/package.json

@@ -36,7 +36,7 @@
     "build:watch": "rollup -c --watch",
     "build:prod": "NODE_ENV=production rollup -c",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [],
   "author": "David Boskovic",

plugins/constraints/src/external.constraint.e2e.spec.ts

@@ -42,7 +42,9 @@ describe('externalConstraint()', () => {
 
   it('correctly assigns an error', async () => {
     await createRecords(sheetId, defaultSimpleValueData)
-    await listener.waitFor('commit:created')
+    await listener.waitFor('commit:created', 2)
+    // Sleep for 1000ms to allow time for the constraint to be applied
+    // await new Promise(resolve => setTimeout(resolve, 1000));
     const records = await getRecords(sheetId)
     expect(records[0].values['name'].messages[0]).toMatchObject({
       type: 'error',

plugins/constraints/src/external.sheet.constraint.e2e.spec.ts

@@ -50,7 +50,7 @@ describe('externalConstraint()', () => {
   it('correctly handles thrown errors', async () => {
     await createRecords(sheetId, defaultSimpleValueData)
 
-    await listener.waitFor('commit:created')
+    await listener.waitFor('commit:created', 2)
     const records = await getRecords(sheetId)
     expect(records[0].values['name'].messages[0]).toMatchObject({
       type: 'error',

plugins/dedupe/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/dedupe/package.json

@@ -36,7 +36,7 @@
     "build:watch": "rollup -c --watch",
     "build:prod": "NODE_ENV=production rollup -c",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [
     "flatfile-plugins",

plugins/delimiter-extractor/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/delimiter-extractor/package.json

@@ -18,7 +18,7 @@
     "build:watch": "parcel watch",
     "build:prod": "NODE_ENV=production parcel build",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [
     "flatfile-plugins",

plugins/dxp-configure/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/dxp-configure/package.json

@@ -42,7 +42,7 @@
     "build:watch": "rollup -c --watch",
     "build:prod": "NODE_ENV=production rollup -c",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [
     "flatfile-plugins",

plugins/export-workbook/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/foreign-db-extractor/jest.config.cjs

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/graphql-schema/jest.config.cjs

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

plugins/graphql-schema/package.json

@@ -27,7 +27,7 @@
     "build:watch": "rollup -c --watch",
     "build:prod": "NODE_ENV=production rollup -c",
     "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
-    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+    "test": "jest src/*.spec.ts"
   },
   "keywords": [],
   "author": "Flatfile, Inc.",