Add lazy workflow and step discovery in Next.js

.changeset/stale-bushes-listen.md

@@ -0,0 +1,6 @@
+---
+"@workflow/builders": patch
+"@workflow/next": patch
+---
+
+Add lazy workflow and step discovery in Next.js

.github/workflows/tests.yml

@@ -8,6 +8,9 @@ on:
       - "!*"
   pull_request:
 
+env:
+  NEXT_TELEMETRY_DISABLED: 1
+
 concurrency:
   # Unique group for this workflow and branch
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -534,12 +537,22 @@ jobs:
       - name: Run E2E Tests (Next.js)
         run: |
           cd workbench/nextjs-turbopack
-          $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev }
+          $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev *>&1 | Tee-Object -FilePath "$using:PWD\dev.log" }
           Start-Sleep -Seconds 15
           cd ../..
-          pnpm vitest run packages/core/e2e/dev.test.ts
-          pnpm run test:e2e --reporter=default --reporter=json --outputFile=e2e-windows-nextjs-turbopack.json
-          Stop-Job $job
+
+          try {
+            pnpm run test:e2e --reporter=default --reporter=json --outputFile=e2e-windows-nextjs-turbopack.json
+            pnpm vitest run packages/core/e2e/dev.test.ts
+          } finally {
+            Stop-Job $job
+            Write-Host "`n=== Dev Server Logs ===`n"
+            if (Test-Path "workbench/nextjs-turbopack/dev.log") {
+              Get-Content "workbench/nextjs-turbopack/dev.log"
+            } else {
+              Write-Host "No dev.log file found"
+            }
+          }
         shell: powershell
         env:
           APP_NAME: "nextjs-turbopack"

packages/builders/src/base-builder.ts

@@ -137,6 +137,12 @@ export abstract class BaseBuilder {
     discoveredSteps: string[];
     discoveredWorkflows: string[];
   }> {
+    if (this.config.buildTarget === 'next') {
+      return {
+        discoveredWorkflows: inputs,
+        discoveredSteps: inputs,
+      };
+    }
     const previousResult = this.discoveredEntries.get(inputs);
 
     if (previousResult) {
@@ -252,7 +258,11 @@ export abstract class BaseBuilder {
       }
     }
 
-    if (result.warnings && result.warnings.length > 0) {
+    if (
+      this.config.buildTarget !== 'next' &&
+      result.warnings &&
+      result.warnings.length > 0
+    ) {
       console.warn(`!  esbuild warnings in ${phase}:`);
       for (const warning of result.warnings) {
         console.warn(`  ${warning.text}`);
@@ -314,9 +324,20 @@ export abstract class BaseBuilder {
       );
     });
 
+    const combinedStepFiles: string[] = [
+      ...stepFiles,
+      ...(resolvedBuiltInSteps
+        ? [
+            resolvedBuiltInSteps,
+            // TODO: expose this in workflow/package.json and use resolve?
+            join(dirname(resolvedBuiltInSteps), '../stdlib.js'),
+          ]
+        : []),
+    ];
+
     // Create a virtual entry that imports all files. All step definitions
     // will get registered thanks to the swc transform.
-    const imports = stepFiles
+    const imports = combinedStepFiles
       .map((file) => {
         // Normalize both paths to forward slashes before calling relative()
         // This is critical on Windows where relative() can produce unexpected results with mixed path formats
@@ -366,6 +387,7 @@ export abstract class BaseBuilder {
       keepNames: true,
       minify: false,
       jsx: 'preserve',
+      logLevel: 'error',
       resolveExtensions: [
         '.ts',
         '.tsx',
@@ -381,12 +403,7 @@ export abstract class BaseBuilder {
       plugins: [
         createSwcPlugin({
           mode: 'step',
-          entriesToBundle: externalizeNonSteps
-            ? [
-                ...stepFiles,
-                ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : []),
-              ]
-            : undefined,
+          entriesToBundle: externalizeNonSteps ? combinedStepFiles : undefined,
           outdir: outfile ? dirname(outfile) : undefined,
           tsBaseUrl,
           tsPaths,

packages/builders/src/index.ts

@@ -1,10 +1,16 @@
 export type { WorkflowManifest } from './apply-swc-transform.js';
 export { applySwcTransform } from './apply-swc-transform.js';
 export { BaseBuilder } from './base-builder.js';
+export { createBuildQueue } from './build-queue.js';
 export { createBaseBuilderConfig } from './config-helpers.js';
 export { STEP_QUEUE_TRIGGER, WORKFLOW_QUEUE_TRIGGER } from './constants.js';
-export { createDiscoverEntriesPlugin } from './discover-entries-esbuild-plugin.js';
+export {
+  createDiscoverEntriesPlugin,
+  useStepPattern,
+  useWorkflowPattern,
+} from './discover-entries-esbuild-plugin.js';
 export { createNodeModuleErrorPlugin } from './node-module-esbuild-plugin.js';
+export { NORMALIZE_REQUEST_CODE } from './request-converter.js';
 export { StandaloneBuilder } from './standalone.js';
 export { createSwcPlugin } from './swc-esbuild-plugin.js';
 export type {
@@ -18,5 +24,3 @@ export type {
 } from './types.js';
 export { isValidBuildTarget, validBuildTargets } from './types.js';
 export { VercelBuildOutputAPIBuilder } from './vercel-build-output-api.js';
-export { createBuildQueue } from './build-queue.js';
-export { NORMALIZE_REQUEST_CODE } from './request-converter.js';

packages/core/e2e/build-errors.test.ts

@@ -16,6 +16,8 @@ const exec = promisify(execOriginal);
  */
 describe('build error messages', () => {
   const restoreFiles: Array<{ path: string; content: string | null }> = [];
+  const appName = process.env.APP_NAME ?? 'nextjs-turbopack';
+  const appPath = getWorkbenchAppPath(appName);
 
   afterEach(async () => {
     // Restore files in reverse order to handle dependencies
@@ -27,6 +29,8 @@ describe('build error messages', () => {
       }
     }
     restoreFiles.length = 0;
+    // previous failures can cause successive tests to fail
+    await fs.rm(path.join(appPath, '.next'), { recursive: true, force: true });
   });
 
   /**
@@ -89,11 +93,8 @@ describe('build error messages', () => {
 
   test(
     'should show helpful error when using Node.js module in workflow',
-    { timeout: 120_000 },
+    { timeout: 60_000 },
     async () => {
-      const appName = process.env.APP_NAME ?? 'nextjs-turbopack';
-      const appPath = getWorkbenchAppPath(appName);
-
       // Note: filename must NOT start with _ (those are skipped by registry generator)
       const badWorkflowContent = `
 import { readFileSync } from 'fs';
@@ -138,10 +139,8 @@ export async function nodeModuleViolationWorkflow() {
     process.env.APP_NAME && process.env.APP_NAME !== 'nextjs-turbopack'
   )(
     'should show top-level package name for external dependencies that use Node.js modules',
-    { timeout: 120_000 },
+    { timeout: 60_000 },
     async () => {
-      const appPath = getWorkbenchAppPath('nextjs-turbopack');
-
       // @vercel/blob internally uses Node.js modules (via undici)
       // The error should show "@vercel/blob" not the internal Node.js module
       const badWorkflowContent = `
@@ -183,11 +182,8 @@ export async function blobViolationWorkflow() {
 
   test(
     'should show helpful error when using Bun module in workflow',
-    { timeout: 120_000 },
+    { timeout: 60_000 },
     async () => {
-      const appName = process.env.APP_NAME ?? 'nextjs-turbopack';
-      const appPath = getWorkbenchAppPath(appName);
-
       // Bun modules should show a different error message than Node.js modules
       const badWorkflowContent = `
 import { serve } from 'bun';
@@ -223,11 +219,8 @@ export async function bunViolationWorkflow() {
 
   test(
     'should report all violations when multiple Node.js modules are used',
-    { timeout: 120_000 },
+    { timeout: 60_000 },
     async () => {
-      const appName = process.env.APP_NAME ?? 'nextjs-turbopack';
-      const appPath = getWorkbenchAppPath(appName);
-
       // Using multiple Node.js modules should report errors for all of them
       const badWorkflowContent = `
 import { readFileSync } from 'fs';

packages/core/e2e/dev.test.ts

@@ -1,5 +1,5 @@
-import fs from 'fs/promises';
-import path from 'path';
+import fs from 'node:fs/promises';
+import path from 'node:path';
 import { afterEach, describe, expect, test } from 'vitest';
 import { getWorkbenchAppPath } from './utils';
 
@@ -44,6 +44,14 @@ export function createDevTests(config?: DevTestConfig) {
     const workflowsDir = finalConfig.workflowsDir ?? 'workflows';
     const restoreFiles: Array<{ path: string; content: string }> = [];
 
+    const warmEndpoint = async () => {
+      // Warm up the Next.js routes to trigger lazy workflow/step discovery and compilation.
+      // This is only required for tests that respond to file updates (HMR tests).
+      // Without this, the routes won't be built yet and file change detection won't work.
+      await fetch(new URL('/api/trigger', process.env.DEPLOYMENT_URL));
+      await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL));
+    };
+
     afterEach(async () => {
       await Promise.all(
         restoreFiles.map(async (item) => {
@@ -54,58 +62,61 @@ export function createDevTests(config?: DevTestConfig) {
           }
         })
       );
+      await warmEndpoint();
       restoreFiles.length = 0;
     });
 
-    test('should rebuild on workflow change', { timeout: 30_000 }, async () => {
-      const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile);
+    test('should rebuild on step change', { timeout: 15_000 }, async () => {
+      const stepFile = path.join(appPath, workflowsDir, testWorkflowFile);
 
-      const content = await fs.readFile(workflowFile, 'utf8');
+      const content = await fs.readFile(stepFile, 'utf8');
 
       await fs.writeFile(
-        workflowFile,
+        stepFile,
         `${content}
 
-export async function myNewWorkflow() {
-  'use workflow'
+export async function myNewStep() {
+  'use step'
   return 'hello world'
 }
 `
       );
-      restoreFiles.push({ path: workflowFile, content });
+      restoreFiles.push({ path: stepFile, content });
 
       while (true) {
         try {
-          const workflowContent = await fs.readFile(generatedWorkflow, 'utf8');
-          expect(workflowContent).toContain('myNewWorkflow');
+          await warmEndpoint();
+          const workflowContent = await fs.readFile(generatedStep, 'utf8');
+          expect(workflowContent).toContain('myNewStep');
           break;
         } catch (_) {
           await new Promise((res) => setTimeout(res, 1_000));
         }
       }
     });
 
-    test('should rebuild on step change', { timeout: 30_000 }, async () => {
-      const stepFile = path.join(appPath, workflowsDir, testWorkflowFile);
+    test('should rebuild on workflow change', { timeout: 15_000 }, async () => {
+      const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile);
 
-      const content = await fs.readFile(stepFile, 'utf8');
+      const content = await fs.readFile(workflowFile, 'utf8');
 
       await fs.writeFile(
-        stepFile,
+        workflowFile,
         `${content}
 
-export async function myNewStep() {
-  'use step'
+export async function myNewWorkflow() {
+  'use workflow'
   return 'hello world'
 }
 `
       );
-      restoreFiles.push({ path: stepFile, content });
+      restoreFiles.push({ path: workflowFile, content });
 
       while (true) {
         try {
-          const workflowContent = await fs.readFile(generatedStep, 'utf8');
-          expect(workflowContent).toContain('myNewStep');
+          await warmEndpoint();
+          const workflowContent = await fs.readFile(generatedWorkflow, 'utf8');
+          expect(workflowContent).toContain('myNewWorkflow');
           break;
         } catch (_) {
           await new Promise((res) => setTimeout(res, 1_000));
@@ -115,7 +126,7 @@ export async function myNewStep() {
 
     test(
       'should rebuild on adding workflow file',
-      { timeout: 30_000 },
+      { timeout: 15_000 },
       async () => {
         const workflowFile = path.join(
           appPath,
@@ -131,11 +142,11 @@ export async function myNewStep() {
 }
 `
         );
-        restoreFiles.push({ path: workflowFile, content: '' });
         const apiFile = path.join(appPath, finalConfig.apiFilePath);
-
         const apiFileContent = await fs.readFile(apiFile, 'utf8');
+
         restoreFiles.push({ path: apiFile, content: apiFileContent });
+        restoreFiles.push({ path: workflowFile, content: '' });
 
         await fs.writeFile(
           apiFile,
@@ -145,6 +156,7 @@ ${apiFileContent}`
 
         while (true) {
           try {
+            await warmEndpoint();
             const workflowContent = await fs.readFile(
               generatedWorkflow,
               'utf8'

packages/core/e2e/local-build.test.ts

@@ -25,8 +25,12 @@ describe.each([
 
     const result = await exec('pnpm build', {
       cwd: getWorkbenchAppPath(project),
+      timeout: 120_000,
     });
 
+    console.log(result.stdout);
+    console.log(result.stderr);
+
     expect(result.stderr).not.toContain('Error:');
   });
 });

packages/next/package.json

@@ -18,8 +18,7 @@
   },
   "exports": {
     ".": "./dist/index.js",
-    "./loader": "./dist/loader.js",
-    "./runtime": "./dist/runtime.js"
+    "./loader": "./dist/loader.js"
   },
   "scripts": {
     "build": "tsc",
@@ -30,15 +29,11 @@
     "@swc/core": "catalog:",
     "@workflow/builders": "workspace:*",
     "@workflow/core": "workspace:*",
-    "@workflow/swc-plugin": "workspace:*",
-    "semver": "7.7.3",
-    "watchpack": "2.4.4"
+    "@workflow/swc-plugin": "workspace:*"
   },
   "devDependencies": {
     "@workflow/tsconfig": "workspace:*",
     "@types/node": "catalog:",
-    "@types/semver": "7.7.1",
-    "@types/watchpack": "2.4.4",
     "next": "16.0.10"
   },
   "peerDependencies": {