Add lazy workflow/step discovery via deferredEntries in next

.changeset/slick-lamps-press.md

@@ -0,0 +1,7 @@
+---
+"@workflow/builders": patch
+"@workflow/core": patch
+"@workflow/next": patch
+---
+
+Add lazy workflow/step discovery via deferredEntries in next

packages/builders/src/base-builder.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, realpath, rename, writeFile } from 'node:fs/promises';
 import { basename, dirname, join, relative, resolve } from 'node:path';
 import { promisify } from 'node:util';
 import { pluralize } from '@workflow/utils';
@@ -25,6 +25,12 @@ const enhancedResolve = promisify(enhancedResolveOriginal);
 const EMIT_SOURCEMAPS_FOR_DEBUGGING =
   process.env.WORKFLOW_EMIT_SOURCEMAPS_FOR_DEBUGGING === '1';
 
+export interface DiscoveredEntries {
+  discoveredSteps: string[];
+  discoveredWorkflows: string[];
+  discoveredSerdeFiles: string[];
+}
+
 /**
  * Base class for workflow builders. Provides common build logic for transforming
  * workflow source files into deployable bundles using esbuild and SWC.
@@ -100,11 +106,7 @@ export abstract class BaseBuilder {
   protected async discoverEntries(
     inputs: string[],
     outdir: string
-  ): Promise<{
-    discoveredSteps: string[];
-    discoveredWorkflows: string[];
-    discoveredSerdeFiles: string[];
-  }> {
+  ): Promise<DiscoveredEntries> {
     const previousResult = this.discoveredEntries.get(inputs);
 
     if (previousResult) {
@@ -270,23 +272,26 @@ export abstract class BaseBuilder {
     outfile,
     externalizeNonSteps,
     tsconfigPath,
+    discoveredEntries,
   }: {
     tsconfigPath?: string;
     inputFiles: string[];
     outfile: string;
     format?: 'cjs' | 'esm';
     externalizeNonSteps?: boolean;
+    discoveredEntries?: DiscoveredEntries;
   }): Promise<{
     context: esbuild.BuildContext | undefined;
     manifest: WorkflowManifest;
   }> {
     // These need to handle watching for dev to scan for
     // new entries and changes to existing ones
-    const {
-      discoveredSteps: stepFiles,
-      discoveredWorkflows: workflowFiles,
-      discoveredSerdeFiles: serdeFiles,
-    } = await this.discoverEntries(inputFiles, dirname(outfile));
+    const discovered =
+      discoveredEntries ??
+      (await this.discoverEntries(inputFiles, dirname(outfile)));
+    const stepFiles = [...discovered.discoveredSteps].sort();
+    const workflowFiles = [...discovered.discoveredWorkflows].sort();
+    const serdeFiles = [...discovered.discoveredSerdeFiles].sort();
 
     // Include serde files that aren't already step files for cross-context class registration.
     // Classes need to be registered in the step bundle so they can be deserialized
@@ -368,6 +373,31 @@ export abstract class BaseBuilder {
     export { stepEntrypoint as POST } from 'workflow/runtime';`;
 
     // Bundle with esbuild and our custom SWC plugin
+    const entriesToBundle = externalizeNonSteps
+      ? [
+          ...stepFiles,
+          ...serdeFiles,
+          ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : []),
+        ]
+      : undefined;
+    const normalizedEntriesToBundle = entriesToBundle
+      ? Array.from(
+          new Set(
+            (
+              await Promise.all(
+                entriesToBundle.map(async (entryToBundle) => {
+                  const resolvedEntry = await realpath(entryToBundle).catch(
+                    () => undefined
+                  );
+                  return resolvedEntry
+                    ? [entryToBundle, resolvedEntry]
+                    : [entryToBundle];
+                })
+              )
+            ).flat()
+          )
+        )
+      : undefined;
     const esbuildCtx = await esbuild.context({
       banner: {
         js: '// biome-ignore-all lint: generated file\n/* eslint-disable */\n',
@@ -414,13 +444,7 @@ export abstract class BaseBuilder {
         createPseudoPackagePlugin(),
         createSwcPlugin({
           mode: 'step',
-          entriesToBundle: externalizeNonSteps
-            ? [
-                ...stepFiles,
-                ...serdeFiles,
-                ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : []),
-              ]
-            : undefined,
+          entriesToBundle: normalizedEntriesToBundle,
           outdir: outfile ? dirname(outfile) : undefined,
           workflowManifest,
         }),
@@ -495,21 +519,24 @@ export abstract class BaseBuilder {
     outfile,
     bundleFinalOutput = true,
     tsconfigPath,
+    discoveredEntries,
   }: {
     tsconfigPath?: string;
     inputFiles: string[];
     outfile: string;
     format?: 'cjs' | 'esm';
     bundleFinalOutput?: boolean;
+    discoveredEntries?: DiscoveredEntries;
   }): Promise<{
     manifest: WorkflowManifest;
     interimBundleCtx?: esbuild.BuildContext;
     bundleFinal?: (interimBundleResult: string) => Promise<void>;
   }> {
-    const {
-      discoveredWorkflows: workflowFiles,
-      discoveredSerdeFiles: serdeFiles,
-    } = await this.discoverEntries(inputFiles, dirname(outfile));
+    const discovered =
+      discoveredEntries ??
+      (await this.discoverEntries(inputFiles, dirname(outfile)));
+    const workflowFiles = [...discovered.discoveredWorkflows].sort();
+    const serdeFiles = [...discovered.discoveredSerdeFiles].sort();
 
     // Include serde files that aren't already workflow files for cross-context class registration.
     // Classes need to be registered in the workflow bundle so they can be deserialized

packages/builders/src/index.ts

@@ -38,6 +38,7 @@ export {
   workflowSerdeImportPattern,
   workflowSerdeSymbolPattern,
 } from './transform-utils.js';
+export { resolveWorkflowAliasRelativePath } from './workflow-alias.js';
 export type {
   AstroConfig,
   BuildTarget,

packages/builders/src/swc-esbuild-plugin.ts

@@ -11,6 +11,7 @@ import {
   jsTsRegex,
   parentHasChild,
 } from './discover-entries-esbuild-plugin.js';
+import { resolveWorkflowAliasRelativePath } from './workflow-alias.js';
 
 export interface SwcPluginOptions {
   mode: 'step' | 'workflow' | 'client';
@@ -187,10 +188,16 @@ export function createSwcPlugin(options: SwcPluginOptions): Plugin {
             // Handle files discovered outside the working directory
             // These come back as ../path/to/file, but we want just path/to/file
             if (relativeFilepath.startsWith('../')) {
-              relativeFilepath = relativeFilepath
-                .split('/')
-                .filter((part) => part !== '..')
-                .join('/');
+              const aliasedRelativePath =
+                await resolveWorkflowAliasRelativePath(args.path, workingDir);
+              if (aliasedRelativePath) {
+                relativeFilepath = aliasedRelativePath;
+              } else {
+                relativeFilepath = relativeFilepath
+                  .split('/')
+                  .filter((part) => part !== '..')
+                  .join('/');
+              }
             }
           }
 

packages/builders/src/workflow-alias.test.ts

@@ -0,0 +1,68 @@
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import {
+  clearWorkflowAliasResolutionCache,
+  resolveWorkflowAliasRelativePath,
+} from './workflow-alias.js';
+
+function writeFile(path: string): void {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, "'use workflow';\n", 'utf-8');
+}
+
+describe('resolveWorkflowAliasRelativePath', () => {
+  let testRoot: string;
+  let workingDir: string;
+
+  beforeEach(() => {
+    clearWorkflowAliasResolutionCache();
+    testRoot = mkdtempSync(join(tmpdir(), 'workflow-alias-'));
+    workingDir = join(testRoot, 'app');
+    mkdirSync(workingDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    clearWorkflowAliasResolutionCache();
+    rmSync(testRoot, { recursive: true, force: true });
+  });
+
+  it('maps files in workflows/ to workflows/* aliases', async () => {
+    const filePath = join(workingDir, 'workflows', 'foo.ts');
+    writeFile(filePath);
+
+    await expect(
+      resolveWorkflowAliasRelativePath(filePath, workingDir)
+    ).resolves.toBe('workflows/foo.ts');
+  });
+
+  it('maps files in src/workflows/ to src/workflows/* aliases', async () => {
+    const filePath = join(workingDir, 'src', 'workflows', 'foo.ts');
+    writeFile(filePath);
+
+    await expect(
+      resolveWorkflowAliasRelativePath(filePath, workingDir)
+    ).resolves.toBe('src/workflows/foo.ts');
+  });
+
+  it('returns undefined for files that are not under workflows paths', async () => {
+    const filePath = join(workingDir, 'lib', 'foo.ts');
+    writeFile(filePath);
+
+    await expect(
+      resolveWorkflowAliasRelativePath(filePath, workingDir)
+    ).resolves.toBeUndefined();
+  });
+
+  it('returns undefined when basename matches but realpath differs', async () => {
+    const workflowFilePath = join(workingDir, 'workflows', 'foo.ts');
+    const externalFilePath = join(testRoot, 'external', 'workflows', 'foo.ts');
+    writeFile(workflowFilePath);
+    writeFile(externalFilePath);
+
+    await expect(
+      resolveWorkflowAliasRelativePath(externalFilePath, workingDir)
+    ).resolves.toBeUndefined();
+  });
+});

packages/builders/src/workflow-alias.ts

@@ -0,0 +1,64 @@
+import { access, realpath } from 'node:fs/promises';
+import { basename, resolve } from 'node:path';
+
+const workflowAliasResolutionCache = new Map<
+  string,
+  Promise<string | undefined>
+>();
+
+export function clearWorkflowAliasResolutionCache(): void {
+  workflowAliasResolutionCache.clear();
+}
+
+export async function resolveWorkflowAliasRelativePath(
+  absoluteFilePath: string,
+  workingDir: string
+): Promise<string | undefined> {
+  const normalizedAbsolutePath = absoluteFilePath.replace(/\\/g, '/');
+  // Only workflow source files can map to app-level `workflows/*` aliases.
+  if (!normalizedAbsolutePath.includes('/workflows/')) {
+    return undefined;
+  }
+
+  const cacheKey = `${workingDir}::${normalizedAbsolutePath}`;
+  const cached = workflowAliasResolutionCache.get(cacheKey);
+  if (cached) {
+    return cached;
+  }
+
+  const resolutionPromise = (async () => {
+    const fileName = basename(absoluteFilePath);
+    const aliasDirs = ['workflows', 'src/workflows'];
+    const resolvedFilePath = await realpath(absoluteFilePath).catch(
+      () => undefined
+    );
+    if (!resolvedFilePath) {
+      return undefined;
+    }
+
+    const aliases = await Promise.all(
+      aliasDirs.map(async (aliasDir) => {
+        const candidatePath = resolve(workingDir, aliasDir, fileName);
+        try {
+          await access(candidatePath);
+        } catch {
+          return undefined;
+        }
+        const resolvedCandidatePath = await realpath(candidatePath).catch(
+          () => undefined
+        );
+        if (!resolvedCandidatePath) {
+          return undefined;
+        }
+        return resolvedCandidatePath === resolvedFilePath
+          ? `${aliasDir}/${fileName}`
+          : undefined;
+      })
+    );
+
+    return aliases.find((aliasPath): aliasPath is string => Boolean(aliasPath));
+  })();
+
+  workflowAliasResolutionCache.set(cacheKey, resolutionPromise);
+  return resolutionPromise;
+}

packages/core/e2e/dev.test.ts

@@ -1,6 +1,6 @@
 import fs from 'fs/promises';
 import path from 'path';
-import { afterEach, describe, expect, test } from 'vitest';
+import { afterEach, beforeAll, describe, expect, test } from 'vitest';
 import { getWorkbenchAppPath } from './utils';
 
 export interface DevTestConfig {
@@ -35,6 +35,7 @@
   }
   describe('dev e2e', () => {
     const appPath = getWorkbenchAppPath();
+    const deploymentUrl = process.env.DEPLOYMENT_URL;
     const generatedStep = path.join(appPath, finalConfig.generatedStepPath);
     const generatedWorkflow = path.join(
       appPath,
@@ -44,6 +45,28 @@
     const workflowsDir = finalConfig.workflowsDir ?? 'workflows';
     const restoreFiles: Array<{ path: string; content: string }> = [];
 
+    const fetchWithTimeout = (pathname: string) => {
+      if (!deploymentUrl) {
+        return Promise.resolve();
+      }
+
+      return fetch(new URL(pathname, deploymentUrl), {
+        signal: AbortSignal.timeout(5_000),
+      });
+    };
+
+    const prewarm = async () => {
+      // Pre-warm the app with bounded requests so cleanup hooks cannot hang.
+      await Promise.all([
+        fetchWithTimeout('/').catch(() => {}),
+        fetchWithTimeout('/api/chat').catch(() => {}),
+      ]);
+    };
+
+    beforeAll(async () => {
+      await prewarm();
+    });
+
     afterEach(async () => {
       await Promise.all(
         restoreFiles.map(async (item) => {
@@ -54,10 +77,11 @@
           }
         })
       );
+      await prewarm();
       restoreFiles.length = 0;
     });
 
     test('should rebuild on workflow change', { timeout: 30_000 }, async () => {
       const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile);
 
       const content = await fs.readFile(workflowFile, 'utf8');
@@ -85,7 +109,7 @@
       }
     });
 
     test('should rebuild on step change', { timeout: 30_000 }, async () => {
       const stepFile = path.join(appPath, workflowsDir, testWorkflowFile);
 
       const content = await fs.readFile(stepFile, 'utf8');
@@ -145,6 +169,7 @@
 
         while (true) {
           try {
+            await fetchWithTimeout('/api/chat');
             const workflowContent = await fs.readFile(
               generatedWorkflow,
               'utf8'