From 0498a4855ae2f65c55dc9a00554601434864ef35 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 14:40:16 -0800 Subject: [PATCH 01/27] Add lazy workflow and step discovery in Next.js --- .changeset/stale-bushes-listen.md | 6 + packages/builders/src/base-builder.ts | 6 + packages/builders/src/index.ts | 10 +- packages/next/package.json | 4 +- packages/next/src/builder.ts | 554 ++++++++++++-------------- packages/next/src/index.ts | 8 +- packages/next/src/loader.ts | 154 ++++++- packages/next/src/runtime.ts | 4 - packages/next/tsconfig.json | 4 +- pnpm-lock.yaml | 35 -- 10 files changed, 436 insertions(+), 349 deletions(-) create mode 100644 .changeset/stale-bushes-listen.md delete mode 100644 packages/next/src/runtime.ts diff --git a/.changeset/stale-bushes-listen.md b/.changeset/stale-bushes-listen.md new file mode 100644 index 0000000000..67537aef6f --- /dev/null +++ b/.changeset/stale-bushes-listen.md @@ -0,0 +1,6 @@ +--- +"@workflow/builders": patch +"@workflow/next": patch +--- + +Add lazy workflow and step discovery in Next.js diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index 984fa3f1cc..6e5b47df74 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -137,6 +137,12 @@ export abstract class BaseBuilder { discoveredSteps: string[]; discoveredWorkflows: string[]; }> { + if (this.config.buildTarget === 'next') { + return { + discoveredWorkflows: inputs, + discoveredSteps: inputs, + }; + } const previousResult = this.discoveredEntries.get(inputs); if (previousResult) { diff --git a/packages/builders/src/index.ts b/packages/builders/src/index.ts index ef2f077dd6..2ee8716445 100644 --- a/packages/builders/src/index.ts +++ b/packages/builders/src/index.ts @@ -1,10 +1,16 @@ export type { WorkflowManifest } from './apply-swc-transform.js'; export { applySwcTransform } from './apply-swc-transform.js'; export { BaseBuilder } from './base-builder.js'; +export { createBuildQueue } from './build-queue.js'; export { createBaseBuilderConfig } from './config-helpers.js'; export { STEP_QUEUE_TRIGGER, WORKFLOW_QUEUE_TRIGGER } from './constants.js'; -export { createDiscoverEntriesPlugin } from './discover-entries-esbuild-plugin.js'; +export { + createDiscoverEntriesPlugin, + useStepPattern, + useWorkflowPattern, +} from './discover-entries-esbuild-plugin.js'; export { createNodeModuleErrorPlugin } from './node-module-esbuild-plugin.js'; +export { NORMALIZE_REQUEST_CODE } from './request-converter.js'; export { StandaloneBuilder } from './standalone.js'; export { createSwcPlugin } from './swc-esbuild-plugin.js'; export type { @@ -18,5 +24,3 @@ export type { } from './types.js'; export { isValidBuildTarget, validBuildTargets } from './types.js'; export { VercelBuildOutputAPIBuilder } from './vercel-build-output-api.js'; -export { createBuildQueue } from './build-queue.js'; -export { NORMALIZE_REQUEST_CODE } from './request-converter.js'; diff --git a/packages/next/package.json b/packages/next/package.json index 639e09fdd2..7d84c3b97b 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -31,14 +31,12 @@ "@workflow/builders": "workspace:*", "@workflow/core": "workspace:*", "@workflow/swc-plugin": "workspace:*", - "semver": "7.7.3", - "watchpack": "2.4.4" + "semver": "7.7.3" }, "devDependencies": { "@workflow/tsconfig": "workspace:*", "@types/node": "catalog:", "@types/semver": "7.7.1", - "@types/watchpack": "2.4.4", "next": "16.0.10" }, "peerDependencies": { diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 660c0bfc86..4e94458995 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -1,7 +1,7 @@ -import { constants } from 'node:fs'; -import { access, mkdir, stat, writeFile } from 'node:fs/promises'; -import { extname, join, resolve } from 'node:path'; -import Watchpack from 'watchpack'; +import { constants, unlinkSync } from 'node:fs'; +import { access, mkdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import type { NextConfig } from 'next'; let CachedNextBuilder: any; @@ -17,13 +17,74 @@ export async function getNextBuilder() { BaseBuilder: BaseBuilderClass, STEP_QUEUE_TRIGGER, WORKFLOW_QUEUE_TRIGGER, - // biome-ignore lint/security/noGlobalEval: Need to use eval here to avoid TypeScript from transpiling the import statement into `require()` - } = (await eval( - 'import("@workflow/builders")' - )) as typeof import('@workflow/builders'); + } = await import('@workflow/builders'); class NextBuilder extends BaseBuilderClass { - async build() { + private socketIO?: any; + private nextConfig?: NextConfig; + + setNextConfig(config: NextConfig) { + this.nextConfig = config; + } + + private getDistDir(): string { + return this.nextConfig?.distDir || '.next'; + } + + private async writeWorkflowsCache( + workflowFiles: Set, + stepFiles: Set + ) { + const cwd = this.config.workingDir; + const distDir = this.getDistDir(); + const cacheDir = join(cwd, distDir, 'cache'); + const cacheFile = join(cacheDir, 'workflows.json'); + + try { + await mkdir(cacheDir, { recursive: true }); + const cacheData = { + workflowFiles: Array.from(workflowFiles), + stepFiles: Array.from(stepFiles), + timestamp: Date.now(), + }; + await writeFile(cacheFile, JSON.stringify(cacheData, null, 2)); + } catch (error) { + console.error('Failed to write workflows cache:', error); + } + } + + private async readWorkflowsCache(): Promise<{ + workflowFiles: string[]; + stepFiles: string[]; + } | null> { + const cwd = this.config.workingDir; + const distDir = this.getDistDir(); + const cacheFile = join(cwd, distDir, 'cache', 'workflows.json'); + + try { + const cacheContent = await readFile(cacheFile, 'utf-8'); + const cacheData = JSON.parse(cacheContent); + return { + workflowFiles: cacheData.workflowFiles || [], + stepFiles: cacheData.stepFiles || [], + }; + } catch { + // Cache file doesn't exist or is invalid, return null + return null; + } + } + + async init() { + const outputDir = await this.findAppDirectory(); + + // Write stub files + await this.writeStubFiles(outputDir); + + // Create socket server for file path communication + await this.createSocketServer(outputDir); + } + + async build(inputFiles?: string[]) { const outputDir = await this.findAppDirectory(); const workflowGeneratedDir = join(outputDir, '.well-known/workflow/v1'); @@ -33,308 +94,25 @@ export async function getNextBuilder() { await writeFile(join(workflowGeneratedDir, '.gitignore'), '*'); - const inputFiles = await this.getInputFiles(); + // Use provided inputFiles or discover them + const files = inputFiles || (await this.getInputFiles()); const tsConfig = await this.getTsConfigOptions(); const options = { - inputFiles, + inputFiles: files, workflowGeneratedDir, tsBaseUrl: tsConfig.baseUrl, tsPaths: tsConfig.paths, }; - const stepsBuildContext = await this.buildStepsFunction(options); - const workflowsBundle = await this.buildWorkflowsFunction(options); + await this.buildStepsFunction(options); + await this.buildWorkflowsFunction(options); await this.buildWebhookRoute({ workflowGeneratedDir }); await this.writeFunctionsConfig(outputDir); - if (this.config.watch) { - if (!stepsBuildContext) { - throw new Error( - 'Invariant: expected steps build context in watch mode' - ); - } - if (!workflowsBundle) { - throw new Error('Invariant: expected workflows bundle in watch mode'); - } - - let stepsCtx = stepsBuildContext; - let workflowsCtx = workflowsBundle; - - const normalizePath = (pathname: string) => - pathname.replace(/\\/g, '/'); - const knownFiles = new Set(); - type WatchpackTimeInfoEntry = { - safeTime: number; - timestamp?: number; - }; - let previousTimeInfo = new Map(); - - const watchableExtensions = new Set([ - '.js', - '.jsx', - '.ts', - '.tsx', - '.mts', - '.cts', - '.cjs', - '.mjs', - ]); - const ignoredPathFragments = [ - '/.git/', - '/node_modules/', - '/.next/', - '/.turbo/', - '/.vercel/', - '/dist/', - '/build/', - '/out/', - '/.cache/', - '/.yarn/', - '/.pnpm-store/', - '/.parcel-cache/', - '/.well-known/workflow/', - ]; - const normalizedGeneratedDir = workflowGeneratedDir.replace(/\\/g, '/'); - ignoredPathFragments.push(normalizedGeneratedDir); - - // There is a node.js bug on MacOS which causes closing file watchers to be really slow. - // This limits the number of watchers to mitigate the issue. - // https://github.com/nodejs/node/issues/29949 - process.env.WATCHPACK_WATCHER_LIMIT = - process.platform === 'darwin' ? '20' : undefined; - - const watcher = new Watchpack({ - // Watchpack default is 200ms which adds 200ms of dead time on bootup. - aggregateTimeout: 5, - ignored: (pathname: string) => { - const normalizedPath = pathname.replace(/\\/g, '/'); - const extension = extname(normalizedPath); - if (extension && !watchableExtensions.has(extension)) { - return true; - } - if (normalizedPath.startsWith(normalizedGeneratedDir)) { - return true; - } - for (const fragment of ignoredPathFragments) { - if (normalizedPath.includes(fragment)) { - return true; - } - } - return false; - }, - }); - - const readTimeInfoEntries = () => { - const rawEntries = watcher.getTimeInfoEntries() as Map< - string, - WatchpackTimeInfoEntry - >; - const normalizedEntries = new Map(); - for (const [path, info] of rawEntries) { - normalizedEntries.set(normalizePath(path), info); - } - return normalizedEntries; - }; - - let rebuildQueue = Promise.resolve(); - - const enqueue = (task: () => Promise) => { - rebuildQueue = rebuildQueue.then(task).catch((error) => { - console.error('Failed to process file change', error); - }); - return rebuildQueue; - }; - - const fullRebuild = async () => { - const newInputFiles = await this.getInputFiles(); - options.inputFiles = newInputFiles; - - await stepsCtx.dispose(); - const newStepsCtx = await this.buildStepsFunction(options); - if (!newStepsCtx) { - throw new Error( - 'Invariant: expected steps build context after rebuild' - ); - } - stepsCtx = newStepsCtx; - - await workflowsCtx.interimBundleCtx.dispose(); - const newWorkflowsCtx = await this.buildWorkflowsFunction(options); - if (!newWorkflowsCtx) { - throw new Error( - 'Invariant: expected workflows bundle context after rebuild' - ); - } - workflowsCtx = newWorkflowsCtx; - }; - - const logBuildMessages = ( - result: { - errors?: import('esbuild').Message[]; - warnings?: import('esbuild').Message[]; - }, - label: string - ) => { - const logByType = ( - messages: import('esbuild').Message[] | undefined, - method: 'error' | 'warn' - ) => { - if (!messages || messages.length === 0) { - return; - } - const descriptor = method === 'error' ? 'errors' : 'warnings'; - console[method](`${descriptor} while rebuilding ${label}`); - for (const message of messages) { - console[method](message); - } - }; - - logByType(result.errors, 'error'); - logByType(result.warnings, 'warn'); - }; - - const rebuildExistingFiles = async () => { - const rebuiltStepStart = Date.now(); - const stepsResult = await stepsCtx.rebuild(); - logBuildMessages(stepsResult, 'steps bundle'); - console.log( - 'Rebuilt steps bundle', - `${Date.now() - rebuiltStepStart}ms` - ); - - const rebuiltWorkflowStart = Date.now(); - const workflowResult = await workflowsCtx.interimBundleCtx.rebuild(); - logBuildMessages(workflowResult, 'workflows bundle'); - - if ( - !workflowResult.outputFiles || - workflowResult.outputFiles.length === 0 - ) { - console.error( - 'No output generated while rebuilding workflows bundle' - ); - return; - } - await workflowsCtx.bundleFinal(workflowResult.outputFiles[0].text); - console.log( - 'Rebuilt workflow bundle', - `${Date.now() - rebuiltWorkflowStart}ms` - ); - }; - - const isWatchableFile = (path: string) => - watchableExtensions.has(extname(path)); - - const getComparableTimestamp = (entry: WatchpackTimeInfoEntry) => - entry.timestamp ?? entry.safeTime; - - const findRemovedFiles = ( - currentEntries: Map, - previousEntries: Map - ) => { - const removed: string[] = []; - for (const path of previousEntries.keys()) { - if (!currentEntries.has(path) && isWatchableFile(path)) { - removed.push(path); - } - } - return removed; - }; - - const findAddedAndModifiedFiles = ( - currentEntries: Map, - previousEntries: Map - ) => { - const added: string[] = []; - const modified: string[] = []; - - for (const [path, info] of currentEntries) { - if (!isWatchableFile(path)) { - continue; - } - - const previous = previousEntries.get(path); - if (!previous) { - added.push(path); - continue; - } - - if ( - getComparableTimestamp(info) !== getComparableTimestamp(previous) - ) { - modified.push(path); - } - } - - return { added, modified }; - }; - - const determineFileChanges = ( - currentEntries: Map, - previousEntries: Map - ) => { - const removedFiles = findRemovedFiles( - currentEntries, - previousEntries - ); - const { added, modified } = findAddedAndModifiedFiles( - currentEntries, - previousEntries - ); - - return { - addedFiles: added, - modifiedFiles: modified, - removedFiles, - }; - }; - - let isInitial = true; - - watcher.on('aggregated', () => { - const currentEntries = readTimeInfoEntries(); - const { addedFiles, modifiedFiles, removedFiles } = - determineFileChanges(currentEntries, previousTimeInfo); - - previousTimeInfo = currentEntries; - - if (isInitial) { - isInitial = false; - return; - } - - if ( - addedFiles.length === 0 && - modifiedFiles.length === 0 && - removedFiles.length === 0 - ) { - return; - } - - for (const removal of removedFiles) { - knownFiles.delete(removal); - } - for (const added of addedFiles) { - knownFiles.add(added); - } - - enqueue(async () => { - if (addedFiles.length > 0 || removedFiles.length > 0) { - await fullRebuild(); - return; - } - - if (modifiedFiles.length > 0) { - await rebuildExistingFiles(); - } - }); - }); - - watcher.watch({ - directories: [this.config.workingDir], - startTime: 0, - }); + // Signal build complete to connected clients + if (this.socketIO) { + this.socketIO.emit('build-complete'); } } @@ -467,6 +245,182 @@ export async function getNextBuilder() { } } } + + private async createSocketServer(_usersAppDir: string): Promise { + const { createServer } = await import('node:net'); + const { tmpdir } = await import('node:os'); + const { unlink } = await import('node:fs/promises'); + + const workflowFiles = new Set(); + const stepFiles = new Set(); + const clients = new Set(); + let debounceTimer: NodeJS.Timeout | null = null; + + const BUILD_DEBOUNCE_MS = + process.env.NODE_ENV === 'development' ? 300 : 1_000; + + // Attempt to load cached workflows/steps from previous build + const cache = await this.readWorkflowsCache(); + if (cache) { + for (const file of cache.workflowFiles) { + workflowFiles.add(file); + } + for (const file of cache.stepFiles) { + stepFiles.add(file); + } + } + + // Debounced build trigger + const triggerBuild = () => { + if (debounceTimer) { + clearTimeout(debounceTimer); + } + + debounceTimer = setTimeout(async () => { + // Combine workflow and step files into single array + const allFiles = new Set([...workflowFiles, ...stepFiles]); + const inputFiles = Array.from(allFiles); + + if (inputFiles.length > 0) { + console.log( + `Triggering build with ${inputFiles.length} discovered files` + ); + try { + await this.build(inputFiles); + // Write cache after successful build + await this.writeWorkflowsCache(workflowFiles, stepFiles); + } catch (error) { + console.error('Build failed:', error); + } + } + }, BUILD_DEBOUNCE_MS); + }; + + // Generate socket path in tmp directory + const socketPath = join( + tmpdir(), + `workflow-${process.pid}-${Date.now()}.sock` + ); + + process.on('exit', () => { + try { + unlinkSync(socketPath); + } catch {} + }); + + // Create Unix domain socket server + const server = createServer((socket) => { + clients.add(socket); + + let buffer = ''; + + socket.on('data', (data) => { + buffer += data.toString(); + + // Process complete messages (newline-delimited JSON) + let newlineIndex = buffer.indexOf('\n'); + while (newlineIndex !== -1) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + newlineIndex = buffer.indexOf('\n'); + + if (line.trim()) { + try { + const message = JSON.parse(line); + + if (message.type === 'file-discovered') { + const { filePath, hasWorkflow, hasStep } = message; + + if (hasWorkflow) { + workflowFiles.add(filePath); + } + + if (hasStep) { + stepFiles.add(filePath); + } + + // Trigger debounced build + triggerBuild(); + } else if (message.type === 'trigger-build') { + // enqueue new build if one isn't already pending + triggerBuild(); + } + } catch (error) { + console.error('Failed to parse socket message:', error); + } + } + } + }); + + socket.on('end', () => { + clients.delete(socket); + }); + + socket.on('error', (err) => { + console.error('Socket error:', err); + clients.delete(socket); + }); + }); + + // Clean up existing socket file if it exists + try { + await unlink(socketPath); + } catch { + // Ignore error if file doesn't exist + } + + // Listen on Unix domain socket + await new Promise((resolve) => { + server.listen(socketPath, () => { + // Expose the socket path via environment variable + process.env.WORKFLOW_SOCKET_PATH = socketPath; + resolve(); + }); + }); + + // Store the server and broadcast function + this.socketIO = { + emit: (event: string) => { + if (event === 'build-complete') { + const message = JSON.stringify({ type: 'build-complete' }) + '\n'; + for (const client of clients) { + client.write(message); + } + } + }, + }; + } + + private async writeStubFiles(usersAppDir: string): Promise { + const routeStubContent = "export * from './inner'"; + // this needs to change on each build so can refresh workflows + const innerStubContent = 'WORKFLOW_INNER_STUB_FILE_' + Date.now(); + const workflowDir = join(usersAppDir, '.well-known/workflow/v1'); + + // Ensure directories exist + await mkdir(join(workflowDir, 'flow'), { recursive: true }); + await mkdir(join(workflowDir, 'step'), { recursive: true }); + await mkdir(join(workflowDir, 'webhook'), { recursive: true }); + await mkdir(join(workflowDir, 'webhook/[token]'), { recursive: true }); + + // Write route.ts stub files (re-export from inner) + await writeFile(join(workflowDir, 'flow/route.js'), routeStubContent); + await writeFile(join(workflowDir, 'step/route.js'), routeStubContent); + await writeFile(join(workflowDir, 'webhook/route.js'), routeStubContent); + await writeFile( + join(workflowDir, 'webhook/[token]/route.js'), + routeStubContent + ); + + // Write inner.js stub files (actual stub marker) + await writeFile(join(workflowDir, 'flow/inner.js'), innerStubContent); + await writeFile(join(workflowDir, 'step/inner.js'), innerStubContent); + await writeFile(join(workflowDir, 'webhook/inner.js'), innerStubContent); + await writeFile( + join(workflowDir, 'webhook/[token]/inner.js'), + routeStubContent + ); + } } CachedNextBuilder = NextBuilder; diff --git a/packages/next/src/index.ts b/packages/next/src/index.ts index cd24924d20..b749fe9cfa 100644 --- a/packages/next/src/index.ts +++ b/packages/next/src/index.ts @@ -82,6 +82,9 @@ export function withWorkflow( { content: /(use workflow|use step)/, }, + { + path: /.*\.well-known[/\\]workflow.*/, + }, ], }, } @@ -131,7 +134,10 @@ export function withWorkflow( externalPackages: [...(nextConfig.serverExternalPackages || [])], }); - await workflowBuilder.build(); + // Pass the nextConfig to the builder so it can access distDir + workflowBuilder.setNextConfig(nextConfig); + + await workflowBuilder.init(); process.env.WORKFLOW_NEXT_PRIVATE_BUILT = '1'; } diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index e6aca8cb4c..8929ace33d 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -1,5 +1,123 @@ +import { readFile } from 'node:fs/promises'; +import { connect, type Socket } from 'node:net'; import { relative } from 'node:path'; import { transform } from '@swc/core'; +import { useStepPattern, useWorkflowPattern } from '@workflow/builders'; + +// Stub content written by builder to inner.js files +const STUB_CONTENT = 'WORKFLOW_INNER_STUB_FILE'; + +// Cache for socket connection to avoid reconnecting on every file +let socketClientPromise: Promise | null = null; + +async function getSocketClient() { + if (!socketClientPromise) { + socketClientPromise = (async () => { + const socketPath = process.env.WORKFLOW_SOCKET_PATH; + if (!socketPath) { + throw new Error( + `Invariant: no socket path provided for workflow loader` + ); + } + + const socket = connect(socketPath); + + // Wait for connection + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + socket.destroy(); + reject(new Error('Socket connection timeout')); + }, 1000); + + socket.on('connect', () => { + clearTimeout(timeout); + resolve(); + }); + + socket.on('error', (err: Error) => { + clearTimeout(timeout); + reject(err); + }); + }); + + return socket; + })(); + } + + return socketClientPromise; +} + +async function notifySocketServer( + filename: string, + hasWorkflow: boolean, + hasStep: boolean +) { + try { + const socket = await getSocketClient(); + if (!socket) { + return; + } + + // Send single message with both workflow and step information + const message = + JSON.stringify({ + type: 'file-discovered', + filePath: filename, + hasWorkflow, + hasStep, + }) + '\n'; + socket.write(message); + } catch { + // Silently fail - socket server might not be available yet + } +} + +async function waitForBuildComplete(): Promise { + const socket = await getSocketClient(); + + return new Promise((resolve, reject) => { + if (!socket) { + reject(new Error('Socket not available')); + return; + } + + let buffer = ''; + const timeout = setTimeout(() => { + socket.off('data', onData); + reject(new Error('Build complete timeout')); + }, 60000); // 60 second timeout + + const onData = (data: Buffer) => { + buffer += data.toString(); + + let newlineIndex = buffer.indexOf('\n'); + while (newlineIndex !== -1) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + newlineIndex = buffer.indexOf('\n'); + + if (line.trim()) { + try { + const message = JSON.parse(line); + if (message.type === 'build-complete') { + clearTimeout(timeout); + socket.off('data', onData); + resolve(); + } + } catch { + // Ignore parse errors + } + } + } + }; + + socket.on('data', onData); + + // Send trigger-build message + const message = JSON.stringify({ type: 'trigger-build' }) + '\n'; + socket.write(message); + }); +} // This loader applies the "use workflow"/"use step" // client transformation @@ -13,11 +131,45 @@ export default async function workflowLoader( const filename = this.resourcePath; const normalizedSource = source.toString(); + // Normalize path separators for cross-platform compatibility + const normalizedFilename = filename.replace(/\\/g, '/'); + + // Check if this is a .well-known workflow inner.js file with stub content + const isWellKnownInnerFile = + normalizedFilename.includes('.well-known/workflow/v1/') && + (normalizedFilename.includes('/flow/inner.js') || + normalizedFilename.includes('/step/inner.js') || + normalizedFilename.includes('/webhook/inner.js') || + normalizedFilename.includes('/webhook/[token]/inner.js')); + + if ( + isWellKnownInnerFile && + normalizedSource.trim().startsWith(STUB_CONTENT) + ) { + // Wait for build to complete + await waitForBuildComplete(); + + // Read the actual generated file content + const actualContent = await readFile( + filename.replace(/inner\.js/, 'route.js'), + 'utf-8' + ); + return actualContent; + } + + // Check for workflow and step directives + const hasWorkflow = useWorkflowPattern.test(normalizedSource); + normalizedSource; + const hasStep = useStepPattern.test(normalizedSource); + // only apply the transform if file needs it - if (!normalizedSource.match(/(use step|use workflow)/)) { + if (!hasWorkflow && !hasStep) { return normalizedSource; } + // Send message to socket server if workflow or step detected + await notifySocketServer(filename, hasWorkflow, hasStep); + const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx') || diff --git a/packages/next/src/runtime.ts b/packages/next/src/runtime.ts deleted file mode 100644 index 13e4a77f59..0000000000 --- a/packages/next/src/runtime.ts +++ /dev/null @@ -1,4 +0,0 @@ -// re-export runtime as stub for resolving to not -// require @workflow/core be a dependency as well as -// @workflow/next -export * from '@workflow/core/dist/runtime'; diff --git a/packages/next/tsconfig.json b/packages/next/tsconfig.json index 1397247ef5..615e7d950a 100644 --- a/packages/next/tsconfig.json +++ b/packages/next/tsconfig.json @@ -3,8 +3,8 @@ "compilerOptions": { "outDir": "dist", "target": "es2022", - "module": "commonjs", - "moduleResolution": "node" + "module": "nodenext", + "moduleResolution": "nodenext" }, "include": ["src"], "exclude": ["node_modules", "**/*.test.ts"] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c01f41c397..b675c488ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -619,9 +619,6 @@ importers: semver: specifier: 7.7.3 version: 7.7.3 - watchpack: - specifier: 2.4.4 - version: 2.4.4 devDependencies: '@types/node': specifier: 'catalog:' @@ -629,9 +626,6 @@ importers: '@types/semver': specifier: 7.7.1 version: 7.7.1 - '@types/watchpack': - specifier: 2.4.4 - version: 2.4.4 '@workflow/tsconfig': specifier: workspace:* version: link:../tsconfig @@ -6425,9 +6419,6 @@ packages: '@types/geojson@7946.0.16': resolution: {integrity: sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==} - '@types/graceful-fs@4.1.9': - resolution: {integrity: sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==} - '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} @@ -6545,9 +6536,6 @@ packages: '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} - '@types/watchpack@2.4.4': - resolution: {integrity: sha512-SbuSavsPxfOPZwVHBgQUVuzYBe6+8KL7dwiJLXaj5rmv3DxktOMwX5WP1J6UontwUbewjVoc7pCgZvqy6rPn+A==} - '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} @@ -8966,9 +8954,6 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} - glob-to-regexp@0.4.1: - resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} - glob@10.4.5: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true @@ -13157,10 +13142,6 @@ packages: typescript: optional: true - watchpack@2.4.4: - resolution: {integrity: sha512-c5EGNOiyxxV5qmTtAB7rbiXxi1ooX1pQKMLX/MIabJjRA0SJBQOjKF+KSVfHkr9U1cADPon0mRiVe/riyaiDUA==} - engines: {node: '>=10.13.0'} - wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} @@ -19035,10 +19016,6 @@ snapshots: '@types/geojson@7946.0.16': {} - '@types/graceful-fs@4.1.9': - dependencies: - '@types/node': 24.6.2 - '@types/hast@3.0.4': dependencies: '@types/unist': 3.0.3 @@ -19157,11 +19134,6 @@ snapshots: '@types/unist@3.0.3': {} - '@types/watchpack@2.4.4': - dependencies: - '@types/graceful-fs': 4.1.9 - '@types/node': 24.6.2 - '@types/yauzl@2.10.3': dependencies: '@types/node': 24.6.2 @@ -22082,8 +22054,6 @@ snapshots: is-glob: 4.0.3 optional: true - glob-to-regexp@0.4.1: {} - glob@10.4.5: dependencies: foreground-child: 3.3.1 @@ -27628,11 +27598,6 @@ snapshots: optionalDependencies: typescript: 5.9.3 - watchpack@2.4.4: - dependencies: - glob-to-regexp: 0.4.1 - graceful-fs: 4.2.11 - wcwidth@1.0.1: dependencies: defaults: 1.0.4 From 9147a93024ab9ca60913475c58edc1b66667e87a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 14:56:28 -0800 Subject: [PATCH 02/27] fix nitro builder name --- packages/nitro/src/builders.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nitro/src/builders.ts b/packages/nitro/src/builders.ts index 8b36d24d14..d33e802b87 100644 --- a/packages/nitro/src/builders.ts +++ b/packages/nitro/src/builders.ts @@ -40,7 +40,7 @@ export class LocalBuilder extends BaseBuilder { watch: nitro.options.dev, dirs: ['.'], // Different apps that use nitro have different directories }), - buildTarget: 'next', // Placeholder, not actually used + buildTarget: 'nitro' as any, // Placeholder, not actually used }); this.#outDir = outDir; } From 01e6ddc39d020477839e46910882ff2e40bf66fa Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 14:59:20 -0800 Subject: [PATCH 03/27] updates from review --- packages/next/src/builder.ts | 2 +- packages/next/src/loader.ts | 1 - workbench/nextjs-turbopack/next.config.ts | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 4e94458995..0fcf710179 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -418,7 +418,7 @@ export async function getNextBuilder() { await writeFile(join(workflowDir, 'webhook/inner.js'), innerStubContent); await writeFile( join(workflowDir, 'webhook/[token]/inner.js'), - routeStubContent + innerStubContent ); } } diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index 8929ace33d..b69238916e 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -159,7 +159,6 @@ export default async function workflowLoader( // Check for workflow and step directives const hasWorkflow = useWorkflowPattern.test(normalizedSource); - normalizedSource; const hasStep = useStepPattern.test(normalizedSource); // only apply the transform if file needs it diff --git a/workbench/nextjs-turbopack/next.config.ts b/workbench/nextjs-turbopack/next.config.ts index 0428ff83f7..6206788429 100644 --- a/workbench/nextjs-turbopack/next.config.ts +++ b/workbench/nextjs-turbopack/next.config.ts @@ -4,6 +4,8 @@ import { withWorkflow } from 'workflow/next'; const nextConfig: NextConfig = { /* config options here */ serverExternalPackages: ['@node-rs/xxhash'], + + experimental: { turbopackModuleIds: 'named' }, }; // export default nextConfig; From 2386ca9d160c2f80a053e4b181c046b4b87e3772 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 16:35:32 -0800 Subject: [PATCH 04/27] update dev tests --- packages/core/e2e/dev.test.ts | 14 ++++++++++---- packages/next/src/loader.ts | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index 824c807933..19a409e4ea 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -1,6 +1,6 @@ import fs from 'fs/promises'; import path from 'path'; -import { afterEach, describe, expect, test } from 'vitest'; +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; import { getWorkbenchAppPath } from './utils'; export interface DevTestConfig { @@ -57,7 +57,13 @@ export function createDevTests(config?: DevTestConfig) { restoreFiles.length = 0; }); - test('should rebuild on workflow change', { timeout: 30_000 }, async () => { + beforeEach(async () => { + // ensure Next.js is building route + await fetch(new URL('/api/trigger', process.env.DEPLOYMENT_URL)); + await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL)); + }); + + test('should rebuild on workflow change', { timeout: 15_000 }, async () => { const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile); const content = await fs.readFile(workflowFile, 'utf8'); @@ -85,7 +91,7 @@ export async function myNewWorkflow() { } }); - test('should rebuild on step change', { timeout: 30_000 }, async () => { + test('should rebuild on step change', { timeout: 15_000 }, async () => { const stepFile = path.join(appPath, workflowsDir, testWorkflowFile); const content = await fs.readFile(stepFile, 'utf8'); @@ -115,7 +121,7 @@ export async function myNewStep() { test( 'should rebuild on adding workflow file', - { timeout: 30_000 }, + { timeout: 15_000 }, async () => { const workflowFile = path.join( appPath, diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index b69238916e..5ea1ed124b 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -55,7 +55,7 @@ async function notifySocketServer( try { const socket = await getSocketClient(); if (!socket) { - return; + throw new Error(`Invariant: missing workflow socket connection`); } // Send single message with both workflow and step information From 48ce2959ff96e8d4bbe41bb4a6d7c563d4fb8253 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 17:27:50 -0800 Subject: [PATCH 05/27] fix source map issue and stdlib discovery --- packages/next/src/builder.ts | 3 ++- packages/workflow/tsconfig.json | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 0fcf710179..20f0761298 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -392,7 +392,8 @@ export async function getNextBuilder() { } private async writeStubFiles(usersAppDir: string): Promise { - const routeStubContent = "export * from './inner'"; + const routeStubContent = + "import 'workflow'\n" + "export * from './inner'"; // this needs to change on each build so can refresh workflows const innerStubContent = 'WORKFLOW_INNER_STUB_FILE_' + Date.now(); const workflowDir = join(usersAppDir, '.well-known/workflow/v1'); diff --git a/packages/workflow/tsconfig.json b/packages/workflow/tsconfig.json index a78dbf413c..65f2136e46 100644 --- a/packages/workflow/tsconfig.json +++ b/packages/workflow/tsconfig.json @@ -1,7 +1,9 @@ { "extends": "@workflow/tsconfig/base.json", "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "sourceMap": false, + "inlineSourceMap": true }, "include": ["src"], "exclude": ["node_modules", "**/*.test.ts"] From 6c44ef1985299decfa4b7266032d1306587510bb Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 17:59:08 -0800 Subject: [PATCH 06/27] update tests and deduping --- packages/builders/src/base-builder.ts | 1 + packages/core/e2e/dev.test.ts | 7 +++++-- packages/next/src/builder.ts | 11 +++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index 6e5b47df74..4016b3432c 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -372,6 +372,7 @@ export abstract class BaseBuilder { keepNames: true, minify: false, jsx: 'preserve', + logLevel: 'error', resolveExtensions: [ '.ts', '.tsx', diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index 19a409e4ea..5933504d29 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -57,11 +57,11 @@ export function createDevTests(config?: DevTestConfig) { restoreFiles.length = 0; }); - beforeEach(async () => { + const warmEndpoint = async () => { // ensure Next.js is building route await fetch(new URL('/api/trigger', process.env.DEPLOYMENT_URL)); await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL)); - }); + }; test('should rebuild on workflow change', { timeout: 15_000 }, async () => { const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile); @@ -82,6 +82,7 @@ export async function myNewWorkflow() { while (true) { try { + await warmEndpoint(); const workflowContent = await fs.readFile(generatedWorkflow, 'utf8'); expect(workflowContent).toContain('myNewWorkflow'); break; @@ -110,6 +111,7 @@ export async function myNewStep() { while (true) { try { + await warmEndpoint(); const workflowContent = await fs.readFile(generatedStep, 'utf8'); expect(workflowContent).toContain('myNewStep'); break; @@ -151,6 +153,7 @@ ${apiFileContent}` while (true) { try { + await warmEndpoint(); const workflowContent = await fs.readFile( generatedWorkflow, 'utf8' diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 20f0761298..0338171b67 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -247,6 +247,9 @@ export async function getNextBuilder() { } private async createSocketServer(_usersAppDir: string): Promise { + if (process.env.WORKFLOW_SOCKET_PATH) { + return; + } const { createServer } = await import('node:net'); const { tmpdir } = await import('node:os'); const { unlink } = await import('node:fs/promises'); @@ -257,7 +260,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 300 : 1_000; + process.env.NODE_ENV === 'development' ? 250 : 1_000; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); @@ -293,6 +296,7 @@ export async function getNextBuilder() { console.error('Build failed:', error); } } + debounceTimer = null; }, BUILD_DEBOUNCE_MS); }; @@ -327,6 +331,7 @@ export async function getNextBuilder() { if (line.trim()) { try { const message = JSON.parse(line); + console.log('got msg', message); if (message.type === 'file-discovered') { const { filePath, hasWorkflow, hasStep } = message; @@ -343,7 +348,9 @@ export async function getNextBuilder() { triggerBuild(); } else if (message.type === 'trigger-build') { // enqueue new build if one isn't already pending - triggerBuild(); + if (!debounceTimer) { + triggerBuild(); + } } } catch (error) { console.error('Failed to parse socket message:', error); From 1707672fd9b056cdb0d3d690933f16dc454d006e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 18:11:58 -0800 Subject: [PATCH 07/27] update logs and fixture --- .github/workflows/tests.yml | 7 +++++-- packages/core/e2e/dev.test.ts | 13 +++++++------ packages/next/src/builder.ts | 1 - 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17b53b8199..750348ced7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -534,12 +534,13 @@ jobs: - name: Run E2E Tests (Next.js) run: | cd workbench/nextjs-turbopack - $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev } + $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev 2>&1 | Tee-Object -FilePath "../../dev-server.log" } Start-Sleep -Seconds 15 cd ../.. pnpm vitest run packages/core/e2e/dev.test.ts pnpm run test:e2e --reporter=default --reporter=json --outputFile=e2e-windows-nextjs-turbopack.json Stop-Job $job + Get-Content dev-server.log shell: powershell env: APP_NAME: "nextjs-turbopack" @@ -556,7 +557,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: e2e-results-windows-nextjs-turbopack - path: e2e-windows-nextjs-turbopack.json + path: | + e2e-windows-nextjs-turbopack.json + dev-server.log retention-days: 7 if-no-files-found: ignore diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index 5933504d29..229834fcea 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -44,6 +44,12 @@ export function createDevTests(config?: DevTestConfig) { const workflowsDir = finalConfig.workflowsDir ?? 'workflows'; const restoreFiles: Array<{ path: string; content: string }> = []; + const warmEndpoint = async () => { + // ensure Next.js is building route + await fetch(new URL('/api/trigger', process.env.DEPLOYMENT_URL)); + await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL)); + }; + afterEach(async () => { await Promise.all( restoreFiles.map(async (item) => { @@ -54,15 +60,10 @@ export function createDevTests(config?: DevTestConfig) { } }) ); + await warmEndpoint(); restoreFiles.length = 0; }); - const warmEndpoint = async () => { - // ensure Next.js is building route - await fetch(new URL('/api/trigger', process.env.DEPLOYMENT_URL)); - await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL)); - }; - test('should rebuild on workflow change', { timeout: 15_000 }, async () => { const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile); diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 0338171b67..64c0d6e11d 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -331,7 +331,6 @@ export async function getNextBuilder() { if (line.trim()) { try { const message = JSON.parse(line); - console.log('got msg', message); if (message.type === 'file-discovered') { const { filePath, hasWorkflow, hasStep } = message; From 41aba425cebb7b4bfb2378a8a1b14fd4f5176b91 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 18:20:03 -0800 Subject: [PATCH 08/27] remove arbitrary timeout --- packages/next/src/loader.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index 5ea1ed124b..d971af848a 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -82,10 +82,6 @@ async function waitForBuildComplete(): Promise { } let buffer = ''; - const timeout = setTimeout(() => { - socket.off('data', onData); - reject(new Error('Build complete timeout')); - }, 60000); // 60 second timeout const onData = (data: Buffer) => { buffer += data.toString(); @@ -100,7 +96,6 @@ async function waitForBuildComplete(): Promise { try { const message = JSON.parse(line); if (message.type === 'build-complete') { - clearTimeout(timeout); socket.off('data', onData); resolve(); } From 625037c4b0909733d1775726eb3e4835dffa7292 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 18:25:46 -0800 Subject: [PATCH 09/27] update socket path for windows --- packages/next/src/builder.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 64c0d6e11d..05c2080a80 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -251,7 +251,6 @@ export async function getNextBuilder() { return; } const { createServer } = await import('node:net'); - const { tmpdir } = await import('node:os'); const { unlink } = await import('node:fs/promises'); const workflowFiles = new Set(); @@ -300,9 +299,13 @@ export async function getNextBuilder() { }, BUILD_DEBOUNCE_MS); }; - // Generate socket path in tmp directory + // Generate socket path in distDir + const cwd = this.config.workingDir; + const distDir = this.getDistDir(); + const socketDir = join(cwd, distDir); + await mkdir(socketDir, { recursive: true }); const socketPath = join( - tmpdir(), + socketDir, `workflow-${process.pid}-${Date.now()}.sock` ); From 448040846a150b39d96ea9cf16c924dcc2f8290a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 18:34:00 -0800 Subject: [PATCH 10/27] update sock --- packages/next/src/builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 05c2080a80..09bb292c74 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -302,7 +302,7 @@ export async function getNextBuilder() { // Generate socket path in distDir const cwd = this.config.workingDir; const distDir = this.getDistDir(); - const socketDir = join(cwd, distDir); + const socketDir = join(cwd, distDir, 'cache'); await mkdir(socketDir, { recursive: true }); const socketPath = join( socketDir, From ff0a6693117018cebaa7f584e6777dbcd9b7f8ed Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 18:54:49 -0800 Subject: [PATCH 11/27] use a port instead of socket path --- packages/next/src/builder.ts | 44 ++++++++++-------------------------- packages/next/src/loader.ts | 15 ++++++++---- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 09bb292c74..00b23c0ffe 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -1,4 +1,4 @@ -import { constants, unlinkSync } from 'node:fs'; +import { constants } from 'node:fs'; import { access, mkdir, readFile, stat, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import type { NextConfig } from 'next'; @@ -247,11 +247,10 @@ export async function getNextBuilder() { } private async createSocketServer(_usersAppDir: string): Promise { - if (process.env.WORKFLOW_SOCKET_PATH) { + if (process.env.WORKFLOW_SOCKET_PORT) { return; } const { createServer } = await import('node:net'); - const { unlink } = await import('node:fs/promises'); const workflowFiles = new Set(); const stepFiles = new Set(); @@ -259,7 +258,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 250 : 1_000; + process.env.NODE_ENV === 'development' ? 350 : 1_000; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); @@ -299,23 +298,7 @@ export async function getNextBuilder() { }, BUILD_DEBOUNCE_MS); }; - // Generate socket path in distDir - const cwd = this.config.workingDir; - const distDir = this.getDistDir(); - const socketDir = join(cwd, distDir, 'cache'); - await mkdir(socketDir, { recursive: true }); - const socketPath = join( - socketDir, - `workflow-${process.pid}-${Date.now()}.sock` - ); - - process.on('exit', () => { - try { - unlinkSync(socketPath); - } catch {} - }); - - // Create Unix domain socket server + // Create TCP server const server = createServer((socket) => { clients.add(socket); @@ -371,18 +354,15 @@ export async function getNextBuilder() { }); }); - // Clean up existing socket file if it exists - try { - await unlink(socketPath); - } catch { - // Ignore error if file doesn't exist - } - - // Listen on Unix domain socket + // Listen on random available port await new Promise((resolve) => { - server.listen(socketPath, () => { - // Expose the socket path via environment variable - process.env.WORKFLOW_SOCKET_PATH = socketPath; + // Port 0 tells the OS to assign a random available port + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (address && typeof address === 'object') { + // Expose the port via environment variable + process.env.WORKFLOW_SOCKET_PORT = String(address.port); + } resolve(); }); }); diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index d971af848a..e917020e13 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -13,14 +13,21 @@ let socketClientPromise: Promise | null = null; async function getSocketClient() { if (!socketClientPromise) { socketClientPromise = (async () => { - const socketPath = process.env.WORKFLOW_SOCKET_PATH; - if (!socketPath) { + const socketPort = process.env.WORKFLOW_SOCKET_PORT; + if (!socketPort) { throw new Error( - `Invariant: no socket path provided for workflow loader` + `Invariant: no socket port provided for workflow loader` ); } - const socket = connect(socketPath); + const port = Number.parseInt(socketPort, 10); + if (Number.isNaN(port)) { + throw new Error( + `Invariant: invalid socket port provided: ${socketPort}` + ); + } + + const socket = connect({ port, host: '127.0.0.1' }); // Wait for connection await new Promise((resolve, reject) => { From 43e75466bb1cfd7eb842dbd34b25266ac8cb213c Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 19:17:31 -0800 Subject: [PATCH 12/27] skip cache --- .github/workflows/tests.yml | 12 +++++------ package.json | 2 +- packages/core/e2e/dev.test.ts | 40 +++++++++++++++++------------------ 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 750348ced7..c385905313 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -162,7 +162,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --filter='!./workbench/*' + run: pnpm turbo run build --force --filter='!./workbench/*' - name: Generate Workflows Registry run: node workbench/scripts/generate-workflows-registry.js workbench/nextjs-turbopack/workflows workbench/nextjs-turbopack/_workflows.ts @@ -225,7 +225,7 @@ jobs: build-packages: 'false' - name: Build CLI - run: pnpm turbo run build --filter='@workflow/cli' + run: pnpm turbo run build --force --filter='@workflow/cli' - name: Waiting for the Vercel deployment id: waitForDeployment @@ -318,7 +318,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --filter='!./workbench/*' + run: pnpm turbo run build --force --filter='!./workbench/*' - name: Resolve symlinks run: ./scripts/resolve-symlinks.sh workbench/${{ matrix.app.name }} @@ -380,7 +380,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --filter='!./workbench/*' + run: pnpm turbo run build --force --filter='!./workbench/*' - name: Resolve symlinks run: ./scripts/resolve-symlinks.sh workbench/${{ matrix.app.name }} @@ -462,7 +462,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --filter='!./workbench/*' + run: pnpm turbo run build --force --filter='!./workbench/*' - name: Setup PostgreSQL Database run: ./packages/world-postgres/bin/setup.js @@ -529,7 +529,7 @@ jobs: run: pnpm install --frozen-lockfile --ignore-scripts - name: Run Initial Build - run: pnpm turbo run build --filter='!./workbench/*' --filter='!./docs' + run: pnpm turbo run build --force --filter='!./workbench/*' --filter='!./docs' - name: Run E2E Tests (Next.js) run: | diff --git a/package.json b/package.json index 9fd38e3186..f7a2615266 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ }, "scripts": { "prepare": "husky", - "build": "turbo build --filter='./packages/*'", + "build": "turbo build --filter='./packages/*' --force", "test": "turbo test", "clean": "turbo clean", "typecheck": "turbo typecheck", diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index 229834fcea..9f30bf9cc9 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -64,28 +64,28 @@ export function createDevTests(config?: DevTestConfig) { restoreFiles.length = 0; }); - test('should rebuild on workflow change', { timeout: 15_000 }, async () => { - const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile); + test('should rebuild on step change', { timeout: 15_000 }, async () => { + const stepFile = path.join(appPath, workflowsDir, testWorkflowFile); - const content = await fs.readFile(workflowFile, 'utf8'); + const content = await fs.readFile(stepFile, 'utf8'); await fs.writeFile( - workflowFile, + stepFile, `${content} -export async function myNewWorkflow() { - 'use workflow' +export async function myNewStep() { + 'use step' return 'hello world' } ` ); - restoreFiles.push({ path: workflowFile, content }); + restoreFiles.push({ path: stepFile, content }); while (true) { try { await warmEndpoint(); - const workflowContent = await fs.readFile(generatedWorkflow, 'utf8'); - expect(workflowContent).toContain('myNewWorkflow'); + const workflowContent = await fs.readFile(generatedStep, 'utf8'); + expect(workflowContent).toContain('myNewStep'); break; } catch (_) { await new Promise((res) => setTimeout(res, 1_000)); @@ -93,28 +93,28 @@ export async function myNewWorkflow() { } }); - test('should rebuild on step change', { timeout: 15_000 }, async () => { - const stepFile = path.join(appPath, workflowsDir, testWorkflowFile); + test('should rebuild on workflow change', { timeout: 15_000 }, async () => { + const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile); - const content = await fs.readFile(stepFile, 'utf8'); + const content = await fs.readFile(workflowFile, 'utf8'); await fs.writeFile( - stepFile, + workflowFile, `${content} -export async function myNewStep() { - 'use step' +export async function myNewWorkflow() { + 'use workflow' return 'hello world' } ` ); - restoreFiles.push({ path: stepFile, content }); + restoreFiles.push({ path: workflowFile, content }); while (true) { try { await warmEndpoint(); - const workflowContent = await fs.readFile(generatedStep, 'utf8'); - expect(workflowContent).toContain('myNewStep'); + const workflowContent = await fs.readFile(generatedWorkflow, 'utf8'); + expect(workflowContent).toContain('myNewWorkflow'); break; } catch (_) { await new Promise((res) => setTimeout(res, 1_000)); @@ -140,11 +140,11 @@ export async function myNewStep() { } ` ); - restoreFiles.push({ path: workflowFile, content: '' }); const apiFile = path.join(appPath, finalConfig.apiFilePath); - const apiFileContent = await fs.readFile(apiFile, 'utf8'); + restoreFiles.push({ path: apiFile, content: apiFileContent }); + restoreFiles.push({ path: workflowFile, content: '' }); await fs.writeFile( apiFile, From 4848acc280cd63b67fe44920a838176fdca1b01c Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 19:42:27 -0800 Subject: [PATCH 13/27] update stdlib discovering --- .github/workflows/tests.yml | 3 +++ packages/next/src/builder.ts | 3 +-- packages/workflow/src/internal/builtins.ts | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c385905313..7d4f41fa33 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,6 +13,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + NEXT_TELEMETRY_DISABLED: 1 + jobs: # Phase 0: Update PR comment to show tests are running pr-comment-start: diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 00b23c0ffe..82bec7f8d9 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -381,8 +381,7 @@ export async function getNextBuilder() { } private async writeStubFiles(usersAppDir: string): Promise { - const routeStubContent = - "import 'workflow'\n" + "export * from './inner'"; + const routeStubContent = "export * from './inner'"; // this needs to change on each build so can refresh workflows const innerStubContent = 'WORKFLOW_INNER_STUB_FILE_' + Date.now(); const workflowDir = join(usersAppDir, '.well-known/workflow/v1'); diff --git a/packages/workflow/src/internal/builtins.ts b/packages/workflow/src/internal/builtins.ts index 5510c1fad0..bd279a8e40 100644 --- a/packages/workflow/src/internal/builtins.ts +++ b/packages/workflow/src/internal/builtins.ts @@ -3,6 +3,7 @@ * similar to "stdlib" except that are not meant to be imported by users, but are instead "just available" * alongside user defined steps. They are used internally by the runtime */ +export * from '../stdlib.js'; export async function __builtin_response_array_buffer(res: Response) { 'use step'; From a24d0632f77e6f7e53942c7b91a10aca8e2d0e8b Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 20:08:32 -0800 Subject: [PATCH 14/27] updates --- .github/workflows/tests.yml | 3 ++- packages/builders/src/base-builder.ts | 14 +++++++------- packages/core/e2e/build-errors.test.ts | 23 ++++++++--------------- packages/next/src/builder.ts | 3 +++ 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d4f41fa33..d660836229 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -173,7 +173,8 @@ jobs: - name: Run Build Error Tests run: pnpm vitest run packages/core/e2e/build-errors.test.ts env: - APP_NAME: "nextjs-turbopack" + # TODO: move back to turbopack after build mode fixed + APP_NAME: "nextjs-webpack" e2e-vercel-prod: name: E2E Vercel Prod Tests (${{ matrix.app.name }}) diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index 4016b3432c..1cd2654b43 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -320,9 +320,14 @@ export abstract class BaseBuilder { ); }); + const combinedStepFiles = [ + ...stepFiles, + ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : []), + ]; + // Create a virtual entry that imports all files. All step definitions // will get registered thanks to the swc transform. - const imports = stepFiles + const imports = combinedStepFiles .map((file) => { // Normalize both paths to forward slashes before calling relative() // This is critical on Windows where relative() can produce unexpected results with mixed path formats @@ -388,12 +393,7 @@ export abstract class BaseBuilder { plugins: [ createSwcPlugin({ mode: 'step', - entriesToBundle: externalizeNonSteps - ? [ - ...stepFiles, - ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : []), - ] - : undefined, + entriesToBundle: externalizeNonSteps ? combinedStepFiles : undefined, outdir: outfile ? dirname(outfile) : undefined, tsBaseUrl, tsPaths, diff --git a/packages/core/e2e/build-errors.test.ts b/packages/core/e2e/build-errors.test.ts index 43cc6518d7..af500d6e0c 100644 --- a/packages/core/e2e/build-errors.test.ts +++ b/packages/core/e2e/build-errors.test.ts @@ -16,6 +16,8 @@ const exec = promisify(execOriginal); */ describe('build error messages', () => { const restoreFiles: Array<{ path: string; content: string | null }> = []; + const appName = process.env.APP_NAME ?? 'nextjs-turbopack'; + const appPath = getWorkbenchAppPath(appName); afterEach(async () => { // Restore files in reverse order to handle dependencies @@ -27,6 +29,8 @@ describe('build error messages', () => { } } restoreFiles.length = 0; + // previous failures can cause successive tests to fail + await fs.rm(path.join(appPath, '.next'), { recursive: true, force: true }); }); /** @@ -89,11 +93,8 @@ describe('build error messages', () => { test( 'should show helpful error when using Node.js module in workflow', - { timeout: 120_000 }, + { timeout: 60_000 }, async () => { - const appName = process.env.APP_NAME ?? 'nextjs-turbopack'; - const appPath = getWorkbenchAppPath(appName); - // Note: filename must NOT start with _ (those are skipped by registry generator) const badWorkflowContent = ` import { readFileSync } from 'fs'; @@ -138,10 +139,8 @@ export async function nodeModuleViolationWorkflow() { process.env.APP_NAME && process.env.APP_NAME !== 'nextjs-turbopack' )( 'should show top-level package name for external dependencies that use Node.js modules', - { timeout: 120_000 }, + { timeout: 60_000 }, async () => { - const appPath = getWorkbenchAppPath('nextjs-turbopack'); - // @vercel/blob internally uses Node.js modules (via undici) // The error should show "@vercel/blob" not the internal Node.js module const badWorkflowContent = ` @@ -183,11 +182,8 @@ export async function blobViolationWorkflow() { test( 'should show helpful error when using Bun module in workflow', - { timeout: 120_000 }, + { timeout: 60_000 }, async () => { - const appName = process.env.APP_NAME ?? 'nextjs-turbopack'; - const appPath = getWorkbenchAppPath(appName); - // Bun modules should show a different error message than Node.js modules const badWorkflowContent = ` import { serve } from 'bun'; @@ -223,11 +219,8 @@ export async function bunViolationWorkflow() { test( 'should report all violations when multiple Node.js modules are used', - { timeout: 120_000 }, + { timeout: 60_000 }, async () => { - const appName = process.env.APP_NAME ?? 'nextjs-turbopack'; - const appPath = getWorkbenchAppPath(appName); - // Using multiple Node.js modules should report errors for all of them const badWorkflowContent = ` import { readFileSync } from 'fs'; diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 82bec7f8d9..022fe2ee35 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -291,6 +291,9 @@ export async function getNextBuilder() { // Write cache after successful build await this.writeWorkflowsCache(workflowFiles, stepFiles); } catch (error) { + if (process.env.NODE_ENV !== 'development') { + throw error; + } console.error('Build failed:', error); } } From 5cebfeb175fabcda2969a7b51b5915626c9b61a4 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 21:37:06 -0800 Subject: [PATCH 15/27] update resolving --- packages/builders/src/base-builder.ts | 16 +++++++++++++--- packages/next/src/builder.ts | 13 ++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index 1cd2654b43..e91be2bc5b 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -258,7 +258,11 @@ export abstract class BaseBuilder { } } - if (result.warnings && result.warnings.length > 0) { + if ( + this.config.buildTarget !== 'next' && + result.warnings && + result.warnings.length > 0 + ) { console.warn(`! esbuild warnings in ${phase}:`); for (const warning of result.warnings) { console.warn(` ${warning.text}`); @@ -320,9 +324,15 @@ export abstract class BaseBuilder { ); }); - const combinedStepFiles = [ + const combinedStepFiles: string[] = [ ...stepFiles, - ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : []), + ...(resolvedBuiltInSteps + ? [ + resolvedBuiltInSteps, + // TODO: expose this in workflow/package.json and use resolve? + join(dirname(resolvedBuiltInSteps), '../stdlib.js'), + ] + : []), ]; // Create a virtual entry that imports all files. All step definitions diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 022fe2ee35..19d424d508 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -258,7 +258,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 350 : 1_000; + process.env.NODE_ENV === 'development' ? 350 : 3_000; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); @@ -272,6 +272,8 @@ export async function getNextBuilder() { } // Debounced build trigger + let buildTriggered = false; + const triggerBuild = () => { if (debounceTimer) { clearTimeout(debounceTimer); @@ -284,7 +286,8 @@ export async function getNextBuilder() { if (inputFiles.length > 0) { console.log( - `Triggering build with ${inputFiles.length} discovered files` + `Triggering build with ${inputFiles.length} discovered files`, + new Date().toLocaleTimeString() ); try { await this.build(inputFiles); @@ -297,6 +300,7 @@ export async function getNextBuilder() { console.error('Build failed:', error); } } + buildTriggered = true; debounceTimer = null; }, BUILD_DEBOUNCE_MS); }; @@ -336,7 +340,10 @@ export async function getNextBuilder() { triggerBuild(); } else if (message.type === 'trigger-build') { // enqueue new build if one isn't already pending - if (!debounceTimer) { + if ( + !debounceTimer && + !(process.env.NODE_ENV === 'production' && buildTriggered) + ) { triggerBuild(); } } From ef2a5eddfeeba6402e6e5a93b4390c0f691dce7f Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Dec 2025 22:54:17 -0800 Subject: [PATCH 16/27] temp fixes --- .github/workflows/benchmarks.yml | 10 +++++----- .github/workflows/tests.yml | 12 ++++++------ package.json | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 71eb9800d6..6d604c1c02 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -146,7 +146,7 @@ jobs: fail-fast: false matrix: # Note: Use actual directory names, not symlinks (nitro -> nitro-v3) - app: [nextjs-turbopack, nitro-v3, express] + app: [nextjs-webpack, nitro-v3, express] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} @@ -223,7 +223,7 @@ jobs: fail-fast: false matrix: # Note: Use actual directory names, not symlinks (nitro -> nitro-v3) - app: [nextjs-turbopack, nitro-v3, express] + app: [nextjs-webpack, nitro-v3, express] services: postgres: @@ -321,9 +321,9 @@ jobs: fail-fast: false matrix: app: - - name: "nextjs-turbopack" - project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi" - project-slug: "example-nextjs-workflow-turbopack" + - name: "nextjs-webpack" + project-id: "prj_avRPBF3eWjh6iDNQgmhH4VOg27h0" + project-slug: "example-nextjs-workflow-webpack" - name: "nitro-v3" project-id: "prj_e7DZirYdLrQKXNrlxg7KmA6ABx8r" project-slug: "workbench-nitro-workflow" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d660836229..94ed05f21d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -165,7 +165,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --force --filter='!./workbench/*' + run: pnpm turbo run build --filter='!./workbench/*' - name: Generate Workflows Registry run: node workbench/scripts/generate-workflows-registry.js workbench/nextjs-turbopack/workflows workbench/nextjs-turbopack/_workflows.ts @@ -229,7 +229,7 @@ jobs: build-packages: 'false' - name: Build CLI - run: pnpm turbo run build --force --filter='@workflow/cli' + run: pnpm turbo run build --filter='@workflow/cli' - name: Waiting for the Vercel deployment id: waitForDeployment @@ -322,7 +322,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --force --filter='!./workbench/*' + run: pnpm turbo run build --filter='!./workbench/*' - name: Resolve symlinks run: ./scripts/resolve-symlinks.sh workbench/${{ matrix.app.name }} @@ -384,7 +384,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --force --filter='!./workbench/*' + run: pnpm turbo run build --filter='!./workbench/*' - name: Resolve symlinks run: ./scripts/resolve-symlinks.sh workbench/${{ matrix.app.name }} @@ -466,7 +466,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Run Initial Build - run: pnpm turbo run build --force --filter='!./workbench/*' + run: pnpm turbo run build --filter='!./workbench/*' - name: Setup PostgreSQL Database run: ./packages/world-postgres/bin/setup.js @@ -533,7 +533,7 @@ jobs: run: pnpm install --frozen-lockfile --ignore-scripts - name: Run Initial Build - run: pnpm turbo run build --force --filter='!./workbench/*' --filter='!./docs' + run: pnpm turbo run build --filter='!./workbench/*' --filter='!./docs' - name: Run E2E Tests (Next.js) run: | diff --git a/package.json b/package.json index f7a2615266..9fd38e3186 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ }, "scripts": { "prepare": "husky", - "build": "turbo build --filter='./packages/*' --force", + "build": "turbo build --filter='./packages/*'", "test": "turbo test", "clean": "turbo clean", "typecheck": "turbo typecheck", From bb68808e662cd2bd59cefb1aa4b7143455f882ce Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 18 Dec 2025 15:12:36 -0800 Subject: [PATCH 17/27] set config to avoid scope hoisting issue --- .github/workflows/benchmarks.yml | 10 +++++----- .github/workflows/tests.yml | 16 ++++++---------- packages/next/src/builder.ts | 11 ++--------- packages/next/src/index.ts | 7 +++++++ packages/next/src/loader.ts | 30 +++++++++++++----------------- 5 files changed, 33 insertions(+), 41 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6d604c1c02..71eb9800d6 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -146,7 +146,7 @@ jobs: fail-fast: false matrix: # Note: Use actual directory names, not symlinks (nitro -> nitro-v3) - app: [nextjs-webpack, nitro-v3, express] + app: [nextjs-turbopack, nitro-v3, express] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} @@ -223,7 +223,7 @@ jobs: fail-fast: false matrix: # Note: Use actual directory names, not symlinks (nitro -> nitro-v3) - app: [nextjs-webpack, nitro-v3, express] + app: [nextjs-turbopack, nitro-v3, express] services: postgres: @@ -321,9 +321,9 @@ jobs: fail-fast: false matrix: app: - - name: "nextjs-webpack" - project-id: "prj_avRPBF3eWjh6iDNQgmhH4VOg27h0" - project-slug: "example-nextjs-workflow-webpack" + - name: "nextjs-turbopack" + project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi" + project-slug: "example-nextjs-workflow-turbopack" - name: "nitro-v3" project-id: "prj_e7DZirYdLrQKXNrlxg7KmA6ABx8r" project-slug: "workbench-nitro-workflow" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 94ed05f21d..257f797e95 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,14 +8,14 @@ on: - "!*" pull_request: +env: + NEXT_TELEMETRY_DISABLED: 1 + concurrency: # Unique group for this workflow and branch group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - NEXT_TELEMETRY_DISABLED: 1 - jobs: # Phase 0: Update PR comment to show tests are running pr-comment-start: @@ -173,8 +173,7 @@ jobs: - name: Run Build Error Tests run: pnpm vitest run packages/core/e2e/build-errors.test.ts env: - # TODO: move back to turbopack after build mode fixed - APP_NAME: "nextjs-webpack" + APP_NAME: "nextjs-turbopack" e2e-vercel-prod: name: E2E Vercel Prod Tests (${{ matrix.app.name }}) @@ -538,13 +537,12 @@ jobs: - name: Run E2E Tests (Next.js) run: | cd workbench/nextjs-turbopack - $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev 2>&1 | Tee-Object -FilePath "../../dev-server.log" } + $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev } Start-Sleep -Seconds 15 cd ../.. pnpm vitest run packages/core/e2e/dev.test.ts pnpm run test:e2e --reporter=default --reporter=json --outputFile=e2e-windows-nextjs-turbopack.json Stop-Job $job - Get-Content dev-server.log shell: powershell env: APP_NAME: "nextjs-turbopack" @@ -561,9 +559,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: e2e-results-windows-nextjs-turbopack - path: | - e2e-windows-nextjs-turbopack.json - dev-server.log + path: e2e-windows-nextjs-turbopack.json retention-days: 7 if-no-files-found: ignore diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 19d424d508..3b2b15d893 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -258,7 +258,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 350 : 3_000; + process.env.NODE_ENV === 'development' ? 350 : 1_000; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); @@ -272,7 +272,6 @@ export async function getNextBuilder() { } // Debounced build trigger - let buildTriggered = false; const triggerBuild = () => { if (debounceTimer) { @@ -300,7 +299,6 @@ export async function getNextBuilder() { console.error('Build failed:', error); } } - buildTriggered = true; debounceTimer = null; }, BUILD_DEBOUNCE_MS); }; @@ -340,12 +338,7 @@ export async function getNextBuilder() { triggerBuild(); } else if (message.type === 'trigger-build') { // enqueue new build if one isn't already pending - if ( - !debounceTimer && - !(process.env.NODE_ENV === 'production' && buildTriggered) - ) { - triggerBuild(); - } + triggerBuild(); } } catch (error) { console.error('Failed to parse socket message:', error); diff --git a/packages/next/src/index.ts b/packages/next/src/index.ts index b749fe9cfa..bd24161bc4 100644 --- a/packages/next/src/index.ts +++ b/packages/next/src/index.ts @@ -93,6 +93,13 @@ export function withWorkflow( }; } + if (process.env.TURBOPACK) { + if (!nextConfig.experimental) { + nextConfig.experimental = {}; + } + nextConfig.experimental.turbopackScopeHoisting = false; + } + // configure the loader for webpack const existingWebpackModify = nextConfig.webpack; nextConfig.webpack = (...args) => { diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index e917020e13..c69e2eb97d 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -59,24 +59,20 @@ async function notifySocketServer( hasWorkflow: boolean, hasStep: boolean ) { - try { - const socket = await getSocketClient(); - if (!socket) { - throw new Error(`Invariant: missing workflow socket connection`); - } - - // Send single message with both workflow and step information - const message = - JSON.stringify({ - type: 'file-discovered', - filePath: filename, - hasWorkflow, - hasStep, - }) + '\n'; - socket.write(message); - } catch { - // Silently fail - socket server might not be available yet + const socket = await getSocketClient(); + if (!socket) { + throw new Error(`Invariant: missing workflow socket connection`); } + + // Send single message with both workflow and step information + const message = + JSON.stringify({ + type: 'file-discovered', + filePath: filename, + hasWorkflow, + hasStep, + }) + '\n'; + socket.write(message); } async function waitForBuildComplete(): Promise { From b11d7e8226451c45c52ba754585f945cc807f04f Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 18 Dec 2025 15:20:41 -0800 Subject: [PATCH 18/27] bump --- packages/next/src/builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 3b2b15d893..16f6e03205 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -258,7 +258,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 350 : 1_000; + process.env.NODE_ENV === 'development' ? 350 : 2_000; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); From ea47de8f6e32bb97d3bc2c2fab8b3a582fc88d08 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Fri, 19 Dec 2025 13:06:51 -0800 Subject: [PATCH 19/27] add log --- packages/core/e2e/local-build.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/core/e2e/local-build.test.ts b/packages/core/e2e/local-build.test.ts index b8f01e858f..8c4f55cf7c 100644 --- a/packages/core/e2e/local-build.test.ts +++ b/packages/core/e2e/local-build.test.ts @@ -27,6 +27,9 @@ describe.each([ cwd: getWorkbenchAppPath(project), }); + console.log(result.stdout); + console.log(result.stderr); + expect(result.stderr).not.toContain('Error:'); }); }); From 3eded37f63df7a4c8cb3238349a18acca8b03a72 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Fri, 19 Dec 2025 13:14:01 -0800 Subject: [PATCH 20/27] adjust debounce --- packages/next/src/builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 16f6e03205..28a52717b4 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -258,7 +258,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 350 : 2_000; + process.env.NODE_ENV === 'development' ? 500 : 5_000; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); From 730e71a7cf1cf955e19bd7d10be0a34c67579e63 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Fri, 19 Dec 2025 13:21:19 -0800 Subject: [PATCH 21/27] always give heartbeat from loader --- packages/next/src/builder.ts | 2 +- packages/next/src/loader.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 28a52717b4..689cd6904f 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -258,7 +258,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 500 : 5_000; + process.env.NODE_ENV === 'development' ? 500 : 2_500; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index c69e2eb97d..84d7b4291f 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -159,14 +159,14 @@ export default async function workflowLoader( const hasWorkflow = useWorkflowPattern.test(normalizedSource); const hasStep = useStepPattern.test(normalizedSource); + // Send message to socket server if workflow or step detected + await notifySocketServer(filename, hasWorkflow, hasStep); + // only apply the transform if file needs it if (!hasWorkflow && !hasStep) { return normalizedSource; } - // Send message to socket server if workflow or step detected - await notifySocketServer(filename, hasWorkflow, hasStep); - const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx') || From 1da5189642d009c91808443767b8b9a3238bc20b Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Fri, 19 Dec 2025 13:29:13 -0800 Subject: [PATCH 22/27] update --- packages/next/src/builder.ts | 1 + packages/next/src/loader.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 689cd6904f..4a2d809907 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -305,6 +305,7 @@ export async function getNextBuilder() { // Create TCP server const server = createServer((socket) => { + socket.setNoDelay(true); clients.add(socket); let buffer = ''; diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index 84d7b4291f..fe5fc9e0b2 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -37,6 +37,7 @@ async function getSocketClient() { }, 1000); socket.on('connect', () => { + socket.setNoDelay(true); clearTimeout(timeout); resolve(); }); From 3f1ac4f49b22911d552198b66716ab8d4d062f55 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Fri, 19 Dec 2025 16:56:44 -0800 Subject: [PATCH 23/27] remove condition --- packages/core/e2e/local-build.test.ts | 1 + packages/next/package.json | 4 +- packages/next/src/builder.ts | 130 +++++++++++++-------- packages/next/src/index.ts | 19 --- packages/next/src/loader.ts | 7 +- packages/world-testing/src/idempotency.mts | 4 +- pnpm-lock.yaml | 11 -- 7 files changed, 88 insertions(+), 88 deletions(-) diff --git a/packages/core/e2e/local-build.test.ts b/packages/core/e2e/local-build.test.ts index 8c4f55cf7c..e27ba4323c 100644 --- a/packages/core/e2e/local-build.test.ts +++ b/packages/core/e2e/local-build.test.ts @@ -25,6 +25,7 @@ describe.each([ const result = await exec('pnpm build', { cwd: getWorkbenchAppPath(project), + timeout: 120_000, }); console.log(result.stdout); diff --git a/packages/next/package.json b/packages/next/package.json index 7d84c3b97b..471be25dcf 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -30,13 +30,11 @@ "@swc/core": "catalog:", "@workflow/builders": "workspace:*", "@workflow/core": "workspace:*", - "@workflow/swc-plugin": "workspace:*", - "semver": "7.7.3" + "@workflow/swc-plugin": "workspace:*" }, "devDependencies": { "@workflow/tsconfig": "workspace:*", "@types/node": "catalog:", - "@types/semver": "7.7.1", "next": "16.0.10" }, "peerDependencies": { diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 4a2d809907..a71e676fca 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -255,59 +255,17 @@ export async function getNextBuilder() { const workflowFiles = new Set(); const stepFiles = new Set(); const clients = new Set(); - let debounceTimer: NodeJS.Timeout | null = null; - - const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 500 : 2_500; - - // Attempt to load cached workflows/steps from previous build - const cache = await this.readWorkflowsCache(); - if (cache) { - for (const file of cache.workflowFiles) { - workflowFiles.add(file); - } - for (const file of cache.stepFiles) { - stepFiles.add(file); - } - } - - // Debounced build trigger - - const triggerBuild = () => { - if (debounceTimer) { - clearTimeout(debounceTimer); - } - - debounceTimer = setTimeout(async () => { - // Combine workflow and step files into single array - const allFiles = new Set([...workflowFiles, ...stepFiles]); - const inputFiles = Array.from(allFiles); - - if (inputFiles.length > 0) { - console.log( - `Triggering build with ${inputFiles.length} discovered files`, - new Date().toLocaleTimeString() - ); - try { - await this.build(inputFiles); - // Write cache after successful build - await this.writeWorkflowsCache(workflowFiles, stepFiles); - } catch (error) { - if (process.env.NODE_ENV !== 'development') { - throw error; - } - console.error('Build failed:', error); - } - } - debounceTimer = null; - }, BUILD_DEBOUNCE_MS); - }; + let buildTriggered = false; // Create TCP server const server = createServer((socket) => { socket.setNoDelay(true); clients.add(socket); + if (buildTriggered && process.env.NODE_ENV === 'production') { + socket.write(JSON.stringify({ type: 'build-complete' }) + '\n'); + } + let buffer = ''; socket.on('data', (data) => { @@ -326,6 +284,13 @@ export async function getNextBuilder() { if (message.type === 'file-discovered') { const { filePath, hasWorkflow, hasStep } = message; + console.log( + 'file-discovered', + message.filePath, + 'pid:', + process.pid, + new Date().toLocaleTimeString() + ); if (hasWorkflow) { workflowFiles.add(filePath); @@ -367,6 +332,10 @@ export async function getNextBuilder() { // Expose the port via environment variable process.env.WORKFLOW_SOCKET_PORT = String(address.port); } + console.log( + 'started socket server', + process.env.WORKFLOW_SOCKET_PORT + ); resolve(); }); }); @@ -382,9 +351,73 @@ export async function getNextBuilder() { } }, }; + + let debounceTimer: NodeJS.Timeout | null = null; + + const BUILD_DEBOUNCE_MS = + process.env.NODE_ENV === 'development' ? 500 : 1_000; + + // Attempt to load cached workflows/steps from previous build + const cache = await this.readWorkflowsCache(); + if (cache) { + for (const file of cache.workflowFiles) { + workflowFiles.add(file); + } + for (const file of cache.stepFiles) { + stepFiles.add(file); + } + } + + // Debounced build trigger + + const triggerBuild = () => { + if (buildTriggered && process.env.NODE_ENV === 'production') { + // can't run another build after one has already been done + // in production mode as it won't have any affect since after + // the first is done we resolve the loaders for the stub entries + // and they can't be refreshed/rebuilt after that in production + return; + } + + if (debounceTimer) { + clearTimeout(debounceTimer); + } + + debounceTimer = setTimeout(async () => { + // Combine workflow and step files into single array + const allFiles = new Set([...workflowFiles, ...stepFiles]); + const inputFiles = Array.from(allFiles); + + if (inputFiles.length > 0) { + console.log( + `Triggering build with ${inputFiles.length} discovered files`, + new Date().toLocaleTimeString(), + 'pid:', + process.pid + ); + try { + await this.build(inputFiles); + // Write cache after successful build + await this.writeWorkflowsCache(workflowFiles, stepFiles); + buildTriggered = true; + } catch (error) { + if (process.env.NODE_ENV !== 'development') { + throw error; + } + console.error('Build failed:', error); + } + } + debounceTimer = null; + }, BUILD_DEBOUNCE_MS); + }; } private async writeStubFiles(usersAppDir: string): Promise { + // NOTE: there is a limitation with turbopack that we can only + // have number of virtual entries with pending promise less than + // CPU count as that's the number of workers it uses so currently + // we're fine with > 3 vCPU but <= 3 vCPUs and we won't be able to + // discover workflows/steps const routeStubContent = "export * from './inner'"; // this needs to change on each build so can refresh workflows const innerStubContent = 'WORKFLOW_INNER_STUB_FILE_' + Date.now(); @@ -393,13 +426,11 @@ export async function getNextBuilder() { // Ensure directories exist await mkdir(join(workflowDir, 'flow'), { recursive: true }); await mkdir(join(workflowDir, 'step'), { recursive: true }); - await mkdir(join(workflowDir, 'webhook'), { recursive: true }); await mkdir(join(workflowDir, 'webhook/[token]'), { recursive: true }); // Write route.ts stub files (re-export from inner) await writeFile(join(workflowDir, 'flow/route.js'), routeStubContent); await writeFile(join(workflowDir, 'step/route.js'), routeStubContent); - await writeFile(join(workflowDir, 'webhook/route.js'), routeStubContent); await writeFile( join(workflowDir, 'webhook/[token]/route.js'), routeStubContent @@ -408,7 +439,6 @@ export async function getNextBuilder() { // Write inner.js stub files (actual stub marker) await writeFile(join(workflowDir, 'flow/inner.js'), innerStubContent); await writeFile(join(workflowDir, 'step/inner.js'), innerStubContent); - await writeFile(join(workflowDir, 'webhook/inner.js'), innerStubContent); await writeFile( join(workflowDir, 'webhook/[token]/inner.js'), innerStubContent diff --git a/packages/next/src/index.ts b/packages/next/src/index.ts index bd24161bc4..b979e5eae1 100644 --- a/packages/next/src/index.ts +++ b/packages/next/src/index.ts @@ -1,5 +1,4 @@ import type { NextConfig } from 'next'; -import semver from 'semver'; import { getNextBuilder } from './builder.js'; export function withWorkflow( @@ -59,8 +58,6 @@ export function withWorkflow( nextConfig.turbopack.rules = {}; } const existingRules = nextConfig.turbopack.rules as any; - const nextVersion = require('next/package.json').version; - const supportsTurboCondition = semver.gte(nextVersion, 'v16.0.0'); for (const key of [ '*.tsx', @@ -73,22 +70,6 @@ export function withWorkflow( '*.cts', ]) { nextConfig.turbopack.rules[key] = { - ...(supportsTurboCondition - ? { - condition: { - ...existingRules[key]?.condition, - any: [ - ...(existingRules[key]?.condition.any || []), - { - content: /(use workflow|use step)/, - }, - { - path: /.*\.well-known[/\\]workflow.*/, - }, - ], - }, - } - : {}), loaders: [...(existingRules[key]?.loaders || []), loaderPath], }; } diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index fe5fc9e0b2..1b71d8adf4 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -113,8 +113,8 @@ async function waitForBuildComplete(): Promise { socket.on('data', onData); // Send trigger-build message - const message = JSON.stringify({ type: 'trigger-build' }) + '\n'; - socket.write(message); + // const message = JSON.stringify({ type: 'trigger-build' }) + '\n'; + // socket.write(message); }); } @@ -138,7 +138,6 @@ export default async function workflowLoader( normalizedFilename.includes('.well-known/workflow/v1/') && (normalizedFilename.includes('/flow/inner.js') || normalizedFilename.includes('/step/inner.js') || - normalizedFilename.includes('/webhook/inner.js') || normalizedFilename.includes('/webhook/[token]/inner.js')); if ( @@ -146,7 +145,9 @@ export default async function workflowLoader( normalizedSource.trim().startsWith(STUB_CONTENT) ) { // Wait for build to complete + console.log('waiting for build complete', filename); await waitForBuildComplete(); + console.log('build complete', filename); // Read the actual generated file content const actualContent = await readFile( diff --git a/packages/world-testing/src/idempotency.mts b/packages/world-testing/src/idempotency.mts index b4bcbe4cdf..212a27ffbe 100644 --- a/packages/world-testing/src/idempotency.mts +++ b/packages/world-testing/src/idempotency.mts @@ -3,7 +3,7 @@ import { hydrateWorkflowReturnValue } from 'workflow/internal/serialization'; import { createFetcher, startServer } from './util.mjs'; export function idempotency(world: string) { - test('idempotency', { timeout: 60_000 }, async () => { + test('idempotency', { timeout: 120_000 }, async () => { const server = await startServer({ world }).then(createFetcher); const result = await server.invoke('workflows/noop.ts', 'brokenWf', [1, 2]); expect(result.runId).toMatch(/^wrun_.+/); @@ -17,7 +17,7 @@ export function idempotency(world: string) { }, { interval: 200, - timeout: 59_000, + timeout: 120_000, } ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b675c488ea..a79fc4d310 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -616,16 +616,10 @@ importers: '@workflow/swc-plugin': specifier: workspace:* version: link:../swc-plugin-workflow - semver: - specifier: 7.7.3 - version: 7.7.3 devDependencies: '@types/node': specifier: 'catalog:' version: 22.19.0 - '@types/semver': - specifier: 7.7.1 - version: 7.7.1 '@workflow/tsconfig': specifier: workspace:* version: link:../tsconfig @@ -6500,9 +6494,6 @@ packages: '@types/seedrandom@3.0.8': resolution: {integrity: sha512-TY1eezMU2zH2ozQoAFAQFOPpvP15g+ZgSfTZt31AUUH/Rxtnz3H+A/Sv1Snw2/amp//omibc+AEkTaA8KUeOLQ==} - '@types/semver@7.7.1': - resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} - '@types/send@0.17.6': resolution: {integrity: sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==} @@ -19093,8 +19084,6 @@ snapshots: '@types/seedrandom@3.0.8': {} - '@types/semver@7.7.1': {} - '@types/send@0.17.6': dependencies: '@types/mime': 1.3.5 From 3c3bba00084880194d38738957dc46b7c2d81e4d Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 20 Dec 2025 11:20:18 -0800 Subject: [PATCH 24/27] add warning --- .github/workflows/tests.yml | 18 ++++++++++++++---- packages/next/package.json | 3 +-- packages/next/src/builder.ts | 32 +++++++++++--------------------- packages/next/src/loader.ts | 2 -- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 257f797e95..d4bbb8a88c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -537,12 +537,22 @@ jobs: - name: Run E2E Tests (Next.js) run: | cd workbench/nextjs-turbopack - $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev } + $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev *>&1 | Tee-Object -FilePath "$using:PWD\dev.log" } Start-Sleep -Seconds 15 cd ../.. - pnpm vitest run packages/core/e2e/dev.test.ts - pnpm run test:e2e --reporter=default --reporter=json --outputFile=e2e-windows-nextjs-turbopack.json - Stop-Job $job + + try { + pnpm run test:e2e --reporter=default --reporter=json --outputFile=e2e-windows-nextjs-turbopack.json + pnpm vitest run packages/core/e2e/dev.test.ts + } finally { + Stop-Job $job + Write-Host "`n=== Dev Server Logs ===`n" + if (Test-Path "workbench/nextjs-turbopack/dev.log") { + Get-Content "workbench/nextjs-turbopack/dev.log" + } else { + Write-Host "No dev.log file found" + } + } shell: powershell env: APP_NAME: "nextjs-turbopack" diff --git a/packages/next/package.json b/packages/next/package.json index 471be25dcf..8d8b98db61 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -18,8 +18,7 @@ }, "exports": { ".": "./dist/index.js", - "./loader": "./dist/loader.js", - "./runtime": "./dist/runtime.js" + "./loader": "./dist/loader.js" }, "scripts": { "build": "tsc", diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index a71e676fca..72a1f7a84f 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -1,5 +1,6 @@ import { constants } from 'node:fs'; import { access, mkdir, readFile, stat, writeFile } from 'node:fs/promises'; +import os from 'node:os'; import { join, resolve } from 'node:path'; import type { NextConfig } from 'next'; @@ -284,13 +285,6 @@ export async function getNextBuilder() { if (message.type === 'file-discovered') { const { filePath, hasWorkflow, hasStep } = message; - console.log( - 'file-discovered', - message.filePath, - 'pid:', - process.pid, - new Date().toLocaleTimeString() - ); if (hasWorkflow) { workflowFiles.add(filePath); @@ -332,10 +326,6 @@ export async function getNextBuilder() { // Expose the port via environment variable process.env.WORKFLOW_SOCKET_PORT = String(address.port); } - console.log( - 'started socket server', - process.env.WORKFLOW_SOCKET_PORT - ); resolve(); }); }); @@ -355,7 +345,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 500 : 1_000; + process.env.NODE_ENV === 'development' ? 500 : 1_500; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); @@ -389,25 +379,18 @@ export async function getNextBuilder() { const inputFiles = Array.from(allFiles); if (inputFiles.length > 0) { - console.log( - `Triggering build with ${inputFiles.length} discovered files`, - new Date().toLocaleTimeString(), - 'pid:', - process.pid - ); try { + buildTriggered = true; await this.build(inputFiles); // Write cache after successful build await this.writeWorkflowsCache(workflowFiles, stepFiles); - buildTriggered = true; } catch (error) { if (process.env.NODE_ENV !== 'development') { throw error; } - console.error('Build failed:', error); + console.error('Workflows build failed:', error); } } - debounceTimer = null; }, BUILD_DEBOUNCE_MS); }; } @@ -418,6 +401,13 @@ export async function getNextBuilder() { // CPU count as that's the number of workers it uses so currently // we're fine with > 3 vCPU but <= 3 vCPUs and we won't be able to // discover workflows/steps + const parallelismCount = os.availableParallelism(); + if (parallelismCount < 4) { + console.warn( + `Available parallelism of ${parallelismCount} is less than needed 4. This can cause workflows/steps to fail to discover properly in turbopack` + ); + } + const routeStubContent = "export * from './inner'"; // this needs to change on each build so can refresh workflows const innerStubContent = 'WORKFLOW_INNER_STUB_FILE_' + Date.now(); diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index 1b71d8adf4..7dd150ad84 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -145,9 +145,7 @@ export default async function workflowLoader( normalizedSource.trim().startsWith(STUB_CONTENT) ) { // Wait for build to complete - console.log('waiting for build complete', filename); await waitForBuildComplete(); - console.log('build complete', filename); // Read the actual generated file content const actualContent = await readFile( From df01356e0d958369f665369ccb752e5c2f43c27a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 20 Dec 2025 14:36:09 -0800 Subject: [PATCH 25/27] only recompile if workflow related event --- packages/next/src/builder.ts | 66 ++++++++++++++++++++++-------------- packages/next/src/index.ts | 4 +-- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 72a1f7a84f..887b5825db 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -22,10 +22,12 @@ export async function getNextBuilder() { class NextBuilder extends BaseBuilderClass { private socketIO?: any; + private isDevServer?: boolean; private nextConfig?: NextConfig; - setNextConfig(config: NextConfig) { + setNextConfig(config: NextConfig, phase: string) { this.nextConfig = config; + this.isDevServer = phase === 'phase-development-server'; } private getDistDir(): string { @@ -129,7 +131,7 @@ export async function getNextBuilder() { private async writeFunctionsConfig(outputDir: string) { // we don't run this in development mode as it's not needed - if (process.env.NODE_ENV === 'development') { + if (this.isDevServer) { return; } const generatedConfig = { @@ -263,7 +265,7 @@ export async function getNextBuilder() { socket.setNoDelay(true); clients.add(socket); - if (buildTriggered && process.env.NODE_ENV === 'production') { + if (buildTriggered && !this.isDevServer) { socket.write(JSON.stringify({ type: 'build-complete' }) + '\n'); } @@ -286,16 +288,32 @@ export async function getNextBuilder() { if (message.type === 'file-discovered') { const { filePath, hasWorkflow, hasStep } = message; + const knownFile = + workflowFiles.has(filePath) || stepFiles.has(filePath); + if (hasWorkflow) { workflowFiles.add(filePath); + } else { + workflowFiles.delete(filePath); } if (hasStep) { stepFiles.add(filePath); + } else { + stepFiles.delete(filePath); } - // Trigger debounced build - triggerBuild(); + // Trigger debounced build if the file was previously seen + // or was steps or workflows currently + if ( + // in non-dev we always update debounce on activity + !this.isDevServer || + hasWorkflow || + hasStep || + knownFile + ) { + triggerBuild(); + } } else if (message.type === 'trigger-build') { // enqueue new build if one isn't already pending triggerBuild(); @@ -344,8 +362,7 @@ export async function getNextBuilder() { let debounceTimer: NodeJS.Timeout | null = null; - const BUILD_DEBOUNCE_MS = - process.env.NODE_ENV === 'development' ? 500 : 1_500; + const BUILD_DEBOUNCE_MS = this.isDevServer ? 500 : 2_000; // Attempt to load cached workflows/steps from previous build const cache = await this.readWorkflowsCache(); @@ -361,35 +378,32 @@ export async function getNextBuilder() { // Debounced build trigger const triggerBuild = () => { - if (buildTriggered && process.env.NODE_ENV === 'production') { - // can't run another build after one has already been done - // in production mode as it won't have any affect since after - // the first is done we resolve the loaders for the stub entries - // and they can't be refreshed/rebuilt after that in production - return; - } - if (debounceTimer) { clearTimeout(debounceTimer); } debounceTimer = setTimeout(async () => { + if (buildTriggered && !this.isDevServer) { + // can't run another build after one has already been done + // in production mode as it won't have any affect since after + // the first is done we resolve the loaders for the stub entries + // and they can't be refreshed/rebuilt after that in production + return; + } // Combine workflow and step files into single array const allFiles = new Set([...workflowFiles, ...stepFiles]); const inputFiles = Array.from(allFiles); - if (inputFiles.length > 0) { - try { - buildTriggered = true; - await this.build(inputFiles); - // Write cache after successful build - await this.writeWorkflowsCache(workflowFiles, stepFiles); - } catch (error) { - if (process.env.NODE_ENV !== 'development') { - throw error; - } - console.error('Workflows build failed:', error); + try { + buildTriggered = true; + await this.build(inputFiles); + // Write cache after successful build + await this.writeWorkflowsCache(workflowFiles, stepFiles); + } catch (error) { + if (!this.isDevServer) { + throw error; } + console.error('Workflows build failed:', error); } }, BUILD_DEBOUNCE_MS); }; diff --git a/packages/next/src/index.ts b/packages/next/src/index.ts index b979e5eae1..c18ccaf911 100644 --- a/packages/next/src/index.ts +++ b/packages/next/src/index.ts @@ -108,10 +108,8 @@ export function withWorkflow( !process.env.WORKFLOW_NEXT_PRIVATE_BUILT && phase !== 'phase-production-server' ) { - const shouldWatch = process.env.NODE_ENV === 'development'; const NextBuilder = await getNextBuilder(); const workflowBuilder = new NextBuilder({ - watch: shouldWatch, // discover workflows from pages/app entries dirs: ['pages', 'app', 'src/pages', 'src/app'], workingDir: process.cwd(), @@ -123,7 +121,7 @@ export function withWorkflow( }); // Pass the nextConfig to the builder so it can access distDir - workflowBuilder.setNextConfig(nextConfig); + workflowBuilder.setNextConfig(nextConfig, phase); await workflowBuilder.init(); process.env.WORKFLOW_NEXT_PRIVATE_BUILT = '1'; From 88dcb1b0251b92655ec37e8ef83dd51ad9135a8e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 22 Dec 2025 13:09:32 -0600 Subject: [PATCH 26/27] add socket validation and comments from review --- packages/core/e2e/dev.test.ts | 10 +- packages/next/src/builder.ts | 157 ++++++----------- packages/next/src/loader.ts | 115 ++++++++++--- packages/next/src/socket-server.ts | 191 +++++++++++++++++++++ packages/workflow/src/internal/builtins.ts | 2 + packages/workflow/tsconfig.json | 1 + workbench/nextjs-turbopack/next.config.ts | 1 + 7 files changed, 340 insertions(+), 137 deletions(-) create mode 100644 packages/next/src/socket-server.ts diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index 9f30bf9cc9..c059ea3a4b 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -1,6 +1,6 @@ -import fs from 'fs/promises'; -import path from 'path'; -import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, describe, expect, test } from 'vitest'; import { getWorkbenchAppPath } from './utils'; export interface DevTestConfig { @@ -45,7 +45,9 @@ export function createDevTests(config?: DevTestConfig) { const restoreFiles: Array<{ path: string; content: string }> = []; const warmEndpoint = async () => { - // ensure Next.js is building route + // Warm up the Next.js routes to trigger lazy workflow/step discovery and compilation. + // This is only required for tests that respond to file updates (HMR tests). + // Without this, the routes won't be built yet and file change detection won't work. await fetch(new URL('/api/trigger', process.env.DEPLOYMENT_URL)); await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL)); }; diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 887b5825db..6ef82131df 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -3,6 +3,11 @@ import { access, mkdir, readFile, stat, writeFile } from 'node:fs/promises'; import os from 'node:os'; import { join, resolve } from 'node:path'; import type { NextConfig } from 'next'; +import { + createSocketServer, + type SocketIO, + type SocketServerConfig, +} from './socket-server'; let CachedNextBuilder: any; @@ -21,7 +26,7 @@ export async function getNextBuilder() { } = await import('@workflow/builders'); class NextBuilder extends BaseBuilderClass { - private socketIO?: any; + private socketIO?: SocketIO; private isDevServer?: boolean; private nextConfig?: NextConfig; @@ -253,115 +258,11 @@ export async function getNextBuilder() { if (process.env.WORKFLOW_SOCKET_PORT) { return; } - const { createServer } = await import('node:net'); const workflowFiles = new Set(); const stepFiles = new Set(); - const clients = new Set(); - let buildTriggered = false; - - // Create TCP server - const server = createServer((socket) => { - socket.setNoDelay(true); - clients.add(socket); - - if (buildTriggered && !this.isDevServer) { - socket.write(JSON.stringify({ type: 'build-complete' }) + '\n'); - } - - let buffer = ''; - - socket.on('data', (data) => { - buffer += data.toString(); - - // Process complete messages (newline-delimited JSON) - let newlineIndex = buffer.indexOf('\n'); - while (newlineIndex !== -1) { - const line = buffer.slice(0, newlineIndex); - buffer = buffer.slice(newlineIndex + 1); - newlineIndex = buffer.indexOf('\n'); - - if (line.trim()) { - try { - const message = JSON.parse(line); - - if (message.type === 'file-discovered') { - const { filePath, hasWorkflow, hasStep } = message; - - const knownFile = - workflowFiles.has(filePath) || stepFiles.has(filePath); - - if (hasWorkflow) { - workflowFiles.add(filePath); - } else { - workflowFiles.delete(filePath); - } - - if (hasStep) { - stepFiles.add(filePath); - } else { - stepFiles.delete(filePath); - } - - // Trigger debounced build if the file was previously seen - // or was steps or workflows currently - if ( - // in non-dev we always update debounce on activity - !this.isDevServer || - hasWorkflow || - hasStep || - knownFile - ) { - triggerBuild(); - } - } else if (message.type === 'trigger-build') { - // enqueue new build if one isn't already pending - triggerBuild(); - } - } catch (error) { - console.error('Failed to parse socket message:', error); - } - } - } - }); - - socket.on('end', () => { - clients.delete(socket); - }); - - socket.on('error', (err) => { - console.error('Socket error:', err); - clients.delete(socket); - }); - }); - - // Listen on random available port - await new Promise((resolve) => { - // Port 0 tells the OS to assign a random available port - server.listen(0, '127.0.0.1', () => { - const address = server.address(); - if (address && typeof address === 'object') { - // Expose the port via environment variable - process.env.WORKFLOW_SOCKET_PORT = String(address.port); - } - resolve(); - }); - }); - - // Store the server and broadcast function - this.socketIO = { - emit: (event: string) => { - if (event === 'build-complete') { - const message = JSON.stringify({ type: 'build-complete' }) + '\n'; - for (const client of clients) { - client.write(message); - } - } - }, - }; - let debounceTimer: NodeJS.Timeout | null = null; - + let buildTriggered = false; const BUILD_DEBOUNCE_MS = this.isDevServer ? 500 : 2_000; // Attempt to load cached workflows/steps from previous build @@ -376,7 +277,6 @@ export async function getNextBuilder() { } // Debounced build trigger - const triggerBuild = () => { if (debounceTimer) { clearTimeout(debounceTimer); @@ -390,6 +290,7 @@ export async function getNextBuilder() { // and they can't be refreshed/rebuilt after that in production return; } + // Combine workflow and step files into single array const allFiles = new Set([...workflowFiles, ...stepFiles]); const inputFiles = Array.from(allFiles); @@ -407,6 +308,46 @@ export async function getNextBuilder() { } }, BUILD_DEBOUNCE_MS); }; + + // Configure and create socket server + const config: SocketServerConfig = { + isDevServer: this.isDevServer || false, + onFileDiscovered: ( + filePath: string, + hasWorkflow: boolean, + hasStep: boolean + ) => { + const knownFile = + workflowFiles.has(filePath) || stepFiles.has(filePath); + + if (hasWorkflow) { + workflowFiles.add(filePath); + } else { + workflowFiles.delete(filePath); + } + + if (hasStep) { + stepFiles.add(filePath); + } else { + stepFiles.delete(filePath); + } + + // Trigger debounced build if the file was previously seen + // or has workflows/steps currently + if ( + // in non-dev we always update debounce on activity + !this.isDevServer || + hasWorkflow || + hasStep || + knownFile + ) { + triggerBuild(); + } + }, + onTriggerBuild: triggerBuild, + }; + + this.socketIO = await createSocketServer(config); } private async writeStubFiles(usersAppDir: string): Promise { @@ -424,7 +365,7 @@ export async function getNextBuilder() { const routeStubContent = "export * from './inner'"; // this needs to change on each build so can refresh workflows - const innerStubContent = 'WORKFLOW_INNER_STUB_FILE_' + Date.now(); + const innerStubContent = `WORKFLOW_INNER_STUB_FILE_${Date.now()}`; const workflowDir = join(usersAppDir, '.well-known/workflow/v1'); // Ensure directories exist diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index 7dd150ad84..e4927ad981 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -3,6 +3,11 @@ import { connect, type Socket } from 'node:net'; import { relative } from 'node:path'; import { transform } from '@swc/core'; import { useStepPattern, useWorkflowPattern } from '@workflow/builders'; +import { + parseMessage, + type SocketMessage, + serializeMessage, +} from './socket-server'; // Stub content written by builder to inner.js files const STUB_CONTENT = 'WORKFLOW_INNER_STUB_FILE'; @@ -65,15 +70,21 @@ async function notifySocketServer( throw new Error(`Invariant: missing workflow socket connection`); } - // Send single message with both workflow and step information - const message = - JSON.stringify({ - type: 'file-discovered', - filePath: filename, - hasWorkflow, - hasStep, - }) + '\n'; - socket.write(message); + const authToken = process.env.WORKFLOW_SOCKET_AUTH; + if (!authToken) { + throw new Error( + `Invariant: no socket auth token provided for workflow loader` + ); + } + + // Send authenticated message with workflow and step information + const message: SocketMessage = { + type: 'file-discovered', + filePath: filename, + hasWorkflow, + hasStep, + }; + socket.write(serializeMessage(message, authToken)); } async function waitForBuildComplete(): Promise { @@ -86,35 +97,89 @@ async function waitForBuildComplete(): Promise { } let buffer = ''; + let timeout: NodeJS.Timeout | null = null; + let settled = false; + + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + socket.off('data', onData); + socket.off('error', onError); + socket.off('end', onEnd); + socket.off('close', onClose); + }; + + const settle = (callback: () => void) => { + if (!settled) { + settled = true; + cleanup(); + callback(); + } + }; const onData = (data: Buffer) => { buffer += data.toString(); + const authToken = process.env.WORKFLOW_SOCKET_AUTH; + if (!authToken) { + settle(() => reject(new Error('No socket auth token available'))); + return; + } + let newlineIndex = buffer.indexOf('\n'); while (newlineIndex !== -1) { const line = buffer.slice(0, newlineIndex); buffer = buffer.slice(newlineIndex + 1); newlineIndex = buffer.indexOf('\n'); - if (line.trim()) { - try { - const message = JSON.parse(line); - if (message.type === 'build-complete') { - socket.off('data', onData); - resolve(); - } - } catch { - // Ignore parse errors - } + const message = parseMessage(line, authToken); + if (message && message.type === 'build-complete') { + settle(() => resolve()); } } }; - socket.on('data', onData); + const onError = (err: Error) => { + settle(() => reject(new Error(`Socket error: ${err.message}`))); + }; + + const onEnd = () => { + settle(() => + reject( + new Error( + 'Socket ended unexpectedly before build-complete message received' + ) + ) + ); + }; + + const onClose = () => { + settle(() => + reject( + new Error( + 'Socket closed unexpectedly before build-complete message received' + ) + ) + ); + }; - // Send trigger-build message - // const message = JSON.stringify({ type: 'trigger-build' }) + '\n'; - // socket.write(message); + // Set timeout to prevent indefinite hanging + timeout = setTimeout(() => { + settle(() => + reject( + new Error( + 'Timeout waiting for build-complete message (60 seconds elapsed)' + ) + ) + ); + }, 60000); // 60 second timeout + + socket.on('data', onData); + socket.on('error', onError); + socket.on('end', onEnd); + socket.on('close', onClose); }); } @@ -149,7 +214,7 @@ export default async function workflowLoader( // Read the actual generated file content const actualContent = await readFile( - filename.replace(/inner\.js/, 'route.js'), + filename.replace(/inner\.js$/, 'route.js'), 'utf-8' ); return actualContent; @@ -186,7 +251,7 @@ export default async function workflowLoader( const lowerPath = normalizedFilepath.toLowerCase(); let relativeFilename: string; - if (lowerPath.startsWith(lowerWd + '/')) { + if (lowerPath.startsWith(`${lowerWd}/`)) { // File is under working directory - manually calculate relative path relativeFilename = normalizedFilepath.substring( normalizedWorkingDir.length + 1 diff --git a/packages/next/src/socket-server.ts b/packages/next/src/socket-server.ts new file mode 100644 index 0000000000..bb6fbc92f4 --- /dev/null +++ b/packages/next/src/socket-server.ts @@ -0,0 +1,191 @@ +import { randomBytes } from 'node:crypto'; +import { createServer, type Server, type Socket } from 'node:net'; + +/** + * Magic preamble that must prefix all messages to authenticate them as workflow messages. + * This prevents accidental processing of messages from port scanners or other local processes. + */ +const MESSAGE_PREAMBLE = 'WF:'; + +/** + * Generate a random authentication token for this server session. + * Clients must include this token in all messages. + */ +function generateAuthToken(): string { + return randomBytes(16).toString('hex'); +} + +/** + * Message types that can be sent between loader and builder + */ +export type SocketMessage = + | { + type: 'file-discovered'; + filePath: string; + hasWorkflow: boolean; + hasStep: boolean; + } + | { type: 'trigger-build' } + | { type: 'build-complete' }; + +/** + * Configuration for the socket server + */ +export interface SocketServerConfig { + isDevServer: boolean; + onFileDiscovered: ( + filePath: string, + hasWorkflow: boolean, + hasStep: boolean + ) => void; + onTriggerBuild: () => void; +} + +/** + * Interface for the socket IO instance returned by createSocketServer + */ +export interface SocketIO { + emit(event: 'build-complete'): void; + getAuthToken(): string; +} + +/** + * Serialize a message with authentication preamble + */ +export function serializeMessage( + message: SocketMessage, + authToken: string +): string { + return `${MESSAGE_PREAMBLE}${authToken}:${JSON.stringify(message)}\n`; +} + +/** + * Parse and authenticate a message from the socket + * Returns the parsed message if valid, null otherwise + */ +export function parseMessage( + line: string, + authToken: string +): SocketMessage | null { + const trimmed = line.trim(); + if (!trimmed) { + return null; + } + + // Check for preamble + if (!trimmed.startsWith(MESSAGE_PREAMBLE)) { + console.warn('Received message without valid preamble, ignoring'); + return null; + } + + // Extract auth token and payload + const withoutPreamble = trimmed.slice(MESSAGE_PREAMBLE.length); + const colonIndex = withoutPreamble.indexOf(':'); + if (colonIndex === -1) { + console.warn('Received message without auth token separator, ignoring'); + return null; + } + + const messageToken = withoutPreamble.slice(0, colonIndex); + const payload = withoutPreamble.slice(colonIndex + 1); + + // Verify auth token + if (messageToken !== authToken) { + console.warn('Received message with invalid auth token, ignoring'); + return null; + } + + // Parse JSON payload + try { + return JSON.parse(payload) as SocketMessage; + } catch (error) { + console.error('Failed to parse socket message JSON:', error); + return null; + } +} + +/** + * Create a TCP socket server for loader<->builder communication. + * Returns a SocketIO interface for broadcasting messages and the auth token. + * + * SECURITY: Server listens on 127.0.0.1 (localhost only) and uses + * message authentication to prevent processing of unauthorized messages. + */ +export async function createSocketServer( + config: SocketServerConfig +): Promise { + const authToken = generateAuthToken(); + const clients = new Set(); + let buildTriggered = false; + + const server: Server = createServer((socket: Socket) => { + socket.setNoDelay(true); + clients.add(socket); + + // Send build-complete if build already finished (production mode) + if (buildTriggered && !config.isDevServer) { + socket.write(serializeMessage({ type: 'build-complete' }, authToken)); + } + + let buffer = ''; + + socket.on('data', (data: Buffer) => { + buffer += data.toString(); + + // Process complete messages (newline-delimited) + let newlineIndex = buffer.indexOf('\n'); + while (newlineIndex !== -1) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + newlineIndex = buffer.indexOf('\n'); + + const message = parseMessage(line, authToken); + if (!message) { + continue; + } + + if (message.type === 'file-discovered') { + config.onFileDiscovered( + message.filePath, + message.hasWorkflow, + message.hasStep + ); + } else if (message.type === 'trigger-build') { + config.onTriggerBuild(); + } + } + }); + + socket.on('end', () => { + clients.delete(socket); + }); + + socket.on('error', (err: Error) => { + console.error('Socket error:', err); + clients.delete(socket); + }); + }); + + // Listen on random available port (localhost only) + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (address && typeof address === 'object') { + process.env.WORKFLOW_SOCKET_PORT = String(address.port); + process.env.WORKFLOW_SOCKET_AUTH = authToken; + } + resolve(); + }); + }); + + return { + emit: (_event: 'build-complete') => { + buildTriggered = true; + const message = serializeMessage({ type: 'build-complete' }, authToken); + for (const client of clients) { + client.write(message); + } + }, + getAuthToken: () => authToken, + }; +} diff --git a/packages/workflow/src/internal/builtins.ts b/packages/workflow/src/internal/builtins.ts index bd279a8e40..06248c28b8 100644 --- a/packages/workflow/src/internal/builtins.ts +++ b/packages/workflow/src/internal/builtins.ts @@ -3,6 +3,8 @@ * similar to "stdlib" except that are not meant to be imported by users, but are instead "just available" * alongside user defined steps. They are used internally by the runtime */ +// Re-export stdlib for discovery - needed because lazy discovery doesn't pick it up via eager scanning. +// The workflow builder injects this as a builtin module available in the workflow runtime sandbox. export * from '../stdlib.js'; export async function __builtin_response_array_buffer(res: Response) { diff --git a/packages/workflow/tsconfig.json b/packages/workflow/tsconfig.json index 65f2136e46..0f4008973e 100644 --- a/packages/workflow/tsconfig.json +++ b/packages/workflow/tsconfig.json @@ -3,6 +3,7 @@ "compilerOptions": { "outDir": "dist", "sourceMap": false, + // See https://github.com/vercel/workflow/pull/352 "inlineSourceMap": true }, "include": ["src"], diff --git a/workbench/nextjs-turbopack/next.config.ts b/workbench/nextjs-turbopack/next.config.ts index 6206788429..493481c8cc 100644 --- a/workbench/nextjs-turbopack/next.config.ts +++ b/workbench/nextjs-turbopack/next.config.ts @@ -5,6 +5,7 @@ const nextConfig: NextConfig = { /* config options here */ serverExternalPackages: ['@node-rs/xxhash'], + // For debugging/testing: Makes Turbopack module IDs human-readable instead of hashed experimental: { turbopackModuleIds: 'named' }, }; From 60eaeb37b0de0380a5202c7cb2ff3521f7739983 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 22 Dec 2025 22:35:23 -0600 Subject: [PATCH 27/27] cleanup and fix warning --- packages/next/src/builder.ts | 12 +++++------- packages/next/src/index.ts | 5 +---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index 6ef82131df..258c73da64 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -30,11 +30,6 @@ export async function getNextBuilder() { private isDevServer?: boolean; private nextConfig?: NextConfig; - setNextConfig(config: NextConfig, phase: string) { - this.nextConfig = config; - this.isDevServer = phase === 'phase-development-server'; - } - private getDistDir(): string { return this.nextConfig?.distDir || '.next'; } @@ -82,7 +77,10 @@ export async function getNextBuilder() { } } - async init() { + async init(nextConfig: NextConfig, phase: string) { + this.nextConfig = nextConfig; + this.isDevServer = phase === 'phase-development-server'; + const outputDir = await this.findAppDirectory(); // Write stub files @@ -357,7 +355,7 @@ export async function getNextBuilder() { // we're fine with > 3 vCPU but <= 3 vCPUs and we won't be able to // discover workflows/steps const parallelismCount = os.availableParallelism(); - if (parallelismCount < 4) { + if (process.env.TURBOPACK && parallelismCount < 4) { console.warn( `Available parallelism of ${parallelismCount} is less than needed 4. This can cause workflows/steps to fail to discover properly in turbopack` ); diff --git a/packages/next/src/index.ts b/packages/next/src/index.ts index c18ccaf911..a9fa593452 100644 --- a/packages/next/src/index.ts +++ b/packages/next/src/index.ts @@ -120,10 +120,7 @@ export function withWorkflow( externalPackages: [...(nextConfig.serverExternalPackages || [])], }); - // Pass the nextConfig to the builder so it can access distDir - workflowBuilder.setNextConfig(nextConfig, phase); - - await workflowBuilder.init(); + await workflowBuilder.init(nextConfig, phase); process.env.WORKFLOW_NEXT_PRIVATE_BUILT = '1'; }