diff --git a/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts b/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts new file mode 100644 index 0000000000..c430dbbde0 --- /dev/null +++ b/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts @@ -0,0 +1,122 @@ +import path from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, PlatformError, Scope } from "effect"; +import { describe, expect } from "vitest"; + +import { checkpointRefForThreadTurn } from "../Utils.ts"; +import { CheckpointStoreLive } from "./CheckpointStore.ts"; +import { CheckpointStore } from "../Services/CheckpointStore.ts"; +import { GitCoreLive } from "../../git/Layers/GitCore.ts"; +import { GitCore } from "../../git/Services/GitCore.ts"; +import { GitCommandError } from "../../git/Errors.ts"; +import { ServerConfig } from "../../config.ts"; +import { ThreadId } from "@t3tools/contracts"; + +const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), { + prefix: "t3-checkpoint-store-test-", +}); +const GitCoreTestLayer = GitCoreLive.pipe( + Layer.provide(ServerConfigLayer), + Layer.provide(NodeServices.layer), +); +const CheckpointStoreTestLayer = CheckpointStoreLive.pipe( + Layer.provide(GitCoreTestLayer), + Layer.provide(NodeServices.layer), +); +const TestLayer = Layer.mergeAll(NodeServices.layer, GitCoreTestLayer, CheckpointStoreTestLayer); + +function makeTmpDir( + prefix = "checkpoint-store-test-", +): Effect.Effect { + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.makeTempDirectoryScoped({ prefix }); + }); +} + +function writeTextFile( + filePath: string, + contents: string, +): Effect.Effect { + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + yield* fileSystem.writeFileString(filePath, contents); + }); +} + +function git( + cwd: string, + args: ReadonlyArray, +): Effect.Effect { + return Effect.gen(function* () { + const gitCore = yield* GitCore; + const result = yield* gitCore.execute({ + operation: "CheckpointStore.test.git", + cwd, + args, + timeoutMs: 10_000, + }); + return result.stdout.trim(); + }); +} + +function initRepoWithCommit( + cwd: string, +): Effect.Effect< + void, + GitCommandError | PlatformError.PlatformError, + GitCore | FileSystem.FileSystem +> { + return Effect.gen(function* () { + const core = yield* GitCore; + yield* core.initRepo({ cwd }); + yield* git(cwd, ["config", "user.email", "test@test.com"]); + yield* git(cwd, ["config", "user.name", "Test"]); + yield* writeTextFile(path.join(cwd, "README.md"), "# test\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "initial commit"]); + }); +} + +function buildLargeText(lineCount = 20_000): string { + return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`) + .join("\n") + .concat("\n"); +} + +it.layer(TestLayer)("CheckpointStoreLive", (it) => { + describe("diffCheckpoints", () => { + it.effect("returns full oversized checkpoint diffs without truncation", () => + Effect.gen(function* () { + const tmp = yield* makeTmpDir(); + yield* initRepoWithCommit(tmp); + const checkpointStore = yield* CheckpointStore; + const threadId = ThreadId.makeUnsafe("thread-checkpoint-store"); + const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0); + const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1); + + yield* checkpointStore.captureCheckpoint({ + cwd: tmp, + checkpointRef: fromCheckpointRef, + }); + yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText()); + yield* checkpointStore.captureCheckpoint({ + cwd: tmp, + checkpointRef: toCheckpointRef, + }); + + const diff = yield* checkpointStore.diffCheckpoints({ + cwd: tmp, + fromCheckpointRef, + toCheckpointRef, + }); + + expect(diff).toContain("diff --git"); + expect(diff).not.toContain("[truncated]"); + expect(diff).toContain("+line 19999"); + }), + ); + }); +}); diff --git a/apps/server/src/git/Layers/GitCore.test.ts b/apps/server/src/git/Layers/GitCore.test.ts index dc97b93649..547a69e7e1 100644 --- a/apps/server/src/git/Layers/GitCore.test.ts +++ b/apps/server/src/git/Layers/GitCore.test.ts @@ -132,6 +132,12 @@ function commitWithDate( }); } +function buildLargeText(lineCount = 20_000): string { + return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`) + .join("\n") + .concat("\n"); +} + // ── Tests ── it.layer(TestLayer)("git integration", (it) => { @@ -1670,6 +1676,40 @@ it.layer(TestLayer)("git integration", (it) => { }), ); + it.effect("prepareCommitContext truncates oversized staged patches instead of failing", () => + Effect.gen(function* () { + const tmp = yield* makeTmpDir(); + yield* initRepoWithCommit(tmp); + const core = yield* GitCore; + + yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText()); + + const context = yield* core.prepareCommitContext(tmp); + expect(context).not.toBeNull(); + expect(context!.stagedSummary).toContain("README.md"); + expect(context!.stagedPatch).toContain("[truncated]"); + }), + ); + + it.effect("readRangeContext truncates oversized diff patches instead of failing", () => + Effect.gen(function* () { + const tmp = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(tmp); + const core = yield* GitCore; + + yield* core.createBranch({ cwd: tmp, branch: "feature/large-range-context" }); + yield* core.checkoutBranch({ cwd: tmp, branch: "feature/large-range-context" }); + yield* writeTextFile(path.join(tmp, "large.txt"), buildLargeText()); + yield* git(tmp, ["add", "large.txt"]); + yield* git(tmp, ["commit", "-m", "Add large range context"]); + + const rangeContext = yield* core.readRangeContext(tmp, initialBranch); + expect(rangeContext.commitSummary).toContain("Add large range context"); + expect(rangeContext.diffSummary).toContain("large.txt"); + expect(rangeContext.diffPatch).toContain("[truncated]"); + }), + ); + it.effect("pushes with upstream setup and then skips when up to date", () => Effect.gen(function* () { const tmp = yield* makeTmpDir(); diff --git a/apps/server/src/git/Layers/GitCore.ts b/apps/server/src/git/Layers/GitCore.ts index 8bb5844228..fcb2f9c58e 100644 --- a/apps/server/src/git/Layers/GitCore.ts +++ b/apps/server/src/git/Layers/GitCore.ts @@ -32,6 +32,11 @@ import { decodeJsonResult } from "@t3tools/shared/schemaJson"; const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000; +const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]"; +const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000; +const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000; +const RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES = 19_000; +const RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES = 59_000; const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15); const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5); const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048; @@ -53,6 +58,8 @@ interface ExecuteGitOptions { timeoutMs?: number | undefined; allowNonZeroExit?: boolean | undefined; fallbackErrorMessage?: string | undefined; + maxOutputBytes?: number | undefined; + truncateOutputAtMaxBytes?: boolean | undefined; progress?: ExecuteGitProgress | undefined; } @@ -439,12 +446,14 @@ const collectOutput = Effect.fn(function* ( input: Pick, stream: Stream.Stream, maxOutputBytes: number, + truncateOutputAtMaxBytes: boolean, onLine: ((line: string) => Effect.Effect) | undefined, ): Effect.fn.Return { const decoder = new TextDecoder(); let bytes = 0; let text = ""; let lineBuffer = ""; + let truncated = false; const emitCompleteLines = (flush: boolean) => Effect.gen(function* () { @@ -469,8 +478,11 @@ const collectOutput = Effect.fn(function* ( yield* Stream.runForEach(stream, (chunk) => Effect.gen(function* () { - bytes += chunk.byteLength; - if (bytes > maxOutputBytes) { + if (truncateOutputAtMaxBytes && truncated) { + return; + } + const nextBytes = bytes + chunk.byteLength; + if (!truncateOutputAtMaxBytes && nextBytes > maxOutputBytes) { return yield* new GitCommandError({ operation: input.operation, command: quoteGitCommand(input.args), @@ -478,18 +490,26 @@ const collectOutput = Effect.fn(function* ( detail: `${quoteGitCommand(input.args)} output exceeded ${maxOutputBytes} bytes and was truncated.`, }); } - const decoded = decoder.decode(chunk, { stream: true }); + + const chunkToDecode = + truncateOutputAtMaxBytes && nextBytes > maxOutputBytes + ? chunk.subarray(0, Math.max(0, maxOutputBytes - bytes)) + : chunk; + bytes += chunkToDecode.byteLength; + truncated = truncateOutputAtMaxBytes && nextBytes > maxOutputBytes; + + const decoded = decoder.decode(chunkToDecode, { stream: !truncated }); text += decoded; lineBuffer += decoded; yield* emitCompleteLines(false); }), ).pipe(Effect.mapError(toGitCommandError(input, "output stream failed."))); - const remainder = decoder.decode(); + const remainder = truncated ? "" : decoder.decode(); text += remainder; lineBuffer += remainder; yield* emitCompleteLines(true); - return text; + return truncated ? `${text}${OUTPUT_TRUNCATED_MARKER}` : text; }); export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"] }) => @@ -511,6 +531,7 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute" } as const; const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; + const truncateOutputAtMaxBytes = input.truncateOutputAtMaxBytes ?? false; const commandEffect = Effect.gen(function* () { const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe( @@ -537,12 +558,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute" commandInput, child.stdout, maxOutputBytes, + truncateOutputAtMaxBytes, input.progress?.onStdoutLine, ), collectOutput( commandInput, child.stderr, maxOutputBytes, + truncateOutputAtMaxBytes, input.progress?.onStderrLine, ), child.exitCode.pipe( @@ -603,6 +626,10 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute" args, allowNonZeroExit: true, ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}), + ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}), + ...(options.truncateOutputAtMaxBytes !== undefined + ? { truncateOutputAtMaxBytes: options.truncateOutputAtMaxBytes } + : {}), ...(options.progress ? { progress: options.progress } : {}), }).pipe( Effect.flatMap((result) => { @@ -647,6 +674,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute" Effect.map((result) => result.stdout), ); + const runGitStdoutWithOptions = ( + operation: string, + cwd: string, + args: readonly string[], + options: ExecuteGitOptions = {}, + ): Effect.Effect => + executeGit(operation, cwd, args, options).pipe(Effect.map((result) => result.stdout)); + const branchExists = (cwd: string, branch: string): Effect.Effect => executeGit( "GitCore.branchExists", @@ -1162,12 +1197,15 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute" return null; } - const stagedPatch = yield* runGitStdout("GitCore.prepareCommitContext.stagedPatch", cwd, [ - "diff", - "--cached", - "--patch", - "--minimal", - ]); + const stagedPatch = yield* runGitStdoutWithOptions( + "GitCore.prepareCommitContext.stagedPatch", + cwd, + ["diff", "--cached", "--patch", "--minimal"], + { + maxOutputBytes: PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES, + truncateOutputAtMaxBytes: true, + }, + ); return { stagedSummary, @@ -1363,14 +1401,33 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute" const range = `${baseBranch}..HEAD`; const [commitSummary, diffSummary, diffPatch] = yield* Effect.all( [ - runGitStdout("GitCore.readRangeContext.log", cwd, ["log", "--oneline", range]), - runGitStdout("GitCore.readRangeContext.diffStat", cwd, ["diff", "--stat", range]), - runGitStdout("GitCore.readRangeContext.diffPatch", cwd, [ - "diff", - "--patch", - "--minimal", - range, - ]), + runGitStdoutWithOptions( + "GitCore.readRangeContext.log", + cwd, + ["log", "--oneline", range], + { + maxOutputBytes: RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES, + truncateOutputAtMaxBytes: true, + }, + ), + runGitStdoutWithOptions( + "GitCore.readRangeContext.diffStat", + cwd, + ["diff", "--stat", range], + { + maxOutputBytes: RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES, + truncateOutputAtMaxBytes: true, + }, + ), + runGitStdoutWithOptions( + "GitCore.readRangeContext.diffPatch", + cwd, + ["diff", "--patch", "--minimal", range], + { + maxOutputBytes: RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES, + truncateOutputAtMaxBytes: true, + }, + ), ], { concurrency: "unbounded" }, ); diff --git a/apps/server/src/git/Services/GitCore.ts b/apps/server/src/git/Services/GitCore.ts index b74c526897..f1a4e065cd 100644 --- a/apps/server/src/git/Services/GitCore.ts +++ b/apps/server/src/git/Services/GitCore.ts @@ -32,6 +32,7 @@ export interface ExecuteGitInput { readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; readonly maxOutputBytes?: number; + readonly truncateOutputAtMaxBytes?: boolean; readonly progress?: ExecuteGitProgress; }