From b513bae343b649cab7dfcdf8b413465791f8e67b Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 19 May 2026 15:01:22 -0500 Subject: [PATCH 01/19] chore(attachments): start work --- cdk/cdk.json | 3 + cdk/src/constructs/attachments-bucket.ts | 88 + cdk/src/constructs/task-api.ts | 17 + cdk/src/constructs/task-orchestrator.ts | 13 + cdk/src/handlers/shared/context-hydration.ts | 32 + cdk/src/handlers/shared/create-task-core.ts | 186 +- cdk/src/handlers/shared/orchestrator.ts | 13 +- cdk/src/handlers/shared/response.ts | 8 + cdk/src/handlers/shared/types.ts | 148 +- cdk/src/handlers/shared/validation.ts | 323 +++- cdk/src/stacks/agent.ts | 15 +- cdk/test/handlers/orchestrate-task.test.ts | 24 +- .../handlers/shared/create-task-core.test.ts | 4 +- cdk/test/handlers/shared/validation.test.ts | 304 +++ cli/src/types.ts | 34 + docs/design/ATTACHMENTS.md | 1691 +++++++++++++++++ 16 files changed, 2884 insertions(+), 19 deletions(-) create mode 100644 cdk/src/constructs/attachments-bucket.ts create mode 100644 docs/design/ATTACHMENTS.md diff --git a/cdk/cdk.json b/cdk/cdk.json index 6cf84b04..0008419f 100644 --- a/cdk/cdk.json +++ b/cdk/cdk.json @@ -1,6 +1,9 @@ { "app": "npx ts-node -P tsconfig.json --prefer-ts-exts src/main.ts", "output": "cdk.out", + "context": { + "blueprintRepo": "krokoko/agent-plugins" + }, "build": "node -e \"process.exit(0)\"", "watch": { "include": ["src/**/*.ts", "test/**/*.ts"], diff --git a/cdk/src/constructs/attachments-bucket.ts b/cdk/src/constructs/attachments-bucket.ts new file mode 100644 index 00000000..f44b79a0 --- /dev/null +++ b/cdk/src/constructs/attachments-bucket.ts @@ -0,0 +1,88 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Duration, RemovalPolicy } from 'aws-cdk-lib'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import { Construct } from 'constructs'; + +/** Lifecycle expiry for task attachments — matches task record retention. */ +export const ATTACHMENT_TTL_DAYS = 90; + +/** S3 key prefix for all attachments. Layout: attachments//// */ +export const ATTACHMENT_OBJECT_KEY_PREFIX = 'attachments/'; + +/** + * Properties for AttachmentsBucket construct. + */ +export interface AttachmentsBucketProps { + /** + * Removal policy for the bucket. + * @default RemovalPolicy.DESTROY + */ + readonly removalPolicy?: RemovalPolicy; + + /** + * Whether to auto-delete objects when the bucket is removed. + * @default true + */ + readonly autoDeleteObjects?: boolean; +} + +/** + * S3 bucket for task attachment storage. + * + * Attachments (images, files, URL-fetched content) are uploaded during task + * creation (inline or presigned), screened for security, and delivered to + * the agent runtime during execution. + * + * Security: + * - ``blockPublicAccess: BLOCK_ALL`` + ``enforceSSL: true`` + * - ``encryption: S3_MANAGED`` — server-side encryption at rest. + * - Versioning enabled — pins object versions at screening time to prevent + * TOCTOU attacks (client uploads benign content, replaces with malicious + * content before agent downloads). + * - 90-day lifecycle expiry (matches task record TTL). + * - 7-day noncurrent version expiration (covers longest-running tasks). + */ +export class AttachmentsBucket extends Construct { + /** The underlying S3 bucket. */ + public readonly bucket: s3.Bucket; + + constructor(scope: Construct, id: string, props: AttachmentsBucketProps = {}) { + super(scope, id); + + this.bucket = new s3.Bucket(this, 'Bucket', { + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + encryption: s3.BucketEncryption.S3_MANAGED, + enforceSSL: true, + versioned: true, + lifecycleRules: [ + { + id: 'attachments-ttl', + enabled: true, + expiration: Duration.days(ATTACHMENT_TTL_DAYS), + noncurrentVersionExpiration: Duration.days(7), + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, + autoDeleteObjects: props.autoDeleteObjects ?? true, + }); + } +} diff --git a/cdk/src/constructs/task-api.ts b/cdk/src/constructs/task-api.ts index 2443bbe6..63d5e334 100644 --- a/cdk/src/constructs/task-api.ts +++ b/cdk/src/constructs/task-api.ts @@ -127,6 +127,12 @@ export interface TaskApiProps { * When provided, the cancel Lambda gets `ECS_CLUSTER_ARN` env var and `ecs:StopTask` permission. */ readonly ecsClusterArn?: string; + + /** + * S3 bucket for task attachments. When provided, the create-task Lambda + * gets PutObject/DeleteObject grants and the bucket name as env var. + */ + readonly attachmentsBucket?: s3.IBucket; } /** @@ -363,6 +369,9 @@ export class TaskApi extends Construct { createTaskEnv.GUARDRAIL_ID = props.guardrailId; createTaskEnv.GUARDRAIL_VERSION = props.guardrailVersion; } + if (props.attachmentsBucket) { + createTaskEnv.ATTACHMENTS_BUCKET_NAME = props.attachmentsBucket.bucketName; + } const createTaskFn = new lambda.NodejsFunction(this, 'CreateTaskFn', { entry: path.join(handlersDir, 'create-task.ts'), @@ -371,6 +380,8 @@ export class TaskApi extends Construct { architecture: Architecture.ARM_64, environment: createTaskEnv, bundling: commonBundling, + memorySize: 256, + timeout: Duration.seconds(15), }); const getTaskFn = new lambda.NodejsFunction(this, 'GetTaskFn', { @@ -483,6 +494,12 @@ export class TaskApi extends Construct { })); } + // Grant create-task Lambda put/delete on attachments bucket for inline upload + cleanup + if (props.attachmentsBucket) { + props.attachmentsBucket.grantPut(createTaskFn); + props.attachmentsBucket.grantDelete(createTaskFn); + } + // Collect all Lambda functions for cdk-nag suppressions const allFunctions: lambda.NodejsFunction[] = [createTaskFn, getTaskFn, listTasksFn, cancelTaskFn, getTaskEventsFn]; diff --git a/cdk/src/constructs/task-orchestrator.ts b/cdk/src/constructs/task-orchestrator.ts index ceb82733..b5f4d744 100644 --- a/cdk/src/constructs/task-orchestrator.ts +++ b/cdk/src/constructs/task-orchestrator.ts @@ -24,6 +24,7 @@ import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as iam from 'aws-cdk-lib/aws-iam'; import { Runtime, Architecture } from 'aws-cdk-lib/aws-lambda'; import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; import { NagSuppressions } from 'cdk-nag'; import { Construct } from 'constructs'; @@ -135,6 +136,12 @@ export interface TaskOrchestratorProps { readonly taskRoleArn: string; readonly executionRoleArn: string; }; + + /** + * S3 bucket for task attachments. When provided, the orchestrator gets + * ReadWrite grants for URL fetch/screen/upload during hydration. + */ + readonly attachmentsBucket?: s3.IBucket; } /** @@ -203,6 +210,7 @@ export class TaskOrchestrator extends Construct { ECS_SECURITY_GROUP: props.ecsConfig.securityGroup, ECS_CONTAINER_NAME: props.ecsConfig.containerName, }), + ...(props.attachmentsBucket && { ATTACHMENTS_BUCKET_NAME: props.attachmentsBucket.bucketName }), }, bundling: { // Bundle `@aws-sdk/client-bedrock-agentcore` — newer commands (e.g. @@ -229,6 +237,11 @@ export class TaskOrchestrator extends Construct { props.repoTable.grantReadData(this.fn); } + // Attachments bucket grants (URL fetch/screen/upload during hydration) + if (props.attachmentsBucket) { + props.attachmentsBucket.grantReadWrite(this.fn); + } + // Durable execution managed policy this.fn.role!.addManagedPolicy( iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicDurableExecutionRolePolicy'), diff --git a/cdk/src/handlers/shared/context-hydration.ts b/cdk/src/handlers/shared/context-hydration.ts index 021c2c00..1e93679b 100644 --- a/cdk/src/handlers/shared/context-hydration.ts +++ b/cdk/src/handlers/shared/context-hydration.ts @@ -152,6 +152,34 @@ export class GuardrailScreeningError extends Error { } } +/** + * Base class for all attachment-related errors. A single `instanceof AttachmentError` + * check in the hydration catch block covers all current and future attachment error + * types, avoiding a growing allowlist. + */ +export class AttachmentError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'AttachmentError'; + } +} + +/** Attachment could not be resolved (fetch failed, screening blocked, S3 missing). */ +export class AttachmentResolutionError extends AttachmentError { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'AttachmentResolutionError'; + } +} + +/** Image attachments exceed the token budget available for text context. */ +export class AttachmentBudgetExceededError extends AttachmentError { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'AttachmentBudgetExceededError'; + } +} + /** Mapping from policy response keys to assessment detail extraction rules. */ const POLICY_EXTRACTORS: ReadonlyArray<{ readonly policyKey: string; @@ -1159,6 +1187,10 @@ export async function hydrateContext(task: TaskRecord, options?: HydrateContextO if (err instanceof GuardrailScreeningError) { throw err; } + // Attachment errors must propagate — user explicitly provided attachments; proceeding without them is wrong + if (err instanceof AttachmentError) { + throw err; + } // Programming errors (bugs) should fail the task, not silently degrade context if (err instanceof TypeError || err instanceof RangeError || err instanceof ReferenceError) { logger.error('Programming error during context hydration — failing task', { diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index f8d5c69d..cfa94a7d 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -24,15 +24,18 @@ import { BedrockRuntimeClient, ApplyGuardrailCommand } from '@aws-sdk/client-bedrock-runtime'; import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; +import { PutObjectCommand, DeleteObjectsCommand, S3Client } from '@aws-sdk/client-s3'; import { DynamoDBDocumentClient, PutCommand, QueryCommand, GetCommand } from '@aws-sdk/lib-dynamodb'; import type { APIGatewayProxyResult } from 'aws-lambda'; +import { createHash } from 'crypto'; import { ulid } from 'ulid'; import { generateBranchName } from './gateway'; import { logger } from './logger'; import { checkRepoOnboarded } from './repo-config'; import { ErrorCode, errorResponse, successResponse } from './response'; -import { type ChannelSource, type CreateTaskRequest, isPrTaskType, type TaskRecord, type TaskType, toTaskDetail } from './types'; -import { computeTtlEpoch, DEFAULT_MAX_TURNS, hasTaskSpec, isValidIdempotencyKey, isValidRepo, isValidTaskDescriptionLength, isValidTaskType, MAX_TASK_DESCRIPTION_LENGTH, validateMaxBudgetUsd, validateMaxTurns, validatePrNumber } from './validation'; +import { type AttachmentRecord, type ChannelSource, type CreateTaskRequest, createAttachmentRecord, type InlineAttachment, isPrTaskType, type TaskRecord, type TaskType, toTaskDetail } from './types'; +import { computeTtlEpoch, DEFAULT_MAX_TURNS, hasTaskSpec, isValidIdempotencyKey, isValidRepo, isValidTaskDescriptionLength, isValidTaskType, MAX_TASK_DESCRIPTION_LENGTH, validateAttachments, validateMaxBudgetUsd, validateMaxTurns, validatePrNumber } from './validation'; +import { ATTACHMENT_OBJECT_KEY_PREFIX } from '../../constructs/attachments-bucket'; import { TaskStatus } from '../../constructs/task-status'; /** @@ -57,6 +60,8 @@ if (process.env.GUARDRAIL_ID && !process.env.GUARDRAIL_VERSION) { const TABLE_NAME = process.env.TASK_TABLE_NAME!; const EVENTS_TABLE_NAME = process.env.TASK_EVENTS_TABLE_NAME!; const TASK_RETENTION_DAYS = Number(process.env.TASK_RETENTION_DAYS ?? '90'); +const ATTACHMENTS_BUCKET = process.env.ATTACHMENTS_BUCKET_NAME; +const s3Client = ATTACHMENTS_BUCKET ? new S3Client({}) : undefined; /** * Core task creation logic shared by the Cognito create-task handler @@ -132,6 +137,24 @@ export async function createTaskCore( } const userTrace = body.trace === true; + // Validate attachments + const attachmentResult = validateAttachments(body.attachments as unknown[] | undefined); + if (!attachmentResult.valid) { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, attachmentResult.error, requestId); + } + const validatedAttachments = attachmentResult.parsed; + + // Fail-closed: reject requests with attachments when bucket is not configured + if (validatedAttachments.length > 0 && (!s3Client || !ATTACHMENTS_BUCKET)) { + logger.error('Attachments submitted but ATTACHMENTS_BUCKET_NAME is not configured', { + user_id: context.userId, + request_id: requestId, + attachment_count: validatedAttachments.length, + }); + return errorResponse(503, ErrorCode.INTERNAL_ERROR, + 'Attachment storage is not configured. Please contact your administrator.', requestId); + } + // 2. Screen task description with Bedrock Guardrail (fail-closed: unscreened content // must not reach the agent — a Bedrock outage blocks task submissions) if (bedrockClient && body.task_description) { @@ -158,6 +181,118 @@ export async function createTaskCore( } } + // Generate task ID early so attachment S3 keys use the correct task ID + const taskId = ulid(); + + // 2b. Process inline attachments: screen, upload to S3, build records + const attachmentRecords: AttachmentRecord[] = []; + const uploadedS3Keys: string[] = []; + if (validatedAttachments.length > 0 && s3Client && ATTACHMENTS_BUCKET) { + for (const att of validatedAttachments) { + if (att.delivery !== 'inline') continue; + const inlineAtt = att as InlineAttachment; + + // Validate base64 encoding before decode + if (!isValidBase64(inlineAtt.data)) { + return errorResponse(400, ErrorCode.ATTACHMENT_INVALID_CONTENT, + `Attachment '${inlineAtt.filename}' has invalid base64 encoding.`, requestId); + } + + const decoded = Buffer.from(inlineAtt.data, 'base64'); + const attachmentId = ulid(); + + // Screen inline attachment content via Bedrock Guardrail (fail-closed) + if (!bedrockClient) { + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); + logger.error('Inline attachment submitted but guardrail is not configured (fail-closed)', { + request_id: requestId, + attachment_filename: inlineAtt.filename, + }); + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is not configured. Please contact your administrator.', requestId); + } + { + try { + const isImage = inlineAtt.type === 'image'; + const guardrailContent = isImage + ? [{ image: { format: mimeToGuardrailFormat(inlineAtt.content_type), source: { bytes: decoded } } }] + : [{ text: { text: decoded.toString('utf-8') } }]; + + const screenResult = await bedrockClient.send(new ApplyGuardrailCommand({ + guardrailIdentifier: process.env.GUARDRAIL_ID!, + guardrailVersion: process.env.GUARDRAIL_VERSION!, + source: 'INPUT', + content: guardrailContent, + })); + + if (screenResult.action === 'GUARDRAIL_INTERVENED') { + // Clean up any already-uploaded attachments + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); + return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, + `Attachment '${inlineAtt.filename}' was blocked by content policy.`, requestId); + } + } catch (screenErr) { + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); + logger.error('Attachment screening failed (fail-closed)', { + error: String(screenErr), + attachment_filename: inlineAtt.filename, + request_id: requestId, + }); + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is temporarily unavailable. Please try again later.', requestId); + } + } + + // Upload to S3 + const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${context.userId}/${taskId}/${attachmentId}/${inlineAtt.filename}`; + try { + const putResult = await s3Client.send(new PutObjectCommand({ + Bucket: ATTACHMENTS_BUCKET, + Key: s3Key, + Body: decoded, + ContentType: inlineAtt.content_type, + })); + + uploadedS3Keys.push(s3Key); + const checksum = createHash('sha256').update(decoded).digest('hex'); + + attachmentRecords.push(createAttachmentRecord({ + attachment_id: attachmentId, + type: inlineAtt.type, + content_type: inlineAtt.content_type, + filename: inlineAtt.filename, + s3_key: s3Key, + s3_version_id: putResult.VersionId ?? 'unversioned', + size_bytes: decoded.length, + screening: { status: 'passed', screened_at: new Date().toISOString() }, + checksum_sha256: checksum, + })); + } catch (s3Err) { + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); + logger.error('S3 upload failed for inline attachment', { + error: String(s3Err), + attachment_filename: inlineAtt.filename, + request_id: requestId, + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, + `Failed to upload attachment '${inlineAtt.filename}'.`, requestId); + } + } + + // URL attachments get pending records (resolved during hydration) + for (const att of validatedAttachments) { + if (att.delivery !== 'url_fetch') continue; + attachmentRecords.push(createAttachmentRecord({ + attachment_id: ulid(), + type: att.type, + content_type: att.content_type, + filename: att.filename, + screening: { status: 'pending' }, + source_url: att.url, + })); + } + } + // 3. Check idempotency key if (context.idempotencyKey !== undefined && context.idempotencyKey !== null) { if (!isValidIdempotencyKey(context.idempotencyKey)) { @@ -213,7 +348,6 @@ export async function createTaskCore( } // 4. Generate identifiers and timestamps - const taskId = ulid(); const now = new Date().toISOString(); const branchName = isPrTask ? 'pending:pr_resolution' @@ -236,6 +370,7 @@ export async function createTaskCore( ...(context.idempotencyKey && { idempotency_key: context.idempotencyKey }), channel_source: context.channelSource, channel_metadata: context.channelMetadata, + ...(attachmentRecords.length > 0 && { attachments: attachmentRecords }), status_created_at: `${TaskStatus.SUBMITTED}#${now}`, created_at: now, updated_at: now, @@ -307,3 +442,48 @@ export async function createTaskCore( // 9. Return created task return successResponse(201, toTaskDetail(taskRecord), requestId); } + +/** + * Map MIME type to Bedrock GuardrailImageFormat. + * The SDK currently supports 'png' | 'jpeg' — GIF and WebP are mapped + * to 'png' (lossless container) since the guardrail inspects visual + * content, not codec fidelity. + */ +function mimeToGuardrailFormat(contentType: string): 'png' | 'jpeg' { + if (contentType === 'image/jpeg') return 'jpeg'; + return 'png'; +} + +const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/; + +/** Validate that a string is well-formed base64. */ +function isValidBase64(data: string): boolean { + if (data.length === 0) return false; + if (data.length % 4 !== 0) return false; + return BASE64_PATTERN.test(data); +} + +/** + * Clean up S3 objects from a partially-failed inline upload. + * Best-effort — the 90-day lifecycle is the safety net if cleanup fails. + */ +async function cleanupOrphanedAttachments(client: S3Client, keys: string[]): Promise { + if (keys.length === 0 || !ATTACHMENTS_BUCKET) return; + try { + const result = await client.send(new DeleteObjectsCommand({ + Bucket: ATTACHMENTS_BUCKET, + Delete: { Objects: keys.map(Key => ({ Key })) }, + })); + if (result.Errors && result.Errors.length > 0) { + logger.error('Partial cleanup failure — some orphaned objects remain', { + failedKeys: result.Errors.map(e => e.Key), + errorCodes: result.Errors.map(e => e.Code), + }); + } + } catch (err) { + logger.error('Cleanup failed entirely — all objects orphaned (90-day lifecycle is safety net)', { + keys, + error: String(err), + }); + } +} diff --git a/cdk/src/handlers/shared/orchestrator.ts b/cdk/src/handlers/shared/orchestrator.ts index f3c21467..bee2a25f 100644 --- a/cdk/src/handlers/shared/orchestrator.ts +++ b/cdk/src/handlers/shared/orchestrator.ts @@ -624,17 +624,24 @@ export async function failTask( userId: string, releaseConcurrency: boolean, ): Promise { + let transitioned = false; try { await transitionTask(taskId, fromStatus, TaskStatus.FAILED, { completed_at: new Date().toISOString(), error_message: errorMessage, }); + transitioned = true; } catch (err) { logger.warn('Failed to transition task to FAILED', { task_id: taskId, error: err instanceof Error ? err.message : String(err) }); } - await emitTaskEvent(taskId, 'task_failed', { error_message: errorMessage }); - if (releaseConcurrency) { - await decrementConcurrency(userId); + // Only emit / release concurrency after a successful transition. Callers such as + // orchestrate-task rethrow after failTask; Durable Execution retries the step and + // would otherwise re-run emit + decrement while the task is already FAILED. + if (transitioned) { + await emitTaskEvent(taskId, 'task_failed', { error_message: errorMessage }); + if (releaseConcurrency) { + await decrementConcurrency(userId); + } } } diff --git a/cdk/src/handlers/shared/response.ts b/cdk/src/handlers/shared/response.ts index 0e1fad2b..096a6522 100644 --- a/cdk/src/handlers/shared/response.ts +++ b/cdk/src/handlers/shared/response.ts @@ -36,6 +36,14 @@ export const ErrorCode = { REPO_NOT_ONBOARDED: 'REPO_NOT_ONBOARDED', SERVICE_UNAVAILABLE: 'SERVICE_UNAVAILABLE', INTERNAL_ERROR: 'INTERNAL_ERROR', + ATTACHMENT_BLOCKED: 'ATTACHMENT_BLOCKED', + ATTACHMENT_TOO_LARGE: 'ATTACHMENT_TOO_LARGE', + ATTACHMENT_INLINE_TOO_LARGE: 'ATTACHMENT_INLINE_TOO_LARGE', + ATTACHMENTS_TOTAL_TOO_LARGE: 'ATTACHMENTS_TOTAL_TOO_LARGE', + ATTACHMENT_INVALID_TYPE: 'ATTACHMENT_INVALID_TYPE', + ATTACHMENT_INVALID_CONTENT: 'ATTACHMENT_INVALID_CONTENT', + ATTACHMENT_INVALID_FILENAME: 'ATTACHMENT_INVALID_FILENAME', + ATTACHMENT_SCREENING_UNAVAILABLE: 'ATTACHMENT_SCREENING_UNAVAILABLE', } as const; const COMMON_HEADERS = { diff --git a/cdk/src/handlers/shared/types.ts b/cdk/src/handlers/shared/types.ts index 9aff1918..88daebcb 100644 --- a/cdk/src/handlers/shared/types.ts +++ b/cdk/src/handlers/shared/types.ts @@ -26,6 +26,12 @@ import type { TaskStatusType } from '../../constructs/task-status'; /** Valid task types for task creation. */ export type TaskType = 'new_task' | 'pr_iteration' | 'pr_review'; +/** Shared across all attachment interfaces. Add new types here (e.g., 'audio'). */ +export type AttachmentType = 'image' | 'file' | 'url'; + +/** Delivery mechanism — discriminant for the three upload paths. */ +export type AttachmentDelivery = 'inline' | 'presigned' | 'url_fetch'; + /** * Provenance of a task's submission. Shared across inbound adapters: * - ``api``: CLI / Cognito-authenticated submissions @@ -124,6 +130,7 @@ export interface TaskRecord { * dispatch fires successfully. */ readonly github_comment_id?: number; + readonly attachments?: AttachmentRecord[]; } /** Per-channel override for one notification channel. See @@ -198,6 +205,7 @@ export interface TaskDetail { * the field being present; CLI download resolves this via the * ``get-trace-url`` handler rather than hitting S3 directly. */ readonly trace_s3_uri: string | null; + readonly attachments: AttachmentSummary[] | null; } /** @@ -278,14 +286,140 @@ export interface CreateTaskRequest { } /** - * Attachment in create task request. + * Wire format — parsed from untrusted JSON. Validate before use. */ export interface Attachment { - readonly type: 'image' | 'file' | 'url'; + readonly type: AttachmentType; readonly content_type?: string; readonly data?: string; readonly url?: string; readonly filename?: string; + readonly expected_size_bytes?: number; +} + +// --------------------------------------------------------------------------- +// Validated attachment types (post-validation discriminated union) +// --------------------------------------------------------------------------- + +interface BaseValidatedAttachment { + readonly filename: string; + readonly content_type: string; +} + +/** Inline image/file: data present, validated, decoded, magic-bytes checked. */ +export interface InlineAttachment extends BaseValidatedAttachment { + readonly delivery: 'inline'; + readonly type: 'image' | 'file'; + readonly data: string; + readonly url?: never; + readonly decoded_size_bytes: number; +} + +/** Presigned upload: metadata only, no data, no url. */ +export interface PresignedAttachment extends BaseValidatedAttachment { + readonly delivery: 'presigned'; + readonly type: 'image' | 'file'; + readonly data?: never; + readonly url?: never; + readonly expected_size_bytes: number; +} + +/** URL to fetch during hydration. */ +export interface UrlAttachment extends BaseValidatedAttachment { + readonly delivery: 'url_fetch'; + readonly type: 'url'; + readonly url: string; + readonly data?: never; +} + +/** Output of validateAttachments() — illegal combinations are unrepresentable. */ +export type ValidatedAttachment = InlineAttachment | PresignedAttachment | UrlAttachment; + +// --------------------------------------------------------------------------- +// Screening result (persisted in DynamoDB as part of AttachmentRecord) +// --------------------------------------------------------------------------- + +/** Screening outcome — discriminated union prevents invalid combinations. */ +export type ScreeningResult = + | { readonly status: 'pending' } + | { readonly status: 'passed'; readonly screened_at: string } + | { readonly status: 'blocked'; readonly screened_at: string; readonly categories: [string, ...string[]] }; + +// --------------------------------------------------------------------------- +// Attachment record (persisted metadata in TaskRecord) +// --------------------------------------------------------------------------- + +export interface AttachmentRecord { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly content_type: string; + readonly filename: string; + readonly s3_key?: string; + readonly s3_version_id?: string; + readonly size_bytes?: number; + readonly screening: ScreeningResult; + readonly source_url?: string; + readonly checksum_sha256?: string; + readonly token_estimate?: number; +} + +/** Parameters for creating an AttachmentRecord with cross-field invariant validation. */ +export type CreateAttachmentRecordParams = AttachmentRecord; + +/** + * Factory function enforcing cross-field invariants on AttachmentRecord construction. + * Validates that required fields are present based on screening status and type. + */ +export function createAttachmentRecord(params: CreateAttachmentRecordParams): AttachmentRecord { + if (params.screening.status === 'passed') { + if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { + throw new Error('Passed screening requires s3_key, s3_version_id, checksum_sha256, and size_bytes'); + } + } + return params; +} + +// --------------------------------------------------------------------------- +// Attachment summary (API response — metadata only, no binary content) +// --------------------------------------------------------------------------- + +export interface AttachmentSummary { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly filename: string; + readonly content_type: string; + readonly size_bytes: number; + readonly screening_status: 'passed' | 'blocked' | 'pending'; +} + +// --------------------------------------------------------------------------- +// Presigned upload response (returned on PENDING_UPLOADS creation) +// --------------------------------------------------------------------------- + +export interface AttachmentUploadInstruction { + readonly attachment_id: string; + readonly filename: string; + readonly upload_url: string; + readonly upload_fields: Record; + readonly upload_expires_at: string; +} + +// --------------------------------------------------------------------------- +// Agent attachment payload (orchestrator → agent runtime) +// --------------------------------------------------------------------------- + +/** Attachment descriptor sent to the agent runtime. Exported for test assertions. */ +export interface AgentAttachmentPayload { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly content_type: string; + readonly filename: string; + readonly s3_uri: string; + readonly s3_version_id: string; + readonly size_bytes: number; + readonly source_url?: string; + readonly token_estimate?: number; + readonly checksum_sha256: string; } /** @@ -333,6 +467,16 @@ export function toTaskDetail(record: TaskRecord): TaskDetail { prompt_version: record.prompt_version ?? null, trace: record.trace === true, trace_s3_uri: record.trace_s3_uri ?? null, + attachments: record.attachments + ? record.attachments.map(a => ({ + attachment_id: a.attachment_id, + type: a.type, + filename: a.filename, + content_type: a.content_type, + size_bytes: a.size_bytes ?? 0, // 0 for pending attachments (size unknown until resolved) + screening_status: a.screening.status, + })) + : null, }; } diff --git a/cdk/src/handlers/shared/validation.ts b/cdk/src/handlers/shared/validation.ts index 11398c58..18fa369e 100644 --- a/cdk/src/handlers/shared/validation.ts +++ b/cdk/src/handlers/shared/validation.ts @@ -17,7 +17,15 @@ * SOFTWARE. */ -import { type CreateTaskRequest, type TaskType } from './types'; +import { + type CreateTaskRequest, + type TaskType, + type AttachmentType, + type ValidatedAttachment, + type InlineAttachment, + type PresignedAttachment, + type UrlAttachment, +} from './types'; import { TaskStatus } from '../../constructs/task-status'; /** Default maximum agent turns per task. */ @@ -27,7 +35,7 @@ export const MIN_MAX_TURNS = 1; /** Maximum allowed value for max_turns. */ export const MAX_MAX_TURNS = 500; /** Maximum allowed length for task_description. */ -export const MAX_TASK_DESCRIPTION_LENGTH = 2000; +export const MAX_TASK_DESCRIPTION_LENGTH = 10_000; /** Minimum allowed value for max_budget_usd (1 cent). */ export const MIN_MAX_BUDGET_USD = 0.01; /** Maximum allowed value for max_budget_usd ($100). */ @@ -239,3 +247,314 @@ export function validatePrNumber(value: unknown): number | null | undefined { if (value < 1) return null; return value; } + +// --------------------------------------------------------------------------- +// Attachment validation +// --------------------------------------------------------------------------- + +/** Maximum attachments per task. */ +export const MAX_ATTACHMENTS_PER_TASK = 10; +/** Maximum decoded size for inline attachments. */ +export const MAX_INLINE_ATTACHMENT_SIZE_BYTES = 500 * 1024; +/** Maximum total decoded inline size per request. */ +export const MAX_TOTAL_INLINE_SIZE_BYTES = 3 * 1024 * 1024; +/** Maximum size per attachment (inline or presigned, decoded). */ +export const MAX_ATTACHMENT_SIZE_BYTES = 10 * 1024 * 1024; +/** Maximum total attachment size per task. */ +export const MAX_TOTAL_ATTACHMENT_SIZE_BYTES = 50 * 1024 * 1024; + +/** Compile-time exhaustiveness check for AttachmentType. */ +const ATTACHMENT_TYPE_LIST = ['image', 'file', 'url'] as const satisfies readonly AttachmentType[]; +type _AssertAttachmentExhaustive = Exclude extends never ? true : never; +const _attachmentExhaustiveCheck: _AssertAttachmentExhaustive = true; // eslint-disable-line @typescript-eslint/no-unused-vars +const VALID_ATTACHMENT_TYPES = new Set(ATTACHMENT_TYPE_LIST); + +/** Allowed image MIME types (Bedrock vision-supported formats). */ +const ALLOWED_IMAGE_MIME_TYPES = new Set([ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', +]); + +/** Allowed file MIME types. */ +const ALLOWED_FILE_MIME_TYPES = new Set([ + 'text/plain', + 'text/csv', + 'text/markdown', + 'application/json', + 'application/pdf', + 'text/x-log', +]); + +/** + * Magic byte signatures for content type validation. + * Prevents polyglot files from bypassing screening. + */ +const MAGIC_BYTES: ReadonlyArray<{ readonly mime: string; readonly bytes: readonly number[]; readonly offset?: number }> = [ + { mime: 'image/png', bytes: [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A] }, + { mime: 'image/jpeg', bytes: [0xFF, 0xD8, 0xFF] }, + { mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] }, // GIF8 (covers GIF87a and GIF89a) + { mime: 'application/pdf', bytes: [0x25, 0x50, 0x44, 0x46, 0x2D] }, // %PDF- +]; + +// RIFF....WEBP requires checking bytes 0-3 (RIFF) and 8-11 (WEBP) +function isWebP(data: Buffer): boolean { + if (data.length < 12) return false; + return data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46 + && data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50; +} + +/** + * Validate content against declared MIME type using magic bytes. + * For text types, checks for valid UTF-8 and no null bytes. + */ +export function validateMagicBytes(data: Buffer, contentType: string): boolean { + // WebP has a split signature + if (contentType === 'image/webp') return isWebP(data); + + // Check against known binary signatures + const sig = MAGIC_BYTES.find(s => s.mime === contentType); + if (sig) { + if (data.length < sig.bytes.length) return false; + const offset = sig.offset ?? 0; + return sig.bytes.every((b, i) => data[offset + i] === b); + } + + // Text types: valid UTF-8, no null bytes in first 8 KB + if (contentType.startsWith('text/') || contentType === 'application/json') { + const check = data.subarray(0, 8192); + for (let i = 0; i < check.length; i++) { + if (check[i] === 0) return false; + } + return true; + } + + // Unknown type — reject + return false; +} + +/** + * Detect MIME type from magic bytes (for inline attachments without content_type). + */ +export function detectMimeTypeFromMagicBytes(data: Buffer): string | null { + if (isWebP(data)) return 'image/webp'; + for (const sig of MAGIC_BYTES) { + if (data.length >= sig.bytes.length) { + const offset = sig.offset ?? 0; + if (sig.bytes.every((b, i) => data[offset + i] === b)) return sig.mime; + } + } + // Try text detection + const check = data.subarray(0, 8192); + let hasNullByte = false; + for (let i = 0; i < check.length; i++) { + if (check[i] === 0) { hasNullByte = true; break; } + } + if (!hasNullByte && data.length > 0) { + // Guess JSON if it starts with { or [ + const first = data[0]; + if (first === 0x7B || first === 0x5B) return 'application/json'; + return 'text/plain'; + } + return null; +} + +/** Check if a MIME type is in the allowlist for the given attachment type. */ +export function isAllowedMimeType(mimeType: string, attachmentType: string): boolean { + if (attachmentType === 'image') return ALLOWED_IMAGE_MIME_TYPES.has(mimeType); + if (attachmentType === 'file') return ALLOWED_FILE_MIME_TYPES.has(mimeType); + if (attachmentType === 'url') { + return ALLOWED_IMAGE_MIME_TYPES.has(mimeType) || ALLOWED_FILE_MIME_TYPES.has(mimeType); + } + return false; +} + +/** Validate a URL is HTTPS-only. */ +function isValidHttpsUrl(urlStr: string): boolean { + try { + const parsed = new URL(urlStr); + return parsed.protocol === 'https:'; + } catch { + return false; + } +} + +/** Reject filenames with path traversal, null bytes, or unusual characters. */ +export function isValidFilename(filename: string): boolean { + if (filename.length === 0 || filename.length > 255) return false; + if (filename.includes('/') || filename.includes('\\')) return false; + if (filename.includes('\0')) return false; + if (filename.startsWith('.') || filename.startsWith('-')) return false; + if (filename === '.' || filename === '..') return false; + return /^[a-zA-Z0-9][a-zA-Z0-9._\- ]{0,253}[a-zA-Z0-9._]$/.test(filename) + || /^[a-zA-Z0-9][a-zA-Z0-9._]$/.test(filename) + || /^[a-zA-Z0-9]$/.test(filename); +} + +/** Generate a default filename when none was provided. */ +function generateFilename(type: string, contentType: string, index: number): string { + const ext = MIME_TO_EXTENSION[contentType] ?? 'bin'; + return `attachment_${index}.${ext}`; +} + +const MIME_TO_EXTENSION: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/gif': 'gif', + 'image/webp': 'webp', + 'text/plain': 'txt', + 'text/csv': 'csv', + 'text/markdown': 'md', + 'application/json': 'json', + 'application/pdf': 'pdf', + 'text/x-log': 'log', +}; + +export type AttachmentValidationResult = + | { readonly valid: true; readonly parsed: ValidatedAttachment[] } + | { readonly valid: false; readonly error: string }; + +/** + * Synchronous attachment validation. Checks schema, limits, magic bytes, + * MIME types, and filename safety. Returns a discriminated union of + * validated attachments on success. + */ +export function validateAttachments( + attachments: unknown[] | undefined, +): AttachmentValidationResult { + if (!attachments || !Array.isArray(attachments) || attachments.length === 0) { + return { valid: true, parsed: [] }; + } + if (attachments.length > MAX_ATTACHMENTS_PER_TASK) { + return { valid: false, error: `Maximum ${MAX_ATTACHMENTS_PER_TASK} attachments per task` }; + } + + let totalInlineSize = 0; + let totalDeclaredSize = 0; + const parsed: ValidatedAttachment[] = []; + + for (const [i, att] of (attachments as Record[]).entries()) { + // Type validation + if (!att.type || typeof att.type !== 'string' || !VALID_ATTACHMENT_TYPES.has(att.type)) { + return { valid: false, error: `attachments[${i}].type must be 'image', 'file', or 'url'` }; + } + + const attType = att.type as AttachmentType; + + // Mutual exclusivity: data vs url + if (attType === 'url') { + if (!att.url || typeof att.url !== 'string') { + return { valid: false, error: `attachments[${i}]: url required for type 'url'` }; + } + if (att.data) { + return { valid: false, error: `attachments[${i}]: data not allowed for type 'url'` }; + } + if (!isValidHttpsUrl(att.url)) { + return { valid: false, error: `attachments[${i}]: must be a valid HTTPS URL` }; + } + } else { + if (att.data && att.url) { + return { valid: false, error: `attachments[${i}]: provide data or url, not both` }; + } + } + + // Decode inline data + let decoded: Buffer | undefined; + + if (att.data && typeof att.data === 'string') { + decoded = Buffer.from(att.data as string, 'base64'); + if (decoded.length > MAX_INLINE_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `attachments[${i}]: inline data exceeds 500 KB limit. Use presigned upload for larger files.` }; + } + totalInlineSize += decoded.length; + } + + // Declared size validation (for presigned uploads) + if (!att.data && !att.url && attType !== 'url') { + if (typeof att.expected_size_bytes !== 'number' || att.expected_size_bytes <= 0) { + return { valid: false, error: `attachments[${i}]: expected_size_bytes required for presigned uploads` }; + } + if (att.expected_size_bytes > MAX_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `attachments[${i}]: expected size exceeds 10 MB limit` }; + } + totalDeclaredSize += att.expected_size_bytes; + } + + // MIME type resolution and validation + let resolvedContentType: string; + if (att.content_type && typeof att.content_type === 'string') { + if (!isAllowedMimeType(att.content_type, attType)) { + return { valid: false, error: `attachments[${i}]: content_type '${att.content_type}' not allowed for type '${attType}'` }; + } + resolvedContentType = att.content_type; + } else if (decoded) { + // Magic bytes validation before detection + const detected = detectMimeTypeFromMagicBytes(decoded); + if (!detected) { + return { valid: false, error: `attachments[${i}]: could not determine file type. Please provide content_type explicitly.` }; + } + if (!isAllowedMimeType(detected, attType)) { + return { valid: false, error: `attachments[${i}]: detected content_type '${detected}' not allowed for type '${attType}'` }; + } + resolvedContentType = detected; + } else { + return { valid: false, error: `attachments[${i}]: content_type is required for presigned uploads and URL attachments` }; + } + + // Magic bytes check against declared content_type (for inline data with declared type) + if (decoded && att.content_type) { + if (!validateMagicBytes(decoded, resolvedContentType)) { + return { valid: false, error: `attachments[${i}]: content does not match declared type` }; + } + } + + // Filename resolution + const resolvedFilename = (att.filename && typeof att.filename === 'string') + ? att.filename + : generateFilename(attType, resolvedContentType, i); + if (!isValidFilename(resolvedFilename)) { + return { valid: false, error: `attachments[${i}]: invalid filename` }; + } + + // Construct validated variant + if (attType === 'url') { + parsed.push({ + delivery: 'url_fetch', + type: 'url', + url: att.url as string, + filename: resolvedFilename, + content_type: resolvedContentType, + } satisfies UrlAttachment); + } else if (decoded) { + parsed.push({ + delivery: 'inline', + type: attType, + data: att.data as string, + filename: resolvedFilename, + content_type: resolvedContentType, + decoded_size_bytes: decoded.length, + } satisfies InlineAttachment); + } else { + parsed.push({ + delivery: 'presigned', + type: attType, + filename: resolvedFilename, + content_type: resolvedContentType, + expected_size_bytes: att.expected_size_bytes as number, + } satisfies PresignedAttachment); + } + } + + // Total inline size check + if (totalInlineSize > MAX_TOTAL_INLINE_SIZE_BYTES) { + return { valid: false, error: 'Total inline attachment size exceeds 3 MB limit. Use presigned upload for larger files.' }; + } + + // Total declared size check (inline + presigned) + if (totalInlineSize + totalDeclaredSize > MAX_TOTAL_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: 'Total attachment size exceeds 50 MB limit' }; + } + + return { valid: true, parsed }; +} diff --git a/cdk/src/stacks/agent.ts b/cdk/src/stacks/agent.ts index 7d5c3d08..99507dc6 100644 --- a/cdk/src/stacks/agent.ts +++ b/cdk/src/stacks/agent.ts @@ -33,15 +33,16 @@ import { NagSuppressions } from 'cdk-nag'; import { Construct } from 'constructs'; import { AgentMemory } from '../constructs/agent-memory'; import { AgentVpc } from '../constructs/agent-vpc'; +import { AttachmentsBucket } from '../constructs/attachments-bucket'; import { Blueprint } from '../constructs/blueprint'; import { ConcurrencyReconciler } from '../constructs/concurrency-reconciler'; import { DnsFirewall } from '../constructs/dns-firewall'; +// import { EcsAgentCluster } from '../constructs/ecs-agent-cluster'; import { FanOutConsumer } from '../constructs/fanout-consumer'; import { LinearIntegration } from '../constructs/linear-integration'; import { RepoTable } from '../constructs/repo-table'; import { SlackIntegration } from '../constructs/slack-integration'; import { StrandedTaskReconciler } from '../constructs/stranded-task-reconciler'; -// import { EcsAgentCluster } from '../constructs/ecs-agent-cluster'; import { TaskApi } from '../constructs/task-api'; import { TaskDashboard } from '../constructs/task-dashboard'; import { TaskEventsTable } from '../constructs/task-events-table'; @@ -72,6 +73,16 @@ export class AgentStack extends Stack { // written when the submit payload sets ``trace: true``. const traceArtifactsBucket = new TraceArtifactsBucket(this, 'TraceArtifactsBucket'); + // Attachment storage — images, files, and URL-fetched content for tasks. + const attachmentsBucket = new AttachmentsBucket(this, 'AttachmentsBucket'); + + NagSuppressions.addResourceSuppressions(attachmentsBucket.bucket, [ + { + id: 'AwsSolutions-S1', + reason: 'Task attachments: writes from create-task and orchestrator Lambdas only; reads by agent via IAM role. 90-day lifecycle; versioning + screening prevent TOCTOU. Access logging not justified for this use case.', + }, + ]); + // Server access logging intentionally disabled. Rationale: // - writes: only the agent runtime IAM role (``grantPut`` below). // - reads: only via short-lived presigned URL issued by @@ -225,6 +236,7 @@ export class AgentStack extends Stack { guardrailVersion: inputGuardrail.guardrailVersion, agentCoreStopSessionRuntimeArn: lazyRuntimeArn, traceArtifactsBucket: traceArtifactsBucket.bucket, + attachmentsBucket: attachmentsBucket.bucket, }); // --- AgentCore Runtime (IAM-authed orchestrator path) --- @@ -455,6 +467,7 @@ export class AgentStack extends Stack { memoryId: agentMemory.memory.memoryId, guardrailId: inputGuardrail.guardrailId, guardrailVersion: inputGuardrail.guardrailVersion, + attachmentsBucket: attachmentsBucket.bucket, // To wire ECS, uncomment the ecsCluster block above and add: // ecsConfig: { // clusterArn: ecsCluster.cluster.clusterArn, diff --git a/cdk/test/handlers/orchestrate-task.test.ts b/cdk/test/handlers/orchestrate-task.test.ts index 0151fe74..5af9dbfa 100644 --- a/cdk/test/handlers/orchestrate-task.test.ts +++ b/cdk/test/handlers/orchestrate-task.test.ts @@ -725,12 +725,24 @@ describe('failTask', () => { expect(transitionCall.input.ExpressionAttributeValues[':toStatus']).toBe('FAILED'); }); - test('handles transition failure gracefully', async () => { - mockDdbSend - .mockRejectedValueOnce(new Error('Condition failed')) // transitionTask - .mockResolvedValue({}); // emitTaskEvent - // Should not throw - await failTask('TASK001', 'SUBMITTED', 'error', 'user-123', false); + test('handles transition failure gracefully without emitting when not transitioned', async () => { + mockDdbSend.mockRejectedValueOnce(new Error('Condition failed')); // transitionTask only + await expect(failTask('TASK001', 'SUBMITTED', 'error', 'user-123', false)).resolves.toBeUndefined(); + expect(mockDdbSend).toHaveBeenCalledTimes(1); + }); + + test('second failTask does not re-emit or re-decrement when transition fails (idempotent under step retry)', async () => { + mockDdbSend.mockResolvedValue({}); + await failTask('TASK001', 'HYDRATING', 'first failure', 'user-123', true); + expect(mockDdbSend).toHaveBeenCalledTimes(3); // transition + emit + decrement + + mockDdbSend.mockClear(); + const condErr = new Error('The conditional request failed'); + condErr.name = 'ConditionalCheckFailedException'; + mockDdbSend.mockRejectedValueOnce(condErr); // already FAILED — transition no-ops + + await expect(failTask('TASK001', 'HYDRATING', 'durable replay', 'user-123', true)).resolves.toBeUndefined(); + expect(mockDdbSend).toHaveBeenCalledTimes(1); // transition attempt only; no Put, no concurrency Update }); }); diff --git a/cdk/test/handlers/shared/create-task-core.test.ts b/cdk/test/handlers/shared/create-task-core.test.ts index e03323de..445bf0a9 100644 --- a/cdk/test/handlers/shared/create-task-core.test.ts +++ b/cdk/test/handlers/shared/create-task-core.test.ts @@ -366,7 +366,7 @@ describe('createTaskCore', () => { test('returns 400 when task_description exceeds length limit', async () => { const result = await createTaskCore( - { repo: 'org/repo', task_description: 'a'.repeat(2001) }, + { repo: 'org/repo', task_description: 'a'.repeat(10_001) }, makeContext(), 'req-1', ); @@ -376,7 +376,7 @@ describe('createTaskCore', () => { test('accepts task_description at exactly the length limit', async () => { const result = await createTaskCore( - { repo: 'org/repo', task_description: 'a'.repeat(2000) }, + { repo: 'org/repo', task_description: 'a'.repeat(10_000) }, makeContext(), 'req-1', ); diff --git a/cdk/test/handlers/shared/validation.test.ts b/cdk/test/handlers/shared/validation.test.ts index 4aecf4e5..b4b0f152 100644 --- a/cdk/test/handlers/shared/validation.test.ts +++ b/cdk/test/handlers/shared/validation.test.ts @@ -22,21 +22,28 @@ import { decodePaginationToken, encodePaginationToken, hasTaskSpec, + isAllowedMimeType, + isValidFilename, isValidIdempotencyKey, isValidRepo, isValidTaskDescriptionLength, isValidTaskType, isValidUlid, isValidWebhookName, + MAX_ATTACHMENTS_PER_TASK, MAX_TASK_DESCRIPTION_LENGTH, parseBody, parseLimit, parseStatusFilter, + validateAttachments, + validateMagicBytes, VALID_TASK_TYPES, validateMaxTurns, validatePrNumber, } from '../../../src/handlers/shared/validation'; +import { createAttachmentRecord } from '../../../src/handlers/shared/types'; + describe('parseBody', () => { test('parses valid JSON', () => { expect(parseBody('{"key":"value"}')).toEqual({ key: 'value' }); @@ -407,3 +414,300 @@ describe('validatePrNumber', () => { expect(validatePrNumber(true)).toBeNull(); }); }); + +describe('validateAttachments', () => { + // Helper: minimal valid PNG (1x1 pixel) + const PNG_HEADER = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, + 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52]); + const pngBase64 = PNG_HEADER.toString('base64'); + + // Helper: valid JSON content + const jsonContent = Buffer.from('{"key": "value"}'); + const jsonBase64 = jsonContent.toString('base64'); + + test('returns valid with empty parsed array for undefined input', () => { + const result = validateAttachments(undefined); + expect(result.valid).toBe(true); + if (result.valid) expect(result.parsed).toEqual([]); + }); + + test('returns valid with empty parsed array for empty array', () => { + const result = validateAttachments([]); + expect(result.valid).toBe(true); + if (result.valid) expect(result.parsed).toEqual([]); + }); + + test('rejects more than MAX_ATTACHMENTS_PER_TASK', () => { + const tooMany = Array.from({ length: MAX_ATTACHMENTS_PER_TASK + 1 }, () => ({ + type: 'image', data: pngBase64, content_type: 'image/png', filename: 'img.png', + })); + const result = validateAttachments(tooMany); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('Maximum'); + }); + + test('validates a valid inline image attachment', () => { + const result = validateAttachments([{ + type: 'image', data: pngBase64, content_type: 'image/png', filename: 'screenshot.png', + }]); + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.parsed).toHaveLength(1); + expect(result.parsed[0].delivery).toBe('inline'); + expect(result.parsed[0].filename).toBe('screenshot.png'); + } + }); + + test('validates a valid inline file attachment', () => { + const result = validateAttachments([{ + type: 'file', data: jsonBase64, content_type: 'application/json', filename: 'data.json', + }]); + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.parsed).toHaveLength(1); + expect(result.parsed[0].delivery).toBe('inline'); + expect(result.parsed[0].type).toBe('file'); + } + }); + + test('validates a valid URL attachment', () => { + const result = validateAttachments([{ + type: 'url', url: 'https://example.com/image.png', content_type: 'image/png', filename: 'remote.png', + }]); + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.parsed).toHaveLength(1); + expect(result.parsed[0].delivery).toBe('url_fetch'); + } + }); + + test('validates a valid presigned upload attachment', () => { + const result = validateAttachments([{ + type: 'image', content_type: 'image/png', filename: 'big.png', expected_size_bytes: 2_000_000, + }]); + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.parsed).toHaveLength(1); + expect(result.parsed[0].delivery).toBe('presigned'); + } + }); + + test('rejects invalid type', () => { + const result = validateAttachments([{ type: 'video', data: 'abc' }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('type'); + }); + + test('rejects url type without url field', () => { + const result = validateAttachments([{ type: 'url', content_type: 'image/png', filename: 'x.png' }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('url required'); + }); + + test('rejects non-HTTPS URL', () => { + const result = validateAttachments([{ + type: 'url', url: 'http://example.com/img.png', content_type: 'image/png', filename: 'x.png', + }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('HTTPS'); + }); + + test('rejects data + url together', () => { + const result = validateAttachments([{ + type: 'image', data: pngBase64, url: 'https://example.com/x.png', + content_type: 'image/png', filename: 'x.png', + }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('not both'); + }); + + test('rejects inline data exceeding 500 KB', () => { + const bigData = Buffer.alloc(501 * 1024).fill(0x50); // starts with P (text-ish) + const result = validateAttachments([{ + type: 'file', data: bigData.toString('base64'), content_type: 'text/plain', filename: 'big.txt', + }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('500 KB'); + }); + + test('rejects presigned upload without expected_size_bytes', () => { + const result = validateAttachments([{ + type: 'image', content_type: 'image/png', filename: 'big.png', + }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('expected_size_bytes'); + }); + + test('rejects presigned upload exceeding 10 MB', () => { + const result = validateAttachments([{ + type: 'image', content_type: 'image/png', filename: 'huge.png', + expected_size_bytes: 11 * 1024 * 1024, + }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('10 MB'); + }); + + test('rejects disallowed MIME type for image', () => { + const result = validateAttachments([{ + type: 'image', data: pngBase64, content_type: 'image/svg+xml', filename: 'icon.svg', + }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('not allowed'); + }); + + test('rejects magic bytes mismatch (PNG header with JPEG declared type)', () => { + const result = validateAttachments([{ + type: 'image', data: pngBase64, content_type: 'image/jpeg', filename: 'fake.jpg', + }]); + expect(result.valid).toBe(false); + if (!result.valid) expect(result.error).toContain('does not match'); + }); + + test('generates filename when not provided', () => { + const result = validateAttachments([{ + type: 'file', data: jsonBase64, content_type: 'application/json', + }]); + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.parsed[0].filename).toBe('attachment_0.json'); + } + }); +}); + +describe('isValidFilename', () => { + test('accepts valid filenames', () => { + expect(isValidFilename('screenshot.png')).toBe(true); + expect(isValidFilename('my-file_v2.txt')).toBe(true); + expect(isValidFilename('a')).toBe(true); + expect(isValidFilename('a1')).toBe(true); + }); + + test('rejects path traversal', () => { + expect(isValidFilename('../etc/passwd')).toBe(false); + expect(isValidFilename('foo/bar.txt')).toBe(false); + expect(isValidFilename('foo\\bar.txt')).toBe(false); + }); + + test('rejects dotfiles and dash-prefix', () => { + expect(isValidFilename('.hidden')).toBe(false); + expect(isValidFilename('-flag')).toBe(false); + }); + + test('rejects null bytes', () => { + expect(isValidFilename('file\x00.txt')).toBe(false); + }); + + test('rejects empty and too-long filenames', () => { + expect(isValidFilename('')).toBe(false); + expect(isValidFilename('a'.repeat(256))).toBe(false); + }); +}); + +describe('validateMagicBytes', () => { + test('validates PNG magic bytes', () => { + const png = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00]); + expect(validateMagicBytes(png, 'image/png')).toBe(true); + }); + + test('validates JPEG magic bytes', () => { + const jpeg = Buffer.from([0xFF, 0xD8, 0xFF, 0xE0, 0x00]); + expect(validateMagicBytes(jpeg, 'image/jpeg')).toBe(true); + }); + + test('validates text content', () => { + const text = Buffer.from('Hello, world!'); + expect(validateMagicBytes(text, 'text/plain')).toBe(true); + }); + + test('rejects text with null bytes', () => { + const binary = Buffer.from([0x48, 0x65, 0x00, 0x6C]); + expect(validateMagicBytes(binary, 'text/plain')).toBe(false); + }); + + test('rejects mismatched signatures', () => { + const jpeg = Buffer.from([0xFF, 0xD8, 0xFF, 0xE0]); + expect(validateMagicBytes(jpeg, 'image/png')).toBe(false); + }); +}); + +describe('isAllowedMimeType', () => { + test('allows valid image types', () => { + expect(isAllowedMimeType('image/png', 'image')).toBe(true); + expect(isAllowedMimeType('image/jpeg', 'image')).toBe(true); + expect(isAllowedMimeType('image/gif', 'image')).toBe(true); + expect(isAllowedMimeType('image/webp', 'image')).toBe(true); + }); + + test('allows valid file types', () => { + expect(isAllowedMimeType('text/plain', 'file')).toBe(true); + expect(isAllowedMimeType('application/json', 'file')).toBe(true); + expect(isAllowedMimeType('application/pdf', 'file')).toBe(true); + }); + + test('rejects disallowed types', () => { + expect(isAllowedMimeType('application/javascript', 'file')).toBe(false); + expect(isAllowedMimeType('image/svg+xml', 'image')).toBe(false); + expect(isAllowedMimeType('application/zip', 'file')).toBe(false); + }); + + test('url type allows both image and file types', () => { + expect(isAllowedMimeType('image/png', 'url')).toBe(true); + expect(isAllowedMimeType('text/plain', 'url')).toBe(true); + }); +}); + +describe('createAttachmentRecord', () => { + test('creates a valid record with passed screening', () => { + const record = createAttachmentRecord({ + attachment_id: 'att-1', + type: 'file', + content_type: 'text/plain', + filename: 'log.txt', + s3_key: 'attachments/user/task/att/log.txt', + s3_version_id: 'v1', + size_bytes: 100, + screening: { status: 'passed', screened_at: '2026-01-01T00:00:00Z' }, + checksum_sha256: 'a'.repeat(64), + }); + expect(record.attachment_id).toBe('att-1'); + expect(record.screening.status).toBe('passed'); + }); + + test('creates a valid pending record (URL type)', () => { + const record = createAttachmentRecord({ + attachment_id: 'att-2', + type: 'url', + content_type: 'image/png', + filename: 'remote.png', + screening: { status: 'pending' }, + source_url: 'https://example.com/img.png', + }); + expect(record.screening.status).toBe('pending'); + }); + + test('throws when passed screening lacks s3_key', () => { + expect(() => createAttachmentRecord({ + attachment_id: 'att-3', + type: 'file', + content_type: 'text/plain', + filename: 'log.txt', + screening: { status: 'passed', screened_at: '2026-01-01T00:00:00Z' }, + })).toThrow('s3_key'); + }); + + test('accepts image with passed screening without token_estimate (computed during hydration)', () => { + const record = createAttachmentRecord({ + attachment_id: 'att-4', + type: 'image', + content_type: 'image/png', + filename: 'img.png', + s3_key: 'attachments/user/task/att/img.png', + s3_version_id: 'v1', + size_bytes: 5000, + screening: { status: 'passed', screened_at: '2026-01-01T00:00:00Z' }, + checksum_sha256: 'b'.repeat(64), + }); + expect(record.attachment_id).toBe('att-4'); + expect(record.token_estimate).toBeUndefined(); + }); +}); diff --git a/cli/src/types.ts b/cli/src/types.ts index 628d4562..c0417b9d 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -20,6 +20,9 @@ /** Valid task types for task creation. */ export type TaskType = 'new_task' | 'pr_iteration' | 'pr_review'; +/** Shared across all attachment interfaces. Add new types here (e.g., 'audio'). */ +export type AttachmentType = 'image' | 'file' | 'url'; + /** * Provenance of a task's submission. Shared across inbound adapters: * - ``api``: CLI / Cognito-authenticated submissions @@ -96,6 +99,7 @@ export interface TaskDetail { * the URI in ``status --output json`` lets users / scripts detect * completion without an extra round trip. */ readonly trace_s3_uri: string | null; + readonly attachments: AttachmentSummary[] | null; } /** Response body of ``GET /v1/tasks/{task_id}/trace`` (design §10.1). */ @@ -148,6 +152,35 @@ export interface GetTaskEventsQuery { readonly desc?: string; } +/** Wire format — parsed from untrusted JSON. Validate before use. */ +export interface Attachment { + readonly type: AttachmentType; + readonly content_type?: string; + readonly data?: string; + readonly url?: string; + readonly filename?: string; + readonly expected_size_bytes?: number; +} + +/** Attachment metadata in task detail responses. */ +export interface AttachmentSummary { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly filename: string; + readonly content_type: string; + readonly size_bytes: number; + readonly screening_status: 'passed' | 'blocked' | 'pending'; +} + +/** Presigned upload instruction returned on PENDING_UPLOADS creation. */ +export interface AttachmentUploadInstruction { + readonly attachment_id: string; + readonly filename: string; + readonly upload_url: string; + readonly upload_fields: Record; + readonly upload_expires_at: string; +} + /** Create task request body for POST /v1/tasks. */ export interface CreateTaskRequest { readonly repo: string; @@ -157,6 +190,7 @@ export interface CreateTaskRequest { readonly max_budget_usd?: number; readonly task_type?: TaskType; readonly pr_number?: number; + readonly attachments?: Attachment[]; /** * Enable the ``--trace`` debug path (design §10.1). When true, the * agent's ProgressWriter raises its preview-truncation cap from 200 diff --git a/docs/design/ATTACHMENTS.md b/docs/design/ATTACHMENTS.md new file mode 100644 index 00000000..cf744f8c --- /dev/null +++ b/docs/design/ATTACHMENTS.md @@ -0,0 +1,1691 @@ +# Task Attachments (Multimodal) + +End-to-end support for attaching files, images, and URLs to agent tasks. Attachments let users provide non-text context — screenshots of bugs, design mockups, CSV data, log files, code snippets — that the agent can reference during execution. Every channel (CLI, webhook, Slack, Linear) feeds the same schema; every attachment passes through security screening before reaching the agent. + +- **Use this doc for:** understanding the attachment data model, upload flow, security screening pipeline, storage layout, agent consumption, and per-channel behaviour. +- **Related docs:** [API_CONTRACT.md](./API_CONTRACT.md) for the `attachments` request schema (must be updated in tandem — see [API contract sync](#api-contract-sync)), [ORCHESTRATOR.md](./ORCHESTRATOR.md) for the task lifecycle this extends, [SECURITY.md](./SECURITY.md) for guardrail and Cedar context, [ARCHITECTURE.md](./ARCHITECTURE.md) for the platform overview. + +## Motivation + +Today, task context is limited to text: a `task_description`, an `issue_number` (whose body is text), or a `pr_number`. Users cannot show the agent what they see. Common situations where text alone is insufficient: + +1. **Bug reports with screenshots** — "The button is misaligned" means nothing without the screenshot. The user must upload the image to GitHub, create an issue, and reference it — friction that discourages use. +2. **Design mockups** — "Implement this design" requires a mockup image. Today the agent can only read text descriptions of designs. +3. **Log files and data** — A 500-line stack trace or CSV dataset is awkward to paste into a 10,000-char `task_description`. +4. **Code snippets from other repos** — "Port this function from repo X" requires the user to paste code into an issue. +5. **Integration payloads** — Linear issues or Slack threads may contain images, files, or links that are lost when only the text body is forwarded to the agent. + +Attachments solve this by providing a unified carrier for all non-text task context. + +## Design principles + +1. **Store once, reference everywhere.** Binary data lives in S3. Every downstream consumer (orchestrator, agent, audit log) uses S3 references, never inline blobs. The primary upload path bypasses the API entirely (presigned POST policies with size enforcement); a convenience inline path exists for small attachments only. +2. **No unscreened binary content in S3.** Every attachment passes through security screening before its binary content is written to durable S3 storage. DynamoDB metadata (attachment ID, filename, pending status) may be written before screening completes, but the actual blob is gated on a passing screen result. +3. **Fail closed, fail loud.** If screening is unavailable, the task fails. If any attachment fails screening (inline, URL, or presigned), the task fails with a clear error — attachments are not silently dropped. Users can re-submit without the problematic attachment. This principle extends to the hydration fallback path — attachment resolution errors must propagate and fail the task, never be caught by the generic infrastructure fallback (see [Hydration error handling](#hydration-error-handling)). +4. **Channel-agnostic schema.** The `Attachment` type is the same whether the source is a CLI flag, a webhook JSON field, a Slack file upload, or a Linear issue image. Channel-specific adapters normalize to this schema before entering the shared path. +5. **Agent sees files, not infrastructure.** The agent receives attachments as local files in its workspace (images, documents) or as prompt content blocks (images for multimodal models). It does not interact with S3 directly. + +## Data model + +### Shared types + +Extract a shared `AttachmentType` literal union, reused across all attachment-related interfaces (same pattern as `TaskType` and `ChannelSource`): + +```typescript +/** Shared across all attachment interfaces. Add new types here (e.g., 'audio'). */ +export type AttachmentType = 'image' | 'file' | 'url'; +``` + +### Attachment schema (API layer) + +The existing `Attachment` interface in `types.ts` is a flat union of optional fields. For deserialization from untrusted JSON this is fine, but downstream consumers benefit from a validated discriminated union. The design uses two layers: + +**Wire format (deserialization):** The existing `Attachment` interface in `types.ts` (lines 283-289) is modified in-place — adding `expected_size_bytes` for presigned upload budget pre-checks. This is a non-breaking change (new optional field). All fields beyond `type` remain optional because the input is untrusted: + +```typescript +/** Wire format — parsed from untrusted JSON. Validate before use. */ +interface Attachment { + readonly type: AttachmentType; + readonly content_type?: string; + readonly data?: string; // Base64-encoded content (inline upload, max 500 KB decoded) + readonly url?: string; // URL to fetch (url type) + readonly filename?: string; // Original filename + readonly expected_size_bytes?: number; // Declared size for presigned uploads (required for budget pre-check) +} +``` + +**Validated format (post-validation):** After `validateAttachments()` succeeds, the result is a discriminated union that makes illegal states unrepresentable: + +```typescript +/** Delivery mechanism — discriminant for the three upload paths. + * `type` alone cannot distinguish inline from presigned (both are 'image' | 'file'), + * so `delivery` provides the exhaustive three-way branch. */ +type AttachmentDelivery = 'inline' | 'presigned' | 'url_fetch'; + +interface BaseAttachment { + readonly filename: string; // Required after validation (generated if absent) + readonly content_type: string; // Required after validation (detected from magic bytes if absent) +} + +/** Inline image/file: data present, validated, decoded, magic-bytes checked */ +interface InlineAttachment extends BaseAttachment { + readonly delivery: 'inline'; + readonly type: 'image' | 'file'; + readonly data: string; // Validated base64 + readonly url?: never; + readonly decoded_size_bytes: number; +} + +/** Presigned upload: metadata only, no data, no url */ +interface PresignedAttachment extends BaseAttachment { + readonly delivery: 'presigned'; + readonly type: 'image' | 'file'; + readonly data?: never; + readonly url?: never; + readonly expected_size_bytes: number; // Declared by client for early budget validation +} + +/** URL to fetch during hydration */ +interface UrlAttachment extends BaseAttachment { + readonly delivery: 'url_fetch'; + readonly type: 'url'; + readonly url: string; + readonly data?: never; +} + +/** Output of validateAttachments() — illegal combinations are unrepresentable */ +type ValidatedAttachment = InlineAttachment | PresignedAttachment | UrlAttachment; +``` + +The `delivery` field is the primary discriminant for exhaustive `switch` statements: `switch (att.delivery)` branches into the three upload paths without nested `if ('data' in att)` checks. The `type` field (`'image' | 'file' | 'url'`) describes the content kind. The `never` fields prevent accidental inclusion of forbidden properties at compile time. Note: `never` only provides compile-time protection — the validation function must construct each variant explicitly (see [Validation changes](#validation-changes)) to guarantee forbidden fields are absent at runtime. + +### Attachment record (persisted metadata) + +After upload and screening, each attachment becomes an `AttachmentRecord` stored as part of the `TaskRecord` in DynamoDB. The `screening` field is a discriminated union that prevents nonsensical states (e.g., a `passed` record with blocking categories): + +```typescript +/** Screening outcome — discriminated union prevents invalid combinations. + * `categories` uses a non-empty tuple type to make empty-array states unrepresentable. */ +type ScreeningResult = + | { readonly status: 'pending' } + | { readonly status: 'passed'; readonly screened_at: string } + | { readonly status: 'blocked'; readonly screened_at: string; readonly categories: [string, ...string[]] }; + +interface AttachmentRecord { + readonly attachment_id: string; // ULID + readonly type: AttachmentType; + readonly content_type: string; // Resolved MIME type + readonly filename: string; // Original or generated filename + readonly s3_key?: string; // S3 object key — absent when pending (URL not yet fetched) + readonly s3_version_id?: string; // S3 object version — pinned at screening time (see S3 versioning) + readonly size_bytes?: number; // Decoded content size — absent when pending + readonly screening: ScreeningResult; + readonly source_url?: string; // Original URL (for url type or channel-sourced) + readonly checksum_sha256?: string; // Lowercase hex-encoded SHA-256 (64 chars) — required when screening.status === 'passed' (enforced by factory) + readonly token_estimate?: number; // Estimated token cost (images only) +} +``` + +**Runtime validation note:** The `ScreeningResult` discriminated union and the `[string, ...string[]]` non-empty tuple provide compile-time safety, but data read from DynamoDB is untyped at runtime. The `createAttachmentRecord` factory (below) validates these invariants at construction time. For records read back from DynamoDB, a `parseScreeningResult(raw: unknown): ScreeningResult` function must validate: (a) `status` is one of the three allowed values, (b) `screened_at` is present when status is `passed` or `blocked`, (c) `categories` is a non-empty array when status is `blocked`. This parser is called in the DynamoDB → `AttachmentRecord` mapper (same pattern as the existing `toTaskDetail` mapper). Invalid data throws rather than silently returning a malformed object. + +**Changes from prior version:** `s3_key` and `size_bytes` are now `undefined` when pending (not sentinel values `'pending'` and `0`). This is idiomatic TypeScript — the compiler forces null-checks on consumers instead of relying on documentation about magic values. + +The `TaskRecord` gains an `attachments?: AttachmentRecord[]` field. This stores metadata only — binary content lives in S3. + +**Construction (Phase 1 — ships with the type definitions):** A factory function `createAttachmentRecord(params)` centralizes construction validation, matching the codebase pattern of mapper functions like `toTaskDetail`. This ships in Phase 1 alongside the type definitions — not deferred — because cross-field invariants (`s3_key` required when screening passed, `checksum_sha256` required when screening passed, `token_estimate` required for images) are too dangerous to leave unenforced during 4 phases of development: + +```typescript +function createAttachmentRecord(params: CreateAttachmentRecordParams): AttachmentRecord { + // Validates invariants: + // - token_estimate required when type === 'image' + // - s3_key and s3_version_id required when screening.status === 'passed' + // - checksum_sha256 required when screening.status === 'passed' + // - size_bytes required when screening.status === 'passed' + // - categories non-empty when screening.status === 'blocked' + if (params.screening.status === 'passed') { + if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { + throw new Error('Passed screening requires s3_key, s3_version_id, checksum_sha256, and size_bytes'); + } + } + if (params.type === 'image' && params.screening.status === 'passed' && !params.token_estimate) { + throw new Error('Image attachments with passed screening must have token_estimate'); + } + return params as AttachmentRecord; +} +``` + +### Agent payload type + +A named TypeScript interface for the orchestrator → agent payload (not an anonymous `.map()` shape). This enables test contracts and prevents silent drift between the TypeScript producer and Python consumer: + +```typescript +/** Attachment descriptor sent to the agent runtime. Exported for test assertions. */ +export interface AgentAttachmentPayload { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly content_type: string; + readonly filename: string; + readonly s3_uri: string; // s3://bucket/attachments/user/task/att/file.png + readonly s3_version_id: string; // Pinned S3 object version — prevents TOCTOU between screening and download + readonly size_bytes: number; + readonly source_url?: string; // Original URL (for url type) + readonly token_estimate?: number; // Images only + readonly checksum_sha256: string; // Lowercase hex-encoded SHA-256 (64 chars) of screened content — agent verifies after download +} +``` + +A test should assert field-for-field parity between this interface and the Python `AttachmentConfig` Pydantic model. + +### S3 key layout + +``` +attachments//// +``` + +Example: `attachments/us-east-1:abc123/01J5X7.../01J5X8.../screenshot.png` + +The `` segment ensures uniqueness even if multiple attachments share the same filename. The `` suffix preserves the original name for human-readable S3 console browsing and for the agent (which receives the file under its original name). + +## Limits + +| Limit | Value | Rationale | +|---|---|---| +| Max attachments per task | 10 | Bounds screening cost and agent context size | +| Max size per attachment (decoded) | 10 MB | Bedrock image input limit; practical for screenshots/logs | +| Max inline data per attachment | 500 KB decoded | The Lambda synchronous invocation payload limit is **6 MB**. At 500 KB decoded (~667 KB base64) per attachment, even 5 inline attachments plus request JSON stays under 6 MB. The presigned path handles anything larger. | +| Max total inline data per request | 3 MB decoded | Hard cap on total base64-decoded bytes in a single request. Even with base64 overhead (~4 MB encoded) plus JSON fields, this stays under the 6 MB Lambda payload limit. | +| Max total size per task | 50 MB | Prevents abuse; bounds total screening and transfer time | +| Max task_description length | 10,000 chars | Increased from 2,000. **This is a standalone API change that affects all tasks** (not just attachment tasks). Rationale: (a) attachments need rich explanatory context ("implement this design per the attached mockup, paying attention to the header layout"), (b) multiple users have reported the 2K limit as a friction point for complex task descriptions even without attachments, (c) the guardrail screening cost increase is minimal (text screening is cheap), (d) DynamoDB item size impact is negligible (~8 KB vs ~2 KB for the description field). **Requires updating [API_CONTRACT.md](./API_CONTRACT.md) line 82 in tandem.** | +| Allowed image MIME types | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | Bedrock vision-supported formats | +| Allowed file MIME types | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | Useful for code/data context; no executables | +| Max URL fetch size | 10 MB | Same per-attachment limit for fetched content | +| URL fetch timeout | 10 seconds | Prevent SSRF-style long-poll attacks | +| URL scheme | `https` only | No `http`, `file`, `ftp`, or custom schemes | + +**Payload limit note:** The binding constraint for the inline path is **not** the API Gateway 10 MB body limit — it is the **Lambda synchronous invocation payload limit of 6 MB**. API Gateway REST API forwards the request body to Lambda as part of the invocation payload, which includes the body plus API Gateway metadata. The limits above are set conservatively to ensure the full payload stays under 6 MB. + +## Upload flows + +The design provides three upload paths, unified by a common screening + S3 storage backend. The presigned upload is the primary path for files > 500 KB; inline base64 is a convenience for small attachments; URL fetch handles remote resources. + +### Primary path: Presigned URL upload + +For attachments > 500 KB (or any attachment where the client prefers not to base64-encode), the client requests a presigned POST policy and uploads directly to S3. This bypasses the Lambda payload limit entirely. S3 enforces the `content-length-range` condition server-side, rejecting oversized uploads before storing them. + +```mermaid +sequenceDiagram + participant C as Client + participant GW as API Gateway + participant H as Create-Task Lambda + participant S3 as Attachments Bucket + participant SCR as Screening Pipeline + participant DB as TaskTable + + C->>GW: POST /v1/tasks { attachments: [{ type: 'image', filename: 'screenshot.png', content_type: 'image/png', expected_size_bytes: 2500000 }] } + GW->>H: Forward (metadata only, no binary) + H->>H: Validate metadata + declared sizes against limits + H->>S3: Generate presigned POST policy (10 min expiry, content-length-range enforced) + H->>DB: Write TaskRecord (status: PENDING_UPLOADS, attachments with screening: pending) + H-->>C: 202 Accepted { task_id, upload_urls: [...], upload_fields: [...], task_expires_at } + + C->>S3: POST multipart/form-data (binary content, policy enforces size) + S3-->>C: 200 OK + + C->>GW: POST /v1/tasks/{task_id}/confirm-uploads + GW->>H: Forward + H->>H: Check status == PENDING_UPLOADS (short-circuit if already SUBMITTED) + H->>S3: HeadObject per attachment (verify uploads exist, with retry for 404) + loop Each attachment (parallel, bounded concurrency 3) + H->>S3: GetObject (stream content for screening) + H->>SCR: Screen content (with retry: 3 attempts, exponential backoff) + SCR-->>H: Pass / Blocked + alt Screening blocked + H->>S3: DeleteObjects (all uploads for this task) + H->>DB: Update status → FAILED + H-->>C: 400 ATTACHMENT_BLOCKED { attachment_id, reason } + end + end + H->>H: Strip EXIF from images, re-upload cleaned version + H->>DB: Update TaskRecord (status: SUBMITTED, screening: passed) with condition: status = PENDING_UPLOADS + H->>H: Async-invoke Orchestrator + H-->>C: 200 OK { task_id, status: SUBMITTED } +``` + +**Key details:** + +- **New task status: `PENDING_UPLOADS`.** Tasks with presigned-upload attachments are created in `PENDING_UPLOADS` status. They do not enter the orchestration pipeline until uploads are confirmed. See [State machine changes](#state-machine-changes) for the full transition table. +- **Declared sizes for early budget validation.** Clients must include `expected_size_bytes` for presigned attachments. The create-task handler validates total declared size against the 50 MB limit immediately, preventing the user from uploading 100 MB only to be rejected at confirm-uploads. +- **Presigned POST policy generation.** S3 presigned POST policies (via `createPresignedPost` from `@aws-sdk/s3-presigned-post`) support **`content-length-range` conditions**, enforcing the 10 MB per-attachment limit at the S3 layer. This is preferred over presigned PUT URLs, which cannot enforce `Content-Length`. The presigned POST also fixes `Content-Type` and the S3 key. The 10-minute policy expiry bounds the upload window. Clients must use `multipart/form-data` POST (not PUT) to upload. +- **Confirm-uploads endpoint.** `POST /v1/tasks/{task_id}/confirm-uploads` triggers screening and transitions to `SUBMITTED`. See [Confirm-uploads concurrency](#confirm-uploads-concurrency) for the race-condition handling. +- **Parallel screening with bounded concurrency.** Attachments are screened in parallel (max 3 concurrent) to reduce wall time. Sequential screening of 10 large images can exceed the Lambda timeout; parallel processing with concurrency 3 keeps worst-case under the timeout budget (see [CDK construct changes](#cdk-construct-changes)). +- **Atomic failure.** If any attachment fails screening, the entire task fails. All uploaded objects are deleted. The user gets a clear error identifying which attachment was blocked and why. +- **HeadObject retry for incomplete uploads.** When `confirm-uploads` is called immediately after the client receives 200 from S3, there is a brief S3 eventual-consistency window where `HeadObject` may return 404. The handler retries `HeadObject` up to 3 times with 1-second delays before concluding the object is truly missing. After retries exhaust, the handler returns `400 ATTACHMENT_SIZE_MISMATCH` with message: "Upload for `{filename}` not found. Ensure the upload completed successfully before calling confirm-uploads." This differentiates "upload in progress" from "upload never happened" (the latter should not retry indefinitely). + +### Confirm-uploads concurrency + +Two concurrent `confirm-uploads` calls can race. The design prevents corruption through three mechanisms: + +1. **Early short-circuit:** The handler reads the task status first. If status is not `PENDING_UPLOADS`, return the current task status immediately (idempotent success for `SUBMITTED`, error for `FAILED`/`CANCELLED`). This avoids redundant screening work. +2. **Conditional DynamoDB write:** The final status transition uses `ConditionExpression: 'status = :pending_uploads'`. Only one caller wins the write. The loser gets `ConditionalCheckFailedException`, which the handler maps to an idempotent success response (re-read the task and return current status). +3. **Safe cleanup via conditional write result:** On screening failure, the handler attempts a conditional DynamoDB write to `FAILED` (`ConditionExpression: 'status = PENDING_UPLOADS'`). If the write **succeeds**, this caller owns the failure — proceed with S3 cleanup. If the write **fails** (`ConditionalCheckFailedException`), another caller already transitioned to `SUBMITTED` — skip cleanup entirely (those objects are needed by the running agent). The conditional write result is the authoritative signal, eliminating the TOCTOU window that would exist with a separate "read status then delete" approach. + +```typescript +// In confirm-uploads handler: +try { + await dynamoClient.send(new UpdateCommand({ + TableName: TASK_TABLE, + Key: { task_id }, + UpdateExpression: 'SET #s = :submitted, ...', + ConditionExpression: '#s = :pending_uploads', + ... + })); +} catch (err) { + if (err instanceof ConditionalCheckFailedException) { + // Another caller already transitioned — return current state (idempotent) + const current = await getTask(task_id); + return toTaskDetailResponse(current); + } + throw err; +} +``` + +### Idempotency key interaction with PENDING_UPLOADS + +When a client retries a task creation with the same idempotency key, the existing task may be in `PENDING_UPLOADS` with expired presigned POST policies. The idempotency check is special-cased: + +| Existing task status | Presigned URLs expired? | Behaviour | +|---|---|---| +| `PENDING_UPLOADS` | No (< 10 min old) | Return existing response with original upload instructions (true idempotency) | +| `PENDING_UPLOADS` | Yes (> 10 min old) | Generate new attachment IDs + new S3 keys, generate new presigned POST policies, update AttachmentRecords in DynamoDB (conditional write: `ConditionExpression: 'status = PENDING_UPLOADS'` — prevents clobbering a concurrent `confirm-uploads` that transitioned to `SUBMITTED`), return updated response. If the conditional write fails, re-read the task and return current status. | +| `SUBMITTED` or later | N/A | Return existing task (standard idempotency) | +| `FAILED` / `CANCELLED` | N/A | Return existing task (standard idempotency) | + +This prevents the deadlock where a client crash leaves an unreachable `PENDING_UPLOADS` task blocking retries for 30 minutes. + +**Presigned POST policy expiry and clock skew:** The 10-minute policy expiry uses the server's clock (AWS SigV4 signing time). Clients with clock skew > 5 minutes may find policies expire earlier than expected. The 10-minute window provides ~5 minutes of effective skew tolerance. Clients that consistently fail uploads should check their system clock (AWS SDK requests also fail with > 15 minutes skew). The CLI should log the server's `Date` header on upload failure to help users diagnose clock issues. + +**Why new S3 keys on retry:** Regenerating presigned POST policies for the same S3 keys creates a collision risk if the first client instance is still alive (e.g., network partition, not a crash). Both instances would upload to the same key, with one overwriting the other. Using new attachment IDs (and therefore new S3 key paths) ensures concurrent client instances cannot interfere. The orphaned objects from the original attempt are cleaned up by the auto-cancel rule or the 90-day lifecycle. + +### Convenience path: Inline base64 (small attachments) + +For attachments <= 500 KB decoded, clients can include base64-encoded content directly in the `POST /v1/tasks` body. This avoids the two-phase round trip for small files like cropped screenshots and short logs. + +```mermaid +sequenceDiagram + participant C as Client + participant GW as API Gateway + participant H as Create-Task Lambda + participant SCR as Screening Pipeline + participant S3 as Attachments Bucket + participant DB as TaskTable + + C->>GW: POST /v1/tasks { attachments: [{ type: 'image', data: 'base64...', content_type: 'image/png' }] } + GW->>H: Forward (body < 6 MB Lambda payload limit) + H->>H: Validate: decode base64, check size <= 500 KB, magic bytes, total inline <= 3 MB + loop Each inline attachment + H->>SCR: Screen content (with retry: 3 attempts, exponential backoff) + SCR-->>H: Pass / Blocked + alt Screening blocked + H->>H: Cleanup already-uploaded S3 objects + H-->>C: 400 ATTACHMENT_BLOCKED + end + H->>H: Strip EXIF (images) / sanitize; on sharp failure → ATTACHMENT_INVALID_CONTENT + H->>S3: PutObject (cleaned content) + H->>H: Build AttachmentRecord + end + H->>DB: Write TaskRecord (status: SUBMITTED, with attachment metadata) + H->>H: Async-invoke Orchestrator + H-->>C: 201 Created +``` + +**Limit rationale:** The binding constraint is the **6 MB Lambda synchronous invocation payload limit** (not the 10 MB API Gateway body limit). The API Gateway forwards the full request body to Lambda as part of the invocation payload. At 500 KB decoded per inline attachment (~667 KB base64), with 3 MB total decoded (~4 MB base64), plus JSON overhead and API Gateway metadata, the total stays safely under 6 MB. In practice, most inline attachments are small screenshots (100-300 KB) where one or two are included alongside a task description. + +### URL fetch (deferred download) + +For `type: 'url'` attachments, the content is fetched during context hydration (not at submission time), because: + +1. The URL may require the repo's GitHub token to access (e.g., private repo assets). +2. Fetching at submission time blocks the API response on external network calls. +3. The content should be fresh at agent execution time, not stale from hours-ago submission. + +```mermaid +sequenceDiagram + participant H as Create-Task Lambda + participant O as Orchestrator (Hydration) + participant SCR as Screening Pipeline + participant S3 as Attachments Bucket + + H->>H: Validate URL format, scheme (HTTPS only) + Note over H: SSRF pre-check is async — runs in a separate step after sync validation + H->>H: Store AttachmentRecord (s3_key: absent, screening: pending) + Note over O: During context hydration + O->>O: SSRF check (DNS resolve → reject private IPs → connect to resolved IP) + O->>O: Fetch URL content (with timeout + size limit) + O->>SCR: Screen fetched content (with retry) + SCR-->>O: Pass / Blocked + alt Screening passed + O->>O: Strip EXIF (images) / sanitize; on sharp failure → fail task + O->>S3: PutObject + O->>O: Update AttachmentRecord + else Screening blocked or fetch failed + O->>O: Throw AttachmentResolutionError → fails the task + end +``` + +**Failure semantics:** A blocked or unfetchable URL attachment **fails the task** — same as inline attachments. The user chose the URL; they expect it to work. The error message identifies the problematic URL and reason (blocked content, fetch timeout, DNS failure, SSRF violation). The user can re-submit without the problematic URL. + +**User notification for async failures:** URL fetch failures occur during hydration (minutes after submission). The user has already received a `201 Created` response. The failure surfaces through: +- Task status transitions to `FAILED` with `error_message` identifying the attachment and reason. +- A `task_failed` event is written to `TaskEventsTable`. +- Notifications (if configured): Slack reply, Linear comment, or webhook callback. +- The CLI `bgagent submit` with URL attachments should poll for the `HYDRATING → RUNNING` transition before returning, surfacing early failures inline. + +**SSRF protections** (applied at fetch-time in the orchestrator Lambda): + +1. **URL scheme:** Only `https` is allowed. No `http`, `file`, `ftp`, or custom schemes. Validated synchronously at submission time. +2. **DNS resolution pinning (prevents DNS rebinding):** The fetch implementation must: (a) resolve DNS manually (e.g., via `dns.resolve4`/`dns.resolve6`), (b) validate the resolved IP against private ranges (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `169.254.0.0/16`, `127.0.0.0/8`, `::1`, `fd00::/8`), (c) connect to the validated IP directly using `undici`'s `connect` option or equivalent — **pinning** the connection to the resolved IP and preventing the HTTP library from doing a second DNS lookup that could return a different (malicious) IP. This technique is called "DNS resolution pinning" and it mitigates the DNS rebinding attack, where an attacker's DNS server returns a public IP on the first lookup (passing validation) then a private IP on the second lookup (reaching internal services). +3. **Redirect limits:** Follow at most 2 redirects. Re-validate the target URL and resolved IP after each redirect. +4. **Timeout and size:** 10-second timeout, 10 MB max response body. Stream the response and abort if the size limit is exceeded. +5. **No auth headers to non-GitHub URLs:** Only GitHub URLs (matching the repo's installation) receive the GitHub token. All other URLs are fetched without credentials. + +**URL fetch policy (open fetch with SSRF protection):** The default policy allows fetching from any public HTTPS URL that passes SSRF validation. This supports real-world use cases (Figma exports, Google Docs, Confluence, etc.) without requiring per-deployment configuration. Repos can optionally restrict to a URL allowlist via blueprint config: + +```json +{ + "attachments": { + "url_allowlist": ["raw.githubusercontent.com", "*.figma.com", "docs.google.com"], + "url_denylist": ["*.internal.company.com"] + } +} +``` + +When `url_allowlist` is set, only matching URLs are allowed. When absent (default), any public HTTPS URL passing SSRF checks is permitted. `url_denylist` is always evaluated regardless of allowlist. + +## Security screening pipeline + +Every attachment passes through a type-specific screening pipeline before its binary content reaches durable S3 storage or the agent. The pipeline is fail-closed: if any screening step is unavailable after retries, the attachment (and the task) is rejected. + +### Retry strategy + +All Bedrock `ApplyGuardrailCommand` calls use exponential backoff before failing closed: + +- **Max retries:** 3 +- **Backoff:** 200ms, 400ms, 800ms +- **Retryable errors:** HTTP 429 (throttling), HTTP 5xx (transient service errors) +- **Non-retryable errors:** HTTP 4xx (except 429), validation errors, content policy violations (`INTERVENED`) + +This prevents a single transient Bedrock hiccup from failing an entire task after the user waited for 10 attachments to upload. + +### Image screening + +```mermaid +flowchart TD + I[Image attachment] --> MB[Magic bytes: verify image signature] + MB -->|Invalid| R[REJECTED: not a valid image] + MB -->|Valid| V[Validate: decodable, dimensions ≤ 8192x8192] + V --> G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] + G -->|INTERVENED| B[BLOCKED: content policy violation] + G -->|NONE| M[EXIF strip + re-encode via sharp] + M -->|sharp error| SE[REJECTED: ATTACHMENT_INVALID_CONTENT] + M -->|Success| P[PASSED] + G -->|Error after retries| F[FAIL CLOSED: 503] +``` + +**Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the image processing pipeline. + +**Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks. **Prerequisite:** Verify that the installed version of `@aws-sdk/client-bedrock-runtime` in `cdk/package.json` supports image content blocks in `ApplyGuardrail`. If it does not, a dependency upgrade is required before Phase 1. + +**Format limitation:** The Bedrock `GuardrailImageBlock` API **only supports `png` and `jpeg` formats**. GIF and WebP images must be converted to PNG via `sharp` before screening. This conversion happens before the `ApplyGuardrailCommand` call (not after — the screened content must be the same content that reaches the agent): + +```typescript +// Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) +let screeningBuffer: Buffer; +let screeningFormat: 'png' | 'jpeg'; + +if (contentType === 'image/jpeg') { + screeningBuffer = imageBuffer; + screeningFormat = 'jpeg'; +} else if (['image/gif', 'image/webp'].includes(contentType)) { + // GIF/WebP → PNG. For animated GIFs, extract first frame only (prevents OOM). + screeningBuffer = await sharp(imageBuffer, { animated: false }).png().toBuffer(); + screeningFormat = 'png'; + + // Post-conversion size check: PNG expansion of compressed GIF/WebP can exceed 10 MB. + // Fail with a clear error rather than letting a downstream size check reject it opaquely. + if (screeningBuffer.length > MAX_ATTACHMENT_SIZE_BYTES) { + throw new AttachmentResolutionError( + `Image "${filename}" is ${contentType} and its PNG conversion for screening ` + + `exceeds the ${MAX_ATTACHMENT_SIZE_BYTES / (1024 * 1024)} MB limit ` + + `(${(screeningBuffer.length / (1024 * 1024)).toFixed(1)} MB after conversion). ` + + `Please convert to JPEG or reduce image dimensions before uploading.` + ); + } +} else { + // PNG: use as-is + screeningBuffer = imageBuffer; + screeningFormat = 'png'; +} + +const result = await retryWithBackoff(() => + bedrockClient.send(new ApplyGuardrailCommand({ + guardrailIdentifier: GUARDRAIL_ID, + guardrailVersion: GUARDRAIL_VERSION, + source: 'INPUT', + content: [{ + image: { + format: screeningFormat, + source: { bytes: screeningBuffer }, + }, + }], + })), + { maxRetries: 3, baseDelayMs: 200, retryableErrors: [429, 500, 502, 503] }, +); +``` + +The GIF/WebP → PNG conversion uses `{ animated: false }` to extract only the first frame from animated GIFs, preventing unbounded memory usage. The post-conversion size check catches cases where a highly-compressed GIF/WebP expands beyond 10 MB as PNG — the user gets a clear error with remediation (convert to JPEG or reduce dimensions). The conversion error path (`ATTACHMENT_INVALID_CONTENT`) is shared with the EXIF stripping pipeline. + +**EXIF stripping + re-encoding:** After screening passes, the image is processed through `sharp`: +1. Strip all EXIF/IPTC/XMP metadata (GPS coordinates, device info, timestamps — prevents PII leakage). +2. Re-encode the image in the same format. This strips any non-image data that may have been appended to the file (steganography payloads, polyglot trailing data). +3. The re-encoded image is what gets stored in S3 and delivered to the agent. + +**sharp failure handling:** If `sharp` cannot process an image (corrupt image that passes magic bytes check, OOM on large image, library bug), the attachment is **rejected** with `ATTACHMENT_INVALID_CONTENT` and message: "Image could not be processed for security sanitization. Please re-export the image in a standard format and try again." This preserves the security guarantee — no un-sanitized images reach the agent. Fail-closed, not fail-open. + +### File screening + +```mermaid +flowchart TD + F[File attachment] --> MB[Magic bytes: verify against claimed MIME type] + MB -->|Mismatch| R[REJECTED: content does not match declared type] + MB -->|Match| T{Text-based?} + T -->|Yes| TG[Bedrock Guardrail: text content screening, retries] + T -->|No - PDF| PDF[Extract text via pdf-parse, max 50 pages / 1 MB output] + PDF --> TG + TG -->|INTERVENED| B[BLOCKED] + TG -->|NONE| P[PASSED] + TG -->|Error after retries| FC[FAIL CLOSED: 503] +``` + +**Magic bytes validation:** Don't trust `content_type` from the client. Validate the first bytes of the content against known signatures for allowed types: + +| MIME type | Expected magic bytes | +|---|---| +| `application/pdf` | `%PDF-` | +| `application/json` | Starts with `{`, `[`, or UTF-8 BOM + `{`/`[` | +| `text/*` | Valid UTF-8, no null bytes in first 8 KB | +| `image/png` | `\x89PNG\r\n\x1a\n` | +| `image/jpeg` | `\xFF\xD8\xFF` | +| `image/gif` | `GIF87a` or `GIF89a` | +| `image/webp` | `RIFF....WEBP` | + +A file claiming to be `text/plain` but starting with `MZ` (PE executable) or `PK` (ZIP) is rejected immediately. + +**Text content screening:** For text-based files (plain text, CSV, Markdown, JSON), the full content is screened through the same Bedrock Guardrail used for task descriptions. For PDFs, text is extracted first (using `pdf-parse`, capped at 50 pages and 1 MB extracted text output to prevent decompression bombs) and then screened. + +**PDF extraction failure handling:** `pdf-parse` is wrapped in a try-catch with a 15-second timeout. Corrupt PDFs, deeply nested objects, or excessive embedded fonts can cause OOM before the page limit takes effect. On any `pdf-parse` failure (exception, timeout, or OOM), the attachment is **rejected** with `ATTACHMENT_INVALID_CONTENT` and message: "PDF could not be processed. It may be corrupt or use unsupported features. Try exporting to a simpler PDF format." Consider running `pdf-parse` in a child process with a memory limit to prevent Lambda OOM when the confirm-uploads Lambda is processing multiple PDFs concurrently. + +**No executable content:** The MIME allowlist explicitly excludes executables, archives, and scripts. This is a hard boundary — there is no override. If a user needs to share a shell script, they should paste it as text in the task description or commit it to the repo. + +### Screening result caching + +Screening results are not cached. Each attachment is screened exactly once, at upload/fetch time. The `screening` field in `AttachmentRecord` records the outcome for audit purposes. Re-screening is not needed because: + +- Attachments are immutable once stored (no update API). +- Guardrail policies may change, but retroactive re-screening of existing attachments is a separate concern (batch job, not inline). + +## Storage: Attachments S3 bucket + +A new CDK construct, `AttachmentsBucket`, following the same pattern as `TraceArtifactsBucket`: + +```typescript +// cdk/src/constructs/attachments-bucket.ts + +/** Props interface mirrors TraceArtifactsBucketProps for consistency. */ +export interface AttachmentsBucketProps { + readonly removalPolicy?: RemovalPolicy; // Default: DESTROY (dev-friendly; override for prod) + readonly autoDeleteObjects?: boolean; // Default: true (matches removalPolicy: DESTROY) +} + +export class AttachmentsBucket extends Construct { + public readonly bucket: s3.Bucket; + + constructor(scope: Construct, id: string, props: AttachmentsBucketProps = {}) { + super(scope, id); + + this.bucket = new s3.Bucket(this, 'Bucket', { + encryption: s3.BucketEncryption.S3_MANAGED, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + versioned: true, // Required: pins object versions at screening time to prevent TOCTOU + lifecycleRules: [ + { + expiration: Duration.days(ATTACHMENT_TTL_DAYS), // 90 days — matches task retention + noncurrentVersionExpiration: Duration.days(7), // Noncurrent versions kept 7 days (see rationale below) + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, + autoDeleteObjects: props.autoDeleteObjects ?? true, + }); + } +} +``` + +**S3 versioning (TOCTOU prevention):** The bucket has versioning enabled. This prevents a class of attack where a client uploads benign content (passes screening), then replaces the S3 object with malicious content before the agent downloads it. The mitigation flow: + +1. At `confirm-uploads` time, `HeadObject` records the `VersionId`. +2. `GetObject` for screening specifies `VersionId` — screens exactly the uploaded version. +3. After screening passes, the `VersionId` is stored in `AttachmentRecord.s3_version_id`. +4. The `AgentAttachmentPayload` includes `s3_version_id`. The agent downloads with `VersionId` pinned. +5. Even if the presigned URL is still valid and the client uploads a second version, the agent always downloads the screened version. +6. `noncurrentVersionExpiration: 7 days` — old versions are cleaned up after a safe window. The 7-day retention ensures that even the longest-running tasks (max_turns=500 with slow models, potentially 24-48 hours) can always access the screened version. After 7 days, any task still referencing a noncurrent version is either completed or stuck in a terminal state. + +**Lifecycle:** 90 days, matching the task record TTL (confirmed: `TASK_RETENTION_DAYS` defaults to 90 in `task-api.ts`). When a task expires from DynamoDB, its attachments expire from S3 around the same time. No cross-resource cleanup needed. + +**IAM grants:** + +| Principal | Grant | Purpose | +|---|---|---| +| Create-task Lambda | `grantPut`, `grantDelete` | Upload inline attachments; delete on partial failure cleanup | +| Confirm-uploads Lambda | `grantRead`, `grantPut`, `grantDelete` | Read for screening; re-upload cleaned images; delete blocked content | +| Orchestrator Lambda | `grantReadWrite` | Fetch URL attachments during hydration; write screened content | +| Agent runtime (AgentCore / ECS) | `grantRead` | Download attachments into workspace via IAM role | + +**Presigned POST policy generation:** + +```typescript +import { createPresignedPost } from '@aws-sdk/s3-presigned-post'; + +const { url, fields } = await createPresignedPost(s3Client, { + Bucket: ATTACHMENTS_BUCKET, + Key: s3Key, + Conditions: [ + ['content-length-range', 1, MAX_ATTACHMENT_SIZE_BYTES], // 1 byte to 10 MB + ['eq', '$Content-Type', declaredMimeType], + ], + Fields: { + 'Content-Type': declaredMimeType, + }, + Expires: 600, // 10 minutes +}); +// Presigned POST policies enforce content-length-range at the S3 layer. +// S3 rejects uploads outside the declared range BEFORE storing the object. +// The Content-Type is fixed by the policy — mismatched uploads fail. +// Clients must POST with multipart/form-data (not PUT). +``` + +**Why presigned POST over presigned PUT:** Presigned PUT URLs (`getSignedUrl` with `PutObjectCommand`) **cannot enforce `Content-Length` conditions** — the only way to limit upload size with PUT is a bucket resource policy using `s3:content-length-range`, but this condition key is **not documented for `s3:PutObject` actions in bucket policies** and its behavior is unreliable. Presigned POST policies, in contrast, have `content-length-range` as a [documented, first-class condition](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html#sigv4-PolicyConditions) that S3 enforces server-side before writing the object. This provides reliable, documented size enforcement at the S3 layer. + +**Client upload format:** Clients upload via `POST` with `multipart/form-data`, including the policy fields returned in the creation response. The CLI and webhook documentation include code examples. This is a minor ergonomic tradeoff (form-data POST vs raw PUT) for guaranteed size enforcement. + +**Defense-in-depth:** Even with presigned POST size enforcement, the `confirm-uploads` handler still validates object sizes via `HeadObject` (defense-in-depth against implementation bugs). The auto-cancel cleanup Lambda also deletes ALL objects under a task's `attachments///` prefix (not just confirmed ones), catching objects from tasks that were never confirmed. + +## Orchestrator integration + +### Hydration error handling + +The existing `hydrateContext()` has a broad infrastructure-failure catch block (context-hydration.ts lines 1157-1189) that returns minimal context (task_description only) and sets `fallback_error` — allowing the task to proceed with degraded context. This is acceptable for optional context (e.g., a memory lookup failure), but **not acceptable for attachments**. + +If the user explicitly provided attachments, proceeding without them produces incorrect agent output. Attachment resolution errors must **propagate and fail the task**, not be caught by the generic fallback. + +**Implementation (Phase 1, step 10 — ships before any code throws these errors):** Create an `AttachmentError` base class. All attachment-related errors (`AttachmentResolutionError`, `AttachmentBudgetExceededError`, `AttachmentDownloadError`, `AttachmentIntegrityError`) extend this base class. A single `instanceof AttachmentError` check in the hydration catch block covers all current and future attachment error types, avoiding a growing allowlist. This is wired into the catch block in Phase 1 (alongside the class definitions), not deferred — ensuring no deployment window exists where typed errors are thrown but not re-thrown: + +```typescript +// Error class hierarchy: +export class AttachmentError extends Error { } +export class AttachmentResolutionError extends AttachmentError { } +export class AttachmentBudgetExceededError extends AttachmentError { } + +// In hydrateContext() catch block: +} catch (err) { + if ( + err instanceof GuardrailScreeningError || + err instanceof AttachmentError || // NEW: covers all attachment errors (budget, resolution, integrity) + err instanceof TypeError || + err instanceof RangeError || + err instanceof ReferenceError + ) { + throw err; // Do not swallow — these must fail the task + } + // Only non-critical infrastructure errors fall through to the fallback path + ... +} +``` + +Using a base class instead of individual `instanceof` checks prevents the bug where a new error type (e.g., `AttachmentBudgetExceededError`) is added to `resolveAttachments()` but not to the re-throw list — causing it to be silently swallowed by the infrastructure fallback. + +### Context hydration changes + +The `hydrateContext()` function gains an attachment resolution step that runs in parallel with issue/PR/memory fetching: + +```typescript +// In hydrateContext(), added to the parallel fetch block: +const resolvedAttachments = await resolveAttachments( + task.attachments, // AttachmentRecord[] from TaskRecord + attachmentsBucket, + githubToken, // For URL fetches requiring auth + bedrockClient, + guardrailConfig, +); +``` + +`resolveAttachments()` handles: + +1. **Inline/presigned attachments (already screened, already in S3):** Validate S3 key exists, compute `s3_uri` for the agent. +2. **URL attachments (not yet fetched):** Fetch (with SSRF protections), screen (with retry), sanitize (EXIF strip / re-encode), upload to S3, compute `s3_uri`. On any failure, throw `AttachmentResolutionError` which fails the task. +3. **Token budget accounting:** Estimate token cost of image attachments and deduct from the available prompt budget (see [Token budget](#token-budget-accounting)). + +### Payload changes + +The orchestrator payload to the agent gains a top-level `attachments` field using the named `AgentAttachmentPayload` interface: + +```typescript +const payload = { + // ... existing fields ... + attachments: resolvedAttachments.map(a => ({ + attachment_id: a.attachment_id, + type: a.type, + content_type: a.content_type, + filename: a.filename, + s3_uri: a.s3_uri, + s3_version_id: a.s3_version_id, + size_bytes: a.size_bytes, + source_url: a.source_url, + token_estimate: a.token_estimate, + checksum_sha256: a.checksum_sha256, + } satisfies AgentAttachmentPayload)), +}; +``` + +**Top-level placement rationale:** Attachments are placed at the top level of the agent payload (alongside `repo_url`, `task_type`, etc.), **not** inside `HydratedContext`. This avoids: + +- A `HydratedContext` version bump and the deployment ordering constraint it creates (agent image must deploy before orchestrator). +- Pydantic `extra="forbid"` rejection — `HydratedContext` uses `extra="forbid"` (models.py line 68), so adding a field there without updating the agent would crash it. The `TaskConfig` model (models.py line 94) uses only `validate_assignment=True` without `extra="forbid"`, so Pydantic v2's default behaviour (`extra="ignore"`) silently discards unrecognized top-level fields on old agents. +- Conflating raw inputs (attachments) with assembled prompt context (issue body, PR comments, memory). + +### Agent capability check + +While old agents silently ignore the `attachments` top-level field (due to Pydantic `extra="ignore"` default), this produces a bad user experience: the user's attachments have no effect, with no error or warning. To prevent this during incremental rollout: + +**The orchestrator checks the agent's deployment version before including attachments in the payload.** The mechanism: + +1. The agent container image is tagged with a version identifier (already tracked via `prompt_version` in the payload). +2. The orchestrator checks the `prompt_version` (or a new `agent_capabilities` field in the blueprint config) to determine attachment support. +3. If the agent does not support attachments AND the task has attachments, the orchestrator fails the task with: `"Agent runtime version does not support attachments. A deployment is required to enable this feature for repository {repo}."`. +4. If the task has no attachments, the orchestrator proceeds normally regardless of agent version. + +This ensures users never silently lose their attachments during the rollout window. + +### No HydratedContext version bump needed + +Because attachments live at the top level of the payload (not inside `hydrated_context`), no version bump is required. The `HydratedContext` Pydantic model with `extra="forbid"` and `version: 1` remains unchanged. This eliminates the deployment ordering constraint. + +### Token budget accounting + +Images consume tokens when sent as multimodal content blocks. The system's `USER_PROMPT_TOKEN_BUDGET` (default 100K tokens, configured via environment variable) must account for image token costs. + +**Image token estimation:** + +Claude resizes images before tokenizing. The estimation must apply the same resizing rules. Per Anthropic's documentation (as of May 2025): + +1. If either dimension exceeds 1568px, scale down proportionally to fit 1568px on the longest side. +2. Pad each dimension up to the next multiple of 28 pixels. +3. Apply: `ceil(resized_width * resized_height / 750)` tokens, capped at 1568 tokens. +4. Apply a **1.2x safety margin** to account for padding imprecision and API changes. + +**Note:** The documentation describes a single resize step (fit 1568px longest side) and a hard token cap at 1568. An earlier version of this design included a separate 1.15 megapixel step; that has been removed to match the documented behaviour. If targeting Claude Opus 4.7 (which supports 2576px / 4784 tokens max), the constants should be made configurable. + +```typescript +const MAX_IMAGE_SIDE = 1568; // Standard models; Opus 4.7 uses 2576 +const MAX_IMAGE_TOKENS = 1568; // Standard models; Opus 4.7 uses 4784 +const TOKEN_SAFETY_MARGIN = 1.2; +const TILE_SIZE = 28; + +function estimateImageTokens(width: number, height: number): number { + let w = width; + let h = height; + + // Step 1: Scale to fit MAX_IMAGE_SIDE on longest side + const maxSide = Math.max(w, h); + if (maxSide > MAX_IMAGE_SIDE) { + const scale = MAX_IMAGE_SIDE / maxSide; + w = Math.round(w * scale); + h = Math.round(h * scale); + } + + // Step 2: Pad to next multiple of 28 pixels (tile alignment) + w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; + h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; + + // Step 3: Token calculation with safety margin, capped + const rawTokens = Math.min(Math.ceil((w * h) / 750), MAX_IMAGE_TOKENS); + return Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN); +} +``` + +For standard image sizes (after resizing): + +| Original dimensions | Resized to (pre-pad) | Padded to | Estimated tokens (with margin) | +|---|---|---|---| +| 1920x1080 (full screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | +| 3840x2160 (4K screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | +| 800x600 (cropped screenshot) | 800x600 (no resize) | 812x616 | ~800 | +| 4096x4096 (max-size design) | 1568x1568 | 1568x1568 | ~1,882 (capped) | + +**Note:** The 4K screenshot and full HD screenshot produce the **same** token cost after resizing — both scale down to the same dimensions. The token cap at 1568 (+ safety margin) means very large or square images plateau rather than growing linearly. + +**Budget enforcement in attachment resolution:** + +```typescript +async function resolveAttachments(attachments, ...) { + let attachmentTokenBudget = 0; + + for (const att of attachments) { + if (att.type === 'image') { + // getImageDimensions uses sharp.metadata() on the S3 content. + // If dimensions cannot be determined (corrupt image, unsupported format variant), + // throw AttachmentResolutionError — never default to (0,0) or skip the estimate. + let width: number, height: number; + try { + ({ width, height } = await getImageDimensions(att)); + } catch (err) { + throw new AttachmentResolutionError( + `Cannot determine dimensions for image "${att.filename}". ` + + `The image may be corrupt or in an unsupported format variant. ` + + `Re-export the image and try again.`, + { cause: err }, + ); + } + const tokenCost = estimateImageTokens(width, height); + att.token_estimate = tokenCost; + attachmentTokenBudget += tokenCost; + } + } + + // Reserve tokens for attachments; reduce available budget for text context + const availableForText = USER_PROMPT_TOKEN_BUDGET - attachmentTokenBudget; + + if (availableForText < MIN_TEXT_TOKEN_BUDGET) { // MIN_TEXT_TOKEN_BUDGET = 20,000 + throw new AttachmentBudgetExceededError( + `Image attachments require ~${attachmentTokenBudget} tokens, ` + + `leaving insufficient budget for task context (minimum ${MIN_TEXT_TOKEN_BUDGET} required). ` + + `Reduce image count or dimensions.` + ); + } + + return { resolvedAttachments, attachmentTokenBudget, availableForText }; +} +``` + +**Policy:** If image attachments consume more than `USER_PROMPT_TOKEN_BUDGET - MIN_TEXT_TOKEN_BUDGET` tokens (i.e., they would leave fewer than 20K tokens for text context), the task fails with a clear error. The user can reduce image count or downscale images before resubmitting. + +**Token budget vs. payload size:** The token budget above measures **vision tokens** (based on pixel dimensions). This is separate from the **API payload size**, which is affected by base64 encoding overhead (~33% expansion). Image attachments are sent as multimodal content blocks with base64-encoded data, so a 10 MB image becomes ~13.3 MB in the API request. The Anthropic API has its own request size limits (separate from our Lambda payload limits). The `MAX_ATTACHMENT_SIZE_BYTES` (10 MB) is chosen to ensure that even after base64 expansion, individual images stay within the Anthropic API's per-image limits. For multiple large images, the total base64-encoded payload is bounded by the 50 MB total task limit (which produces ~67 MB base64), but in practice the vision token budget is the binding constraint — 10 full-resolution images would consume ~18,820 vision tokens (well within the 100K budget) but produce a very large API payload. The agent should stream images from local files rather than holding all base64 data in memory simultaneously. + +The `availableForText` budget is passed to the existing text trimming logic (`enforceTokenBudget`), which trims issue comments and PR threads to fit within the remaining allocation. + +## Agent consumption + +The agent pipeline (`pipeline.py`) gains an attachment preparation step that runs after repo clone and before the agent session: + +### Step 1: Download attachments from S3 with integrity verification + +The agent runtime has an IAM role with `s3:GetObject` permission on the attachments bucket. It downloads attachments using the AWS SDK — no presigned URLs, no expiry concerns. + +```python +async def prepare_attachments( + attachments: list[AttachmentConfig], + workspace_dir: Path, + s3_client: S3Client, +) -> list[PreparedAttachment]: + """Download attachments from S3 into the workspace with integrity checks.""" + attachments_dir = workspace_dir / ".attachments" + attachments_dir.mkdir(exist_ok=True) + + prepared = [] + for att in attachments: + local_path = attachments_dir / f"{att.attachment_id}_{att.filename}" + bucket, key = parse_s3_uri(att.s3_uri) + try: + # Download the pinned version — prevents TOCTOU between screening and download + await s3_client.download_file( + bucket, key, str(local_path), + ExtraArgs={"VersionId": att.s3_version_id}, + ) + except ClientError as e: + raise AttachmentDownloadError( + f"Failed to download attachment {att.filename}: {e}" + ) from e + + # Verify integrity via SHA-256 checksum (always present — required by factory) + # hexdigest() returns lowercase hex, matching the format enforced by AttachmentConfig validator + actual_hash = hashlib.sha256(local_path.read_bytes()).hexdigest() + if actual_hash != att.checksum_sha256: + raise AttachmentIntegrityError( + f"Checksum mismatch for {att.filename}: " + f"expected {att.checksum_sha256}, got {actual_hash}" + ) + + prepared.append(PreparedAttachment( + attachment_id=att.attachment_id, + type=att.type, + content_type=att.content_type, + filename=att.filename, + local_path=local_path, + token_estimate=att.token_estimate, + )) + return prepared +``` + +**Error handling:** `AttachmentDownloadError` and `AttachmentIntegrityError` are fatal — the task fails with a clear error. The agent does not proceed with missing or corrupted attachments. + +Attachments are downloaded to `.attachments/` in the workspace root. This directory is `.gitignore`d by the agent to prevent accidentally committing binary attachments. + +**No presigned URL expiry problem:** Because the agent downloads via IAM role credentials (which auto-rotate via the instance metadata service or ECS task role), there is no expiry window. Whether the task starts immediately or after a 4-hour approval wait (Change Manifest `AWAITING_APPROVAL` state), the download works identically. + +### Python models + +```python +class AttachmentConfig(BaseModel): + """Attachment descriptor from the orchestrator payload.""" + model_config = ConfigDict(frozen=True, extra="forbid") + + attachment_id: str + type: Literal["image", "file", "url"] + content_type: str + filename: str + s3_uri: str + s3_version_id: str # Pinned S3 object version — prevents TOCTOU + size_bytes: int + source_url: str | None = None + token_estimate: int | None = None + checksum_sha256: str # Required — lowercase hex-encoded SHA-256 (64 chars, e.g., "a1b2c3...") + + @model_validator(mode="after") + def _validate_invariants(self) -> Self: + if self.type == "image" and self.token_estimate is None: + raise ValueError("Image attachments must have token_estimate") + # checksum_sha256 must be lowercase hex, exactly 64 characters + import re + if not re.fullmatch(r"[0-9a-f]{64}", self.checksum_sha256): + raise ValueError( + f"checksum_sha256 must be 64 lowercase hex characters, got: {self.checksum_sha256!r}" + ) + return self + + +class PreparedAttachment(BaseModel): + """Attachment downloaded to the local workspace.""" + model_config = ConfigDict(frozen=True, extra="forbid") + + attachment_id: str + type: Literal["image", "file", "url"] + content_type: str + filename: str + local_path: Path + token_estimate: int | None = None + + @model_validator(mode="after") + def _validate_path_exists(self) -> Self: + if not self.local_path.exists(): + raise ValueError(f"Attachment file not found: {self.local_path}") + return self +``` + +Both models use `frozen=True` and `extra="forbid"`, matching existing patterns in `models.py` (`HydratedContext`, `GitHubIssue`, `MemoryContext`). + +`TaskConfig` gains an optional field: + +```python +class TaskConfig(BaseModel): + # ... existing fields ... + attachments: list[AttachmentConfig] = Field(default_factory=list) +``` + +### Step 2: Inject into agent prompt + +Attachments are referenced in the system prompt and optionally injected as multimodal content blocks in the user message: + +**Image attachments → multimodal content blocks:** + +```python +# In build_user_message(): +content_blocks = [{"type": "text", "text": user_prompt}] + +for att in prepared_attachments: + if att.type == "image": + image_data = att.local_path.read_bytes() + content_blocks.append({ + "type": "image", + "source": { + "type": "base64", + "media_type": att.content_type, + "data": base64.b64encode(image_data).decode(), + }, + }) + +# Append text listing all attachments +attachment_listing = build_attachment_listing(prepared_attachments) +content_blocks[0]["text"] += f"\n\n{attachment_listing}" +``` + +**File attachments → local file references:** + +``` +The following attachments are available in .attachments/: +- screenshot.png (image/png, 245 KB) — included as image content above +- error-log.txt (text/plain, 12 KB) — read with: Read .attachments/01J5X8_error-log.txt +- data.csv (text/csv, 1.2 MB) — read with: Read .attachments/01J5X9_data.csv +``` + +### Agent tool access + +No new tools are needed. The agent uses its existing `Read` tool to access file attachments and sees image attachments directly in the conversation (multimodal content blocks). The `.attachments/` directory is within the workspace and already covered by the agent's filesystem access policy. + +## Channel-specific behaviour + +### CLI (`bgagent submit`) + +New `--attachment` flag (repeatable): + +```bash +# Local files (auto-detects inline vs presigned based on size) +bgagent submit --repo org/app --description "Fix this bug" \ + --attachment screenshot.png \ + --attachment error.log + +# URL reference +bgagent submit --repo org/app --description "Implement this design" \ + --attachment https://figma.com/file/abc123/export.png +``` + +The CLI detects whether the argument is a local file path or URL: +- **Local file <= 500 KB:** Read, base64-encode, detect MIME type from extension/magic bytes, send inline as `{ type: 'image'|'file', data: '...', content_type: '...', filename: '...' }` +- **Local file > 500 KB:** Send metadata only (with `expected_size_bytes`) in task creation, receive presigned POST policy (URL + form fields), upload directly to S3 via multipart form POST, call confirm-uploads. +- **URL:** Send as `{ type: 'url', url: '...' }` + +The CLI enforces size limits client-side (fail fast with a clear error rather than uploading 10 MB only to get a rejection). Progress bars show upload status for large files. + +**URL attachment polling:** When the task includes URL attachments (fetched asynchronously during hydration), the CLI polls for the `HYDRATING → RUNNING` transition before returning. If the task fails during hydration (attachment fetch/screening failure), the error is surfaced inline to the user. + +### Webhook + +Same `attachments` array in the JSON body. For attachments > 500 KB, webhook callers must use the presigned upload flow (same two-phase pattern as CLI). The webhook documentation will include code examples in Python and Node.js. + +### Slack + +When a user mentions `@Shoof` in a message that contains file uploads or image attachments: + +1. The `slack-events.ts` handler extracts `event.files[]` from the Slack event. +2. For each file, it calls the Slack API (`files.info` or uses the `url_private_download`) to get the file content. +3. Files are validated against size limits (10 MB per file). Oversized files are rejected with a Slack reply: "Attachment `{filename}` is too large (max 10 MB). Please reduce the file size or link to it instead." +4. Files are uploaded to S3 directly (the Slack handler Lambda has `grantPut`), screened, and converted to `AttachmentRecord` entries. +5. The records are passed to `createTaskCore()`. + +**Slack error surface — atomic failure (consistent with design principle 3):** If any attachment fails validation, screening, or upload, the **entire task is rejected** — no partial attachment submission. This matches the behaviour of all other channels (CLI, webhook, API). The Slack reply lists all failures so the user can fix them in one re-submission: + +| Failure | Slack reply | +|---|---| +| File too large | "Task not created. `{filename}` is too large ({size} MB, max 10 MB). Please reduce the file size or remove it and try again." | +| Screening blocked | "Task not created. `{filename}` was blocked by content screening ({categories}). Please remove this file and try again." | +| Unsupported MIME type | "Task not created. `{filename}` has unsupported type `{mime}`. Supported: images (png, jpeg, gif, webp) and text files (txt, csv, json, md, pdf, log)." | +| S3 upload failure | "Task not created. Failed to process `{filename}`. Please try again." | +| Multiple failures | "Task not created. 2 attachment errors: `{file1}` (blocked by content screening), `{file2}` (too large, 15 MB > 10 MB limit). Fix or remove these files and try again." | + +**Note:** Slack files bypass the inline base64 path (they go directly from Slack's CDN to our S3 via the Lambda). This avoids the 500 KB inline limit. + +### Linear + +When a Linear issue triggers task creation and the issue body contains embedded images: + +1. The `linear-webhook-processor.ts` extracts image URLs from the issue body markdown (pattern: `![...](url)`). +2. Each image URL becomes `{ type: 'url', url: '...' }` in the attachments array. +3. Images are fetched and screened during context hydration, following the URL fetch flow. + +Linear issue attachments (non-inline files) are not supported in v1, as the Linear API requires separate API calls to list attachments per issue. This can be added later. + +## Validation changes + +Validation is split into two steps to preserve the existing codebase pattern where validation functions are synchronous pure functions: + +**Step 1 — Synchronous validation** (in `validation.ts`): + +```typescript +const MAX_ATTACHMENTS_PER_TASK = 10; +const MAX_INLINE_ATTACHMENT_SIZE_BYTES = 500 * 1024; // 500 KB +const MAX_TOTAL_INLINE_SIZE_BYTES = 3 * 1024 * 1024; // 3 MB +const MAX_ATTACHMENT_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB +const MAX_TOTAL_ATTACHMENT_SIZE_BYTES = 50 * 1024 * 1024; // 50 MB +const MAX_TASK_DESCRIPTION_LENGTH = 10_000; // Increased from 2,000 + +export function validateAttachments( + attachments: unknown[] | undefined, +): { valid: true; parsed: ValidatedAttachment[] } | { valid: false; error: string } { + if (!attachments) return { valid: true, parsed: [] }; + if (!Array.isArray(attachments)) return { valid: false, error: 'attachments must be an array' }; + if (attachments.length > MAX_ATTACHMENTS_PER_TASK) { + return { valid: false, error: `Maximum ${MAX_ATTACHMENTS_PER_TASK} attachments per task` }; + } + + let totalInlineSize = 0; + let totalDeclaredSize = 0; + const parsed: ValidatedAttachment[] = []; + + for (const [i, att] of attachments.entries()) { + // Type validation + if (!att.type || !['image', 'file', 'url'].includes(att.type)) { + return { valid: false, error: `attachments[${i}].type must be 'image', 'file', or 'url'` }; + } + + // Mutual exclusivity: data vs url + if (att.type === 'url') { + if (!att.url) return { valid: false, error: `attachments[${i}]: url required for type 'url'` }; + if (att.data) return { valid: false, error: `attachments[${i}]: data not allowed for type 'url'` }; + if (!isValidHttpsUrl(att.url)) return { valid: false, error: `attachments[${i}]: must be a valid HTTPS URL` }; + // NOTE: SSRF DNS check is async — runs in step 2, not here + } else { + if (att.data && att.url) { + return { valid: false, error: `attachments[${i}]: provide data or url, not both` }; + } + } + + // Decode inline data (hoisted to loop scope — used by size check, magic bytes, MIME detection, and variant construction) + let decoded: Buffer | undefined; + + // Size validation (for inline data) + if (att.data) { + decoded = Buffer.from(att.data, 'base64'); + if (decoded.length > MAX_INLINE_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `attachments[${i}]: inline data exceeds 500 KB limit. Use presigned upload for larger files.` }; + } + // Magic bytes validation + if (!validateMagicBytes(decoded, att.content_type ?? att.type)) { + return { valid: false, error: `attachments[${i}]: content does not match declared type` }; + } + totalInlineSize += decoded.length; + } + + // Declared size validation (for presigned uploads) + if (!att.data && !att.url && att.type !== 'url') { + // Presigned upload — expected_size_bytes required for early budget check + if (typeof att.expected_size_bytes !== 'number' || att.expected_size_bytes <= 0) { + return { valid: false, error: `attachments[${i}]: expected_size_bytes required for presigned uploads` }; + } + if (att.expected_size_bytes > MAX_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `attachments[${i}]: expected size exceeds 10 MB limit` }; + } + totalDeclaredSize += att.expected_size_bytes; + } + + // MIME type resolution and validation + // content_type is required after validation; detect from magic bytes if absent + let resolvedContentType: string; + if (att.content_type) { + if (!isAllowedMimeType(att.content_type, att.type)) { + return { valid: false, error: `attachments[${i}]: content_type '${att.content_type}' not allowed for type '${att.type}'` }; + } + resolvedContentType = att.content_type; + } else if (att.data && decoded) { + // Auto-detect from magic bytes for inline attachments (decoded is set above) + const detected = detectMimeTypeFromMagicBytes(decoded); + if (!detected) { + return { valid: false, error: `attachments[${i}]: could not determine file type. Please provide content_type explicitly.` }; + } + resolvedContentType = detected; + } else { + // Presigned uploads and URLs must declare content_type + return { valid: false, error: `attachments[${i}]: content_type is required for presigned uploads and URL attachments` }; + } + + // Filename resolution (required after validation; generate if absent) + const resolvedFilename = att.filename ?? generateFilename(att.type, resolvedContentType, i); + if (!isValidFilename(resolvedFilename)) { + return { valid: false, error: `attachments[${i}]: invalid filename` }; + } + + // Construct validated variant explicitly — "parse, don't validate" pattern. + // This ensures the discriminated union invariants hold at runtime, not just via a cast. + if (att.type === 'url') { + parsed.push({ + delivery: 'url_fetch', + type: 'url', + url: att.url, + filename: resolvedFilename, + content_type: resolvedContentType, + } satisfies UrlAttachment); + } else if (att.data && decoded) { + parsed.push({ + delivery: 'inline', + type: att.type, + data: att.data, + filename: resolvedFilename, + content_type: resolvedContentType, + decoded_size_bytes: decoded.length, // decoded is hoisted and set in the size validation block above + } satisfies InlineAttachment); + } else { + parsed.push({ + delivery: 'presigned', + type: att.type, + filename: resolvedFilename, + content_type: resolvedContentType, + expected_size_bytes: att.expected_size_bytes, + } satisfies PresignedAttachment); + } + } + + // Total inline size check + if (totalInlineSize > MAX_TOTAL_INLINE_SIZE_BYTES) { + return { valid: false, error: `Total inline attachment size exceeds 3 MB limit. Use presigned upload for larger files.` }; + } + + // Total declared size check (inline + presigned) + if (totalInlineSize + totalDeclaredSize > MAX_TOTAL_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `Total attachment size exceeds 50 MB limit` }; + } + + return { valid: true, parsed }; +} + +/** Reject filenames with path traversal, null bytes, or unusual characters */ +function isValidFilename(filename: string): boolean { + if (filename.length > 255) return false; + if (filename.includes('/') || filename.includes('\\')) return false; + if (filename.includes('\0')) return false; + if (filename.startsWith('.') || filename.startsWith('-')) return false; + if (filename === '.' || filename === '..') return false; + return /^[a-zA-Z0-9][a-zA-Z0-9._\- ]{0,253}[a-zA-Z0-9._]$/.test(filename); +} +``` + +**Step 2 — Async pre-checks** (separate function, called after sync validation): + +```typescript +/** Async validation step: SSRF DNS resolution for URL attachments. */ +export async function validateAttachmentUrls( + attachments: ValidatedAttachment[], +): Promise<{ valid: true } | { valid: false; error: string }> { + for (const [i, att] of attachments.entries()) { + if (att.type === 'url') { + try { + const ssrfCheck = await checkSsrf(att.url, { timeoutMs: 5000 }); + if (!ssrfCheck.safe) { + return { valid: false, error: `attachments[${i}]: ${ssrfCheck.reason}` }; + } + } catch (err) { + // DNS lookup failure — fail closed. Differentiate transient vs security failures: + return { valid: false, error: `attachments[${i}]: DNS resolution failed for the provided URL (transient network error, not a security block). Please check the URL is correct and try again.` }; + } + } + } + return { valid: true }; +} +``` + +This split preserves the existing pattern where `validation.ts` functions are synchronous pure functions (no I/O), while keeping the async SSRF check fail-closed with a 5-second timeout. + +## Partial failure cleanup + +When task creation fails after some attachments have already been uploaded to S3, orphaned objects must be cleaned up: + +```typescript +async function cleanupOrphanedAttachments(uploadedKeys: string[]): Promise { + if (uploadedKeys.length === 0) return; + try { + const result = await s3Client.send(new DeleteObjectsCommand({ + Bucket: ATTACHMENTS_BUCKET, + Delete: { Objects: uploadedKeys.map(Key => ({ Key })) }, + })); + // DeleteObjectsCommand does NOT throw on individual object failures — + // check the Errors array explicitly + if (result.Errors && result.Errors.length > 0) { + logger.error('Partial cleanup failure — some orphaned objects remain', { + failedKeys: result.Errors.map(e => e.Key), + errorCodes: result.Errors.map(e => e.Code), + }); + emitMetric('OrphanedAttachmentCleanupPartialFailure', result.Errors.length); + } + } catch (err) { + // Entire cleanup failed — log and emit metric, do NOT re-throw + // (we are already in an error handler; the 90-day lifecycle is the safety net) + logger.error('Cleanup failed entirely — all objects orphaned', { + keys: uploadedKeys, + error: String(err), + }); + emitMetric('OrphanedAttachmentCleanupFailure', uploadedKeys.length); + } +} +``` + +**Strategy:** The inline upload path accumulates S3 keys as it processes attachments. If any attachment fails screening (or any other error occurs), the error handler calls `cleanupOrphanedAttachments` before returning the error response. The function handles both partial and total cleanup failures gracefully — it never re-throws (we're already in an error handler), instead emitting metrics for operational visibility. + +For the presigned upload path, cleanup happens in `confirm-uploads`: if screening fails for any attachment, the handler attempts a conditional DynamoDB write to `FAILED` (`ConditionExpression: 'status = PENDING_UPLOADS'`). Only if the write succeeds does the handler delete S3 objects. If the write fails (concurrent `confirm-uploads` already transitioned to `SUBMITTED`), cleanup is skipped — those objects are needed by the running agent. + +**Lifecycle as safety net:** Even if cleanup fails, the 90-day lifecycle expiration ensures objects are eventually removed. The `OrphanedAttachmentCleanupFailure` metric tracks these cases for alerting. + +**Pre-DynamoDB-write orphans:** If the inline upload path fails BEFORE the TaskRecord is written to DynamoDB (e.g., S3 PutObject succeeds for attachment 1, then screening crashes for attachment 2, and the task was never persisted), the orphaned S3 objects have no task record to correlate them with. The cleanup function still runs and emits `OrphanedAttachmentNoTask` (distinct from `OrphanedAttachmentCleanupFailure`). Structured logs include the S3 keys so operators can identify these objects. The 90-day lifecycle is the final safety net. + +## State machine changes + +Adding `PENDING_UPLOADS` requires updating `task-status.ts`. The full impact: + +### New status + +```typescript +// In cdk/src/constructs/task-status.ts: +export const TaskStatus = { + // ... existing statuses ... + PENDING_UPLOADS: 'PENDING_UPLOADS', +} as const; +``` + +### Valid transitions + +```typescript +export const VALID_TRANSITIONS: Record = { + // ... existing transitions ... + PENDING_UPLOADS: ['SUBMITTED', 'FAILED', 'CANCELLED'], +}; +``` + +### Status classification + +`PENDING_UPLOADS` is **neither active nor terminal**. It is a new category: **pre-active**. It does not count against user concurrency limits (no agent resources allocated), but it is visible in task lists and can be cancelled. + +```typescript +export const ACTIVE_STATUSES: TaskStatusType[] = [ + // PENDING_UPLOADS is NOT here — does not count against concurrency + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING', +]; + +export const PRE_ACTIVE_STATUSES: TaskStatusType[] = [ + 'PENDING_UPLOADS', +]; + +export const TERMINAL_STATUSES: TaskStatusType[] = [ + 'COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT', +]; +``` + +### State diagram + +```mermaid +stateDiagram-v2 + [*] --> PENDING_UPLOADS : Presigned upload task + [*] --> SUBMITTED : Inline/no-attachment task + PENDING_UPLOADS --> SUBMITTED : confirm-uploads succeeds + PENDING_UPLOADS --> FAILED : confirm-uploads screening fails + PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel + SUBMITTED --> HYDRATING : Admission passes + HYDRATING --> RUNNING : Context assembled + RUNNING --> FINALIZING : Session ends + FINALIZING --> COMPLETED : Success + FINALIZING --> FAILED : Agent failure +``` + +### Auto-cancel mechanism + +Tasks in `PENDING_UPLOADS` for > 30 minutes are auto-cancelled via an **EventBridge scheduled rule** (not DynamoDB TTL — TTL would delete the record, losing audit trail). The rule: + +1. Runs every 5 minutes. +2. Queries `TaskTable` for tasks with `status = PENDING_UPLOADS` and `created_at < (now - 30 minutes)`. +3. For each expired task, attempts a **conditional DynamoDB write**: `ConditionExpression: 'status = PENDING_UPLOADS'`. If the condition fails (task already transitioned to `SUBMITTED` by a concurrent `confirm-uploads` call), the Lambda skips that task — no cleanup, no overwrite. +4. On successful conditional write: transitions to `CANCELLED` with `error_message: "Upload window expired (30 minutes). Please re-submit the task."`. +5. Cleans up ALL S3 objects under the task's `attachments///` prefix (not just confirmed ones — catches oversized abuse uploads too). +6. Writes a `pending_upload_expired` event to `TaskEventsTable`. + +**Race safety:** The conditional write ensures the auto-cancel Lambda and `confirm-uploads` cannot both succeed. If `confirm-uploads` wins the race (transitions to `SUBMITTED` first), the auto-cancel Lambda's conditional write fails harmlessly. If the auto-cancel Lambda wins, `confirm-uploads`'s conditional write fails and returns the current `CANCELLED` status to the client. There is no window where S3 objects are deleted from under a running task. + +The task record is preserved with status `CANCELLED` — `bgagent status ` returns a clear explanation, not a 404. + +### Impact on ORCHESTRATOR.md + +`ORCHESTRATOR.md` must be updated to include `PENDING_UPLOADS` in the state diagram and describe the confirm-uploads → SUBMITTED transition. This is a documentation-only change (the orchestrator Lambda itself does not handle `PENDING_UPLOADS` — it only sees tasks that have already reached `SUBMITTED`). + +### Concurrency tracking changes + +The existing codebase increments user concurrency at task creation time (in `create-task-core.ts`), since tasks currently always start in `SUBMITTED`. With `PENDING_UPLOADS`, this must change: + +- **Create-task handler (PENDING_UPLOADS path):** Skip concurrency increment. No agent resources are allocated; the task may never be confirmed. +- **Confirm-uploads handler (PENDING_UPLOADS → SUBMITTED):** Increment user concurrency. If the concurrency limit is reached at this point, the confirm-uploads call fails with `CONCURRENCY_LIMIT_EXCEEDED` (same as if the user tried to create a new task). The task remains in `PENDING_UPLOADS` and the user must wait for a slot or cancel another running task. +- **Auto-cancel Lambda (PENDING_UPLOADS → CANCELLED):** No concurrency decrement needed (was never incremented). + +This ensures `PENDING_UPLOADS` tasks never count against the user's concurrency limit, while still enforcing the limit at the point where agent resources would actually be allocated. + +### Impact on existing code that assumes binary status classification + +The existing codebase uses `ACTIVE_STATUSES` and `TERMINAL_STATUSES` for filtering, concurrency counting, and dashboard queries. With the new `PRE_ACTIVE_STATUSES` category, code paths that assume `!isActive → isTerminal` must be updated: + +| Code path | Current assumption | Required change | +|---|---|---| +| Concurrency counting | All non-terminal tasks are counted | Count only `ACTIVE_STATUSES` (excludes `PENDING_UPLOADS`) | +| `bgagent list` status filter | Filters by active vs terminal | Add `PRE_ACTIVE_STATUSES` to "all non-terminal" filter | +| Dashboard "active tasks" widget | Uses `ACTIVE_STATUSES` | No change needed (already correct) | +| Task cleanup / retention | Applies to `TERMINAL_STATUSES` | No change needed (PENDING_UPLOADS auto-cancels to CANCELLED first) | + +### Impact on CLI + +The CLI's status display and any status-based filtering must recognize `PENDING_UPLOADS`. The `bgagent list` command should show these tasks with a "[pending upload]" indicator. + +## DynamoDB schema changes + +The `TaskRecord` gains one new field: + +```typescript +interface TaskRecord { + // ... existing 39 fields ... + attachments?: AttachmentRecord[]; // Array of attachment metadata +} +``` + +The `status` field gains the new `PENDING_UPLOADS` value (see [State machine changes](#state-machine-changes)). + +No new DynamoDB tables are needed. The `AttachmentRecord[]` is stored as a nested attribute in the existing `TaskTable`. This is appropriate because: + +- Attachments are always accessed with their parent task (never queried independently). +- The metadata is small (< 1 KB per attachment, max 10 attachments = < 10 KB total). +- No secondary index is needed on attachment fields. +- The TaskRecord with 39 fields + 10 attachment records stays well under DynamoDB's 400 KB item size limit. + +## CDK construct changes + +### New: `AttachmentsBucket` + +New construct at `cdk/src/constructs/attachments-bucket.ts`. Follows `TraceArtifactsBucket` patterns (see [Storage](#storage-attachments-s3-bucket) above). + +### New: `ConfirmUploadsFunction` + +A separate Lambda for the `confirm-uploads` endpoint, with: +- **Memory:** 2048 MB (must hold up to 10 MB raw image + decompressed pixel buffer for `sharp` re-encoding; `sharp` decompresses a 4096x4096 RGBA image to ~64 MB in memory) +- **Timeout:** 180 seconds (3 minutes). Attachments are screened in **parallel with bounded concurrency of 3**. Worst-case: 4 batches of ~45s each (S3 read + Bedrock screen with retries + sharp re-encode + S3 write). The 180s budget accommodates Bedrock retry delays. +- **Internal deadline timer:** The handler sets a deadline at `Lambda timeout - 15 seconds` (165s). If the screening loop has not completed by this deadline, remaining unscreened attachments are aborted and the handler returns a 503 with `Retry-After: 30` header and body: "Attachment screening did not complete within the time limit. Reduce the number or size of attachments and try again, or retry after 30 seconds (already-screened attachments will be skipped on retry)." The `Retry-After` header enables clients to implement automatic backoff. On retry, the per-attachment screening state (above) ensures only unscreened attachments are re-processed, so retries make forward progress. This prevents opaque Lambda timeout errors. +- **Per-attachment screening state with atomic DynamoDB + S3 ordering:** Each attachment's screening pipeline follows a strict order: (1) screen content, (2) sanitize/re-encode via sharp, (3) PutObject to S3 (the cleaned version), (4) update the attachment's `screening` status to `passed` in DynamoDB (with `s3_version_id` from the PutObject response). The DynamoDB write is the **commit point** — if any prior step fails, the attachment remains in `pending` status. On retry (after a timeout or Lambda restart), the handler skips attachments with `screening.status === 'passed'` (already committed to both S3 and DynamoDB). Attachments still in `pending` are re-processed from step 1 — this is safe because S3 PutObject is idempotent and the version ID from the new put supersedes any orphaned partial upload. This ordering ensures no attachment is marked as `passed` in DynamoDB without the corresponding cleaned content being in S3. +- **Bundled dependencies:** `sharp` (for EXIF stripping + re-encoding), `pdf-parse` (for PDF text extraction) +- **Cold-start validation:** On module initialization, the handler verifies `sharp` loads correctly (e.g., `sharp(Buffer.alloc(1)).metadata()` wrapped in try-catch). If the native module fails to load (architecture mismatch, missing binary), the handler short-circuits all requests with 503: "Attachment processing is temporarily unavailable. Please try again later." This prevents opaque `Runtime.ImportModuleError` failures from reaching users. + +Separating this from the create-task Lambda keeps the common path (task creation without attachments) lean and fast. + +### New: `PendingUploadCleanupRule` + +EventBridge scheduled rule for auto-cancelling stale `PENDING_UPLOADS` tasks. Runs every 5 minutes, backed by a lightweight Lambda (256 MB, 30s timeout) that queries and transitions expired tasks. + +### Modified: `CreateTaskFunction` + +- **Memory:** 256 MB (increased from CDK default 128 MB — needed for base64 decode of inline attachments + screening + S3 upload) +- **Timeout:** 15 seconds (increased from CDK default 3s — inline screening of small attachments plus S3 operations) +- Pass the attachments bucket as `ATTACHMENTS_BUCKET` environment variable. +- Grant `grantPut` and `grantDelete` on the attachments bucket. + +### Modified: `TaskOrchestrator` + +- Grant `grantReadWrite` on the attachments bucket (for URL attachment fetch/screen/upload). +- Pass the bucket name as `ATTACHMENTS_BUCKET` environment variable. + +### Modified: `TaskApi` (API Gateway) + +- New resource: `POST /v1/tasks/{task_id}/confirm-uploads` → `ConfirmUploadsFunction` +- No body size limit changes needed (presigned uploads bypass the gateway entirely; inline bodies stay under the default limit). +- The `confirm-uploads` endpoint uses the same Cognito authorizer as other task endpoints. + +## API response changes + +The `TaskDetail` response gains an `attachments` summary: + +```typescript +interface TaskDetail { + // ... existing fields ... + attachments?: AttachmentSummary[]; +} + +interface AttachmentSummary { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly filename: string; + readonly content_type: string; + readonly size_bytes: number; + readonly screening_status: 'passed' | 'blocked' | 'pending'; +} +``` + +Binary content is never returned in API responses. The `AttachmentSummary` is metadata only. + +**New type for presigned upload response:** + +```typescript +/** Returned in the creation response for PENDING_UPLOADS tasks only. */ +interface AttachmentUploadInstruction { + readonly attachment_id: string; + readonly filename: string; + readonly upload_url: string; // Presigned POST URL + readonly upload_fields: Record; // Form fields to include in multipart POST + readonly upload_expires_at: string; // Presigned POST policy expiry (10 min) +} +``` + +**New response for presigned upload tasks:** + +```json +{ + "data": { + "task_id": "01HYX...", + "status": "PENDING_UPLOADS", + "task_expires_at": "2025-03-15T11:00:00Z", + "attachments": [ + { + "attachment_id": "01HYX...", + "filename": "screenshot.png", + "upload_url": "https://s3.amazonaws.com/...", + "upload_expires_at": "2025-03-15T10:40:00Z" + } + ] + } +} +``` + +Note: `task_expires_at` (30-minute auto-cancel window) is distinct from `upload_expires_at` (10-minute presigned POST policy expiry). Both are communicated to the client. + +The `upload_url` and `upload_expires_at` fields are only present in the initial creation response. They are not returned on subsequent `GET /v1/tasks/{task_id}` calls. + +## API contract sync + +This design introduces changes that conflict with the current [API_CONTRACT.md](./API_CONTRACT.md). The following updates must be made to API_CONTRACT.md in tandem with implementation: + +| Section | Current value | New value | +|---|---|---| +| Conventions: max body | "max 1 MB body" | "max 1 MB body (6 MB for task creation with inline attachments)" | +| Create task: `data` field | "max 10 MB decoded" | "max 500 KB decoded (inline); use presigned upload for larger files (up to 10 MB)" | +| Create task: `task_description` | "max 2,000 chars" | "max 10,000 chars" (standalone API change — benefits all tasks, not just attachment tasks) | +| Endpoints table | — | Add `POST /v1/tasks/{task_id}/confirm-uploads` | +| Error codes | — | Add all `ATTACHMENT_*` error codes | + +## CLI type sync + +The CLI `types.ts` must be updated to match the server types. The CDK `CreateTaskRequest` already includes `attachments?: Attachment[]` (types.ts line 275), but the CLI mirror at `cli/src/types.ts` lacks it — a pre-existing sync gap that this implementation must close. + +1. Add `attachments?: Attachment[]` to `CreateTaskRequest`. +2. Add `AttachmentType`, `AttachmentSummary`, `AttachmentUploadInstruction` types. +3. Add `attachments?: AttachmentSummary[]` to `TaskDetail`. +4. Add `PresignedUploadResponse` type for the two-phase flow. +5. Update `MAX_TASK_DESCRIPTION_LENGTH` to 10,000. + +## Error codes + +New error codes for attachment-related failures: + +| Code | Status | Description | +|---|---|---| +| `ATTACHMENT_BLOCKED` | 400 | Attachment content blocked by security screening | +| `ATTACHMENT_TOO_LARGE` | 400 | Individual attachment exceeds 10 MB size limit | +| `ATTACHMENT_INLINE_TOO_LARGE` | 400 | Inline attachment exceeds 500 KB limit (use presigned upload) | +| `ATTACHMENTS_INLINE_TOTAL_TOO_LARGE` | 400 | Total inline attachment size exceeds 3 MB limit | +| `ATTACHMENTS_TOTAL_TOO_LARGE` | 400 | Total attachment size exceeds 50 MB limit | +| `ATTACHMENT_INVALID_TYPE` | 400 | MIME type not in allowlist | +| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or could not be sanitized (sharp failure) | +| `ATTACHMENT_INVALID_FILENAME` | 400 | Filename contains invalid characters or path traversal | +| `ATTACHMENT_SIZE_MISMATCH` | 400 | Uploaded file size does not match declared `expected_size_bytes` (> 10% deviation) | +| `ATTACHMENT_FETCH_FAILED` | 422 | URL attachment could not be fetched (timeout, DNS, SSRF blocked) | +| `ATTACHMENT_BUDGET_EXCEEDED` | 422 | Image attachments exceed token budget (insufficient room for text context) | +| `ATTACHMENT_DOWNLOAD_FAILED` | 500 | Agent could not download attachment from S3 | +| `ATTACHMENT_INTEGRITY_FAILED` | 500 | Attachment checksum mismatch after download | +| `ATTACHMENT_UNSUPPORTED_AGENT` | 422 | Agent runtime version does not support attachments | +| `ATTACHMENT_SCREENING_UNAVAILABLE` | 503 | Screening service unavailable after retries (fail-closed) | +| `UPLOADS_NOT_CONFIRMED` | 409 | Task is in PENDING_UPLOADS but confirm-uploads not yet called | +| `UPLOADS_EXPIRED` | 410 | Upload window expired (> 30 minutes); re-submit the task | + +## Observability + +### New CloudWatch metrics + +| Metric | Dimensions | Purpose | +|---|---|---| +| `AttachmentUploadCount` | Type (`image`/`file`/`url`), Path (`inline`/`presigned`/`url_fetch`) | Track attachment usage patterns | +| `AttachmentUploadSize` | Type | Size distribution | +| `AttachmentScreeningDuration` | Type, Outcome | Screening latency | +| `AttachmentScreeningOutcome` | Type, Outcome (`passed`/`blocked`) | Block rate | +| `AttachmentScreeningRetry` | RetryCount | How often retries are needed | +| `AttachmentFetchDuration` | — | URL fetch latency | +| `AttachmentFetchFailure` | Reason (`timeout`/`ssrf_blocked`/`too_large`/`dns_error`/`http_error`) | URL fetch failure breakdown | +| `AttachmentTokenBudgetUsed` | — | How much of the token budget images consume | +| `AttachmentBudgetExceeded` | — | Tasks failed due to image token budget overflow | +| `OrphanedAttachmentCleanupFailure` | — | Cleanup failures (orphaned S3 objects) | +| `OrphanedAttachmentCleanupPartialFailure` | — | Partial cleanup failures | +| `OrphanedAttachmentNoTask` | — | Objects uploaded before task was persisted to DynamoDB (pre-write failures); no task record to correlate | +| `PendingUploadExpired` | — | Tasks that expired in PENDING_UPLOADS (never confirmed) | +| `ConfirmUploadsRace` | — | Concurrent confirm-uploads detected (condition check failed) | +| `SharpColdStartFailure` | — | `sharp` native module failed to load on Lambda cold start | +| `ScreeningDeadlineExceeded` | — | Confirm-uploads hit internal deadline timer before all attachments screened | + +### Task events + +New event types in `TaskEventsTable`: + +| Event type | When | Metadata | +|---|---|---| +| `attachments_uploaded` | After inline attachments are stored in S3 | Count, total size, types | +| `uploads_confirmed` | After confirm-uploads succeeds | Count, total size, screening duration | +| `attachment_blocked` | Screening rejects an attachment | Attachment ID, screening categories | +| `attachment_fetch_failed` | URL attachment fetch fails | Attachment ID, URL (redacted), reason | +| `attachments_resolved` | All attachments ready for agent | Count, total size, token budget used | +| `pending_upload_expired` | Task auto-cancelled due to upload timeout | Task ID, pending attachment count | +| `attachment_unsupported` | Agent version does not support attachments | Agent version, attachment count | + +## Security considerations + +### Threat model + +| Threat | Vector | Mitigation | +|---|---|---| +| Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; Bedrock image screening; EXIF stripping; image re-encoding through `sharp` strips embedded payloads; sharp failure → reject | +| Prompt injection via file content | Text file containing adversarial instructions | Magic bytes validation; Bedrock Guardrail text screening with retry (same as task descriptions); content trust tagging as `untrusted-external` | +| SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time | +| Data exfiltration via URL attachment | URL pointing to attacker-controlled server (leaks request headers/IP) | No auth headers sent to non-GitHub URLs; minimal request headers; no cookies | +| Denial of service via large attachments | Many large base64 payloads | 500 KB inline limit; 3 MB total inline; 10 MB per-attachment; 50 MB total; 10 count limit; 6 MB Lambda payload limit | +| Path traversal via filename | `filename: "../../etc/passwd"` | Filename sanitization regex; reject path separators, dots-prefix, null bytes; use `attachment_id` as primary path component | +| Zip bomb / decompression bomb | Compressed content that expands massively | No archive types in MIME allowlist; PDF text extraction capped at 50 pages and 1 MB output | +| Polyglot files | File with valid image header + appended executable | Magic bytes validation at upload; image re-encoding strips trailing data | +| Presigned URL abuse | Leaked presigned POST policy used to upload different content | Content-Type fixed in presigned POST policy; `content-length-range` enforced by S3 (rejects > 10 MB before writing); 10-minute expiry; screening runs after upload regardless; size verified via HeadObject (defense-in-depth) | +| S3 object replacement (TOCTOU) | Client uploads benign content (passes screening), then replaces object with malicious content before agent downloads | S3 versioning enabled; `VersionId` pinned at screening time and stored in `AttachmentRecord`; agent downloads with pinned `VersionId`; `noncurrentVersionExpiration: 7 days` prevents storage bloat while allowing long-running tasks to complete | +| Upload slot exhaustion | Create many tasks in PENDING_UPLOADS, never confirm | 30-minute EventBridge auto-cancel with S3 cleanup; PENDING_UPLOADS does not count against concurrency; rate limiting on task creation already exists | +| Confirm-uploads race | Two concurrent confirm-uploads corrupt state | Early status check short-circuit; DynamoDB conditional write; safe cleanup skips if already SUBMITTED | +| DNS rebinding | DNS returns public IP at first lookup (passes validation), private IP at second lookup (reaches internal services) | DNS resolution pinning: resolve DNS manually, validate IP, connect directly to the validated IP (preventing a second DNS lookup); re-validate resolved IP after each redirect | + +### Content trust + +Attachment content inherits the `untrusted-external` trust level in the content trust framework. The agent's system prompt labels attachments accordingly: + +``` +Content trust levels: +- task_description: trusted (from authenticated user) +- issue_body: untrusted-external (from GitHub) +- attachments: untrusted-external (user-provided binary/text content, security-screened) +``` + +This is correct even for inline uploads from authenticated users. The content of a file is fundamentally different from the task description: a user might upload a file they received from an untrusted source (e.g., a customer's screenshot, a downloaded log file). Screening catches known-bad content; trust tagging handles the residual risk by ensuring the agent treats attachment content as context, not as instructions. + +## Cost impact + +| Component | Additional cost per task (with attachments) | Notes | +|---|---|---| +| S3 storage | ~$0.001 | 50 MB * $0.023/GB, 90-day retention | +| S3 PUT/GET | ~$0.00001 | 10 PUTs + 10 GETs | +| Bedrock Guardrail (image) | ~$0.01-0.05 per image | Depends on image size and guardrail config | +| Bedrock Guardrail (text) | ~$0.001-0.01 per file | Same as existing text screening | +| Lambda compute (confirm-uploads) | ~$0.01-0.05 | 30-180s at 2048 MB for screening + re-encoding | +| Lambda compute (create-task, inline) | ~$0.001 | Additional 1-3s at 256 MB for small inline attachments | +| Lambda compute (auto-cancel rule) | ~$0.0001 | 5-minute schedule, mostly no-op | +| Data transfer (URL fetch) | ~$0.001 | Outbound fetch within region is free; cross-region is negligible | +| Claude vision (image in prompt) | ~$0.01-0.05 per image | Multimodal input token cost | + +**Total additional cost per task with attachments:** ~$0.05-0.35, heavily dependent on attachment count and types. Tasks without attachments have zero additional cost. + +## Implementation plan + +The implementation is ordered to deliver value incrementally while maintaining system safety. Security protections ship with the attack vectors they defend against — not deferred to a later phase. + +### Phase 1: Storage + validation + inline upload (small attachments) + +1. Verify `@aws-sdk/client-bedrock-runtime` supports image content blocks in `ApplyGuardrail` (specifically `GuardrailImageBlock` with `format: 'png' | 'jpeg'`); upgrade if needed +2. Create `AttachmentsBucket` construct (with versioning enabled, props interface) +3. Extract `AttachmentType` shared type in `types.ts`; add `AttachmentDelivery` type +4. Add `ValidatedAttachment` discriminated union types (with `delivery` discriminant) +5. Add `AttachmentRecord` with discriminated `ScreeningResult` union (non-empty `categories` tuple, `s3_version_id` field) +6. Add `createAttachmentRecord` factory function (enforces cross-field invariants from day one) +7. Add `AttachmentSummary` and `AttachmentUploadInstruction` response types +8. Add `AgentAttachmentPayload` named interface (with `checksum_sha256` and `s3_version_id`) +9. Add `AttachmentError` base class hierarchy (`AttachmentResolutionError`, `AttachmentBudgetExceededError`) +10. Add `AttachmentError` base class to hydration catch block re-throw list (single `instanceof` check — ships with the error classes so the re-throw is in place before any code throws these errors in Phase 3) +11. Add attachment validation to `validation.ts` (sync: schema, limits, magic bytes, content_type detection, filename generation; async: SSRF DNS pre-check with differentiated error messages) +12. Increase `MAX_TASK_DESCRIPTION_LENGTH` to 10,000 (standalone API change — update API_CONTRACT.md) +13. Add Bedrock screening retry logic (3 retries, exponential backoff) +14. Add GIF/WebP → PNG conversion before Bedrock screening (Bedrock only supports png|jpeg), with post-conversion size check +15. Add inline upload path to `create-task-core.ts` (base64 → magic bytes → screen with retry → EXIF strip/re-encode → S3 with versioning; sharp failure → ATTACHMENT_INVALID_CONTENT) +16. Add SHA-256 checksum computation at upload time (stored in AttachmentRecord, required by factory) +17. Add partial failure cleanup with proper `DeleteObjects` error handling and metrics +18. Add `attachments` field to `TaskRecord` +19. Add `AttachmentSummary` to `TaskDetail` response +20. Sync CLI types (close the pre-existing sync gap) +21. Increase create-task Lambda memory to 256 MB (from CDK default 128 MB), timeout to 15s (from CDK default 3s) +22. Update `API_CONTRACT.md` (inline limit, task_description limit, note Bedrock format constraints) + +**Security included in Phase 1:** magic bytes validation, EXIF stripping, image re-encoding, GIF/WebP conversion (with post-conversion size check), sharp fail-closed, filename sanitization, partial failure cleanup, Bedrock retry, S3 versioning (TOCTOU prevention), SHA-256 integrity. + +### Phase 2: Presigned upload + state machine (large attachments) + +23. Add `PENDING_UPLOADS` to `task-status.ts` (status, transitions, `PRE_ACTIVE_STATUSES` classification) +24. Update all code paths that assume binary status classification (active vs terminal) — see [Impact on existing code](#impact-on-existing-code-that-assumes-binary-status-classification) +25. Add `confirm-uploads` Lambda (2048 MB, 180s timeout) with parallel screening (concurrency 3), internal deadline timer, per-attachment screening state, and sharp cold-start validation +26. Add `POST /v1/tasks/{task_id}/confirm-uploads` API endpoint with concurrent-call safety (early short-circuit, conditional DynamoDB write for both success and failure paths) +27. Move concurrency increment from create-task to confirm-uploads for presigned-upload tasks +28. Add presigned POST policy generation in create-task handler (with `content-length-range` enforcement, `expected_size_bytes` validation, S3 versioning) +29. Add idempotency key special-casing for `PENDING_UPLOADS` (new S3 keys + new attachment IDs on retry to prevent collision, conditional DynamoDB write) +30. Add `PendingUploadCleanupRule` EventBridge rule (5-minute schedule, conditional DynamoDB write for race safety, prefix-level S3 cleanup) +31. Add CLI two-phase upload flow for files > 500 KB (with progress bar, multipart form POST) +32. Update `ORCHESTRATOR.md` state diagram + +### Phase 3: Agent delivery + token budget + +33. Add `attachments` to top-level agent payload (in orchestrator) using `AgentAttachmentPayload` (includes `s3_version_id` and `checksum_sha256`) +34. Add `AttachmentConfig` and `PreparedAttachment` Pydantic models to agent `models.py` (with validators, `s3_version_id` required, `checksum_sha256` required as lowercase hex) +35. Add attachment download from S3 with pinned `VersionId` (via IAM role) and mandatory SHA-256 integrity verification +36. Add multimodal content blocks for image attachments in agent prompt +37. Add token budget accounting with resize-aware formula matching Anthropic docs (1568px cap, 28px tile padding, 1568 token cap, 1.2x safety margin), with explicit error path for `getImageDimensions` failures +38. Add `AttachmentBudgetExceededError` (extends `AttachmentError` base class — already caught by hydration re-throw list from Phase 1 step 10) +39. Add agent capability check in orchestrator (fail task if agent doesn't support attachments) +40. Add parity test: `AgentAttachmentPayload` fields match `AttachmentConfig` fields (including `s3_version_id` and `checksum_sha256`) + +### Phase 4: URL fetch + channels + +41. Add URL fetch in orchestrator hydration with full SSRF suite (manual DNS resolve → IP check → connect to resolved IP via undici; differentiated error messages: "DNS failed" vs "SSRF blocked") +42. Add `--attachment` flag to `bgagent submit` (auto-detect file vs URL, size-based routing to inline vs presigned, URL attachment polling) +43. Add Slack file extraction in `slack-events.ts` with atomic failure semantics (consistent with design principle 3 — all-or-nothing) +44. Add Linear image URL extraction in `linear-webhook-processor.ts` + +**Security included in Phase 4:** Full SSRF protection suite with DNS resolution pinning (TOCTOU mitigation). + +### Phase 5: Observability + hardening + +45. Add all CloudWatch metrics (including retry, race detection, cleanup failure, `OrphanedAttachmentNoTask` for pre-DynamoDB-write failures) +46. Add all task events +47. Add alerting on `OrphanedAttachmentCleanupFailure` and `PendingUploadExpired` +48. Add comprehensive integration tests (all upload paths, all failure modes, all channels, concurrent confirm-uploads, auto-cancel racing with confirm-uploads) +49. Add load testing for screening pipeline (sustained 10-attachment tasks at peak rate) +50. Add monitoring for `sharp` native module health (cold-start validation failures) +51. Add monitoring for format conversion size expansion (GIF/WebP → PNG rejection rate) From 90b9cfc7b6664511b51d2e5c134cdc84f6f3827c Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 19 May 2026 15:03:04 -0500 Subject: [PATCH 02/19] chore(attachments): start work --- cdk/src/handlers/shared/create-task-core.ts | 2 +- cdk/src/handlers/shared/types.ts | 2 +- cdk/src/handlers/shared/validation.ts | 4 +- cdk/test/handlers/shared/validation.test.ts | 14 +- .../content/docs/architecture/Attachments.md | 1695 +++++++++++++++++ 5 files changed, 1708 insertions(+), 9 deletions(-) create mode 100644 docs/src/content/docs/architecture/Attachments.md diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index cfa94a7d..91ecc25d 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -21,13 +21,13 @@ // Idempotent replay: same user + same Idempotency-Key → 200 + TaskDetail (no duplicate write, no orchestrator re-invoke). // Tests: cdk/test/handlers/shared/create-task-core.test.ts, cdk/test/handlers/create-task.test.ts +import { createHash } from 'crypto'; import { BedrockRuntimeClient, ApplyGuardrailCommand } from '@aws-sdk/client-bedrock-runtime'; import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; import { PutObjectCommand, DeleteObjectsCommand, S3Client } from '@aws-sdk/client-s3'; import { DynamoDBDocumentClient, PutCommand, QueryCommand, GetCommand } from '@aws-sdk/lib-dynamodb'; import type { APIGatewayProxyResult } from 'aws-lambda'; -import { createHash } from 'crypto'; import { ulid } from 'ulid'; import { generateBranchName } from './gateway'; import { logger } from './logger'; diff --git a/cdk/src/handlers/shared/types.ts b/cdk/src/handlers/shared/types.ts index 88daebcb..266c2761 100644 --- a/cdk/src/handlers/shared/types.ts +++ b/cdk/src/handlers/shared/types.ts @@ -473,7 +473,7 @@ export function toTaskDetail(record: TaskRecord): TaskDetail { type: a.type, filename: a.filename, content_type: a.content_type, - size_bytes: a.size_bytes ?? 0, // 0 for pending attachments (size unknown until resolved) + size_bytes: a.size_bytes ?? 0, // 0 for pending attachments (size unknown until resolved) screening_status: a.screening.status, })) : null, diff --git a/cdk/src/handlers/shared/validation.ts b/cdk/src/handlers/shared/validation.ts index 18fa369e..56dad904 100644 --- a/cdk/src/handlers/shared/validation.ts +++ b/cdk/src/handlers/shared/validation.ts @@ -294,8 +294,8 @@ const ALLOWED_FILE_MIME_TYPES = new Set([ const MAGIC_BYTES: ReadonlyArray<{ readonly mime: string; readonly bytes: readonly number[]; readonly offset?: number }> = [ { mime: 'image/png', bytes: [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A] }, { mime: 'image/jpeg', bytes: [0xFF, 0xD8, 0xFF] }, - { mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] }, // GIF8 (covers GIF87a and GIF89a) - { mime: 'application/pdf', bytes: [0x25, 0x50, 0x44, 0x46, 0x2D] }, // %PDF- + { mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] }, // GIF8 (covers GIF87a and GIF89a) + { mime: 'application/pdf', bytes: [0x25, 0x50, 0x44, 0x46, 0x2D] }, // %PDF- ]; // RIFF....WEBP requires checking bytes 0-3 (RIFF) and 8-11 (WEBP) diff --git a/cdk/test/handlers/shared/validation.test.ts b/cdk/test/handlers/shared/validation.test.ts index b4b0f152..83e93641 100644 --- a/cdk/test/handlers/shared/validation.test.ts +++ b/cdk/test/handlers/shared/validation.test.ts @@ -17,6 +17,7 @@ * SOFTWARE. */ +import { createAttachmentRecord } from '../../../src/handlers/shared/types'; import { computeTtlEpoch, decodePaginationToken, @@ -42,8 +43,6 @@ import { validatePrNumber, } from '../../../src/handlers/shared/validation'; -import { createAttachmentRecord } from '../../../src/handlers/shared/types'; - describe('parseBody', () => { test('parses valid JSON', () => { expect(parseBody('{"key":"value"}')).toEqual({ key: 'value' }); @@ -514,8 +513,11 @@ describe('validateAttachments', () => { test('rejects data + url together', () => { const result = validateAttachments([{ - type: 'image', data: pngBase64, url: 'https://example.com/x.png', - content_type: 'image/png', filename: 'x.png', + type: 'image', + data: pngBase64, + url: 'https://example.com/x.png', + content_type: 'image/png', + filename: 'x.png', }]); expect(result.valid).toBe(false); if (!result.valid) expect(result.error).toContain('not both'); @@ -540,7 +542,9 @@ describe('validateAttachments', () => { test('rejects presigned upload exceeding 10 MB', () => { const result = validateAttachments([{ - type: 'image', content_type: 'image/png', filename: 'huge.png', + type: 'image', + content_type: 'image/png', + filename: 'huge.png', expected_size_bytes: 11 * 1024 * 1024, }]); expect(result.valid).toBe(false); diff --git a/docs/src/content/docs/architecture/Attachments.md b/docs/src/content/docs/architecture/Attachments.md new file mode 100644 index 00000000..9c0c651a --- /dev/null +++ b/docs/src/content/docs/architecture/Attachments.md @@ -0,0 +1,1695 @@ +--- +title: Attachments +--- + +# Task Attachments (Multimodal) + +End-to-end support for attaching files, images, and URLs to agent tasks. Attachments let users provide non-text context — screenshots of bugs, design mockups, CSV data, log files, code snippets — that the agent can reference during execution. Every channel (CLI, webhook, Slack, Linear) feeds the same schema; every attachment passes through security screening before reaching the agent. + +- **Use this doc for:** understanding the attachment data model, upload flow, security screening pipeline, storage layout, agent consumption, and per-channel behaviour. +- **Related docs:** [API_CONTRACT.md](/architecture/api-contract) for the `attachments` request schema (must be updated in tandem — see [API contract sync](#api-contract-sync)), [ORCHESTRATOR.md](/architecture/orchestrator) for the task lifecycle this extends, [SECURITY.md](/architecture/security) for guardrail and Cedar context, [ARCHITECTURE.md](/architecture/architecture) for the platform overview. + +## Motivation + +Today, task context is limited to text: a `task_description`, an `issue_number` (whose body is text), or a `pr_number`. Users cannot show the agent what they see. Common situations where text alone is insufficient: + +1. **Bug reports with screenshots** — "The button is misaligned" means nothing without the screenshot. The user must upload the image to GitHub, create an issue, and reference it — friction that discourages use. +2. **Design mockups** — "Implement this design" requires a mockup image. Today the agent can only read text descriptions of designs. +3. **Log files and data** — A 500-line stack trace or CSV dataset is awkward to paste into a 10,000-char `task_description`. +4. **Code snippets from other repos** — "Port this function from repo X" requires the user to paste code into an issue. +5. **Integration payloads** — Linear issues or Slack threads may contain images, files, or links that are lost when only the text body is forwarded to the agent. + +Attachments solve this by providing a unified carrier for all non-text task context. + +## Design principles + +1. **Store once, reference everywhere.** Binary data lives in S3. Every downstream consumer (orchestrator, agent, audit log) uses S3 references, never inline blobs. The primary upload path bypasses the API entirely (presigned POST policies with size enforcement); a convenience inline path exists for small attachments only. +2. **No unscreened binary content in S3.** Every attachment passes through security screening before its binary content is written to durable S3 storage. DynamoDB metadata (attachment ID, filename, pending status) may be written before screening completes, but the actual blob is gated on a passing screen result. +3. **Fail closed, fail loud.** If screening is unavailable, the task fails. If any attachment fails screening (inline, URL, or presigned), the task fails with a clear error — attachments are not silently dropped. Users can re-submit without the problematic attachment. This principle extends to the hydration fallback path — attachment resolution errors must propagate and fail the task, never be caught by the generic infrastructure fallback (see [Hydration error handling](#hydration-error-handling)). +4. **Channel-agnostic schema.** The `Attachment` type is the same whether the source is a CLI flag, a webhook JSON field, a Slack file upload, or a Linear issue image. Channel-specific adapters normalize to this schema before entering the shared path. +5. **Agent sees files, not infrastructure.** The agent receives attachments as local files in its workspace (images, documents) or as prompt content blocks (images for multimodal models). It does not interact with S3 directly. + +## Data model + +### Shared types + +Extract a shared `AttachmentType` literal union, reused across all attachment-related interfaces (same pattern as `TaskType` and `ChannelSource`): + +```typescript +/** Shared across all attachment interfaces. Add new types here (e.g., 'audio'). */ +export type AttachmentType = 'image' | 'file' | 'url'; +``` + +### Attachment schema (API layer) + +The existing `Attachment` interface in `types.ts` is a flat union of optional fields. For deserialization from untrusted JSON this is fine, but downstream consumers benefit from a validated discriminated union. The design uses two layers: + +**Wire format (deserialization):** The existing `Attachment` interface in `types.ts` (lines 283-289) is modified in-place — adding `expected_size_bytes` for presigned upload budget pre-checks. This is a non-breaking change (new optional field). All fields beyond `type` remain optional because the input is untrusted: + +```typescript +/** Wire format — parsed from untrusted JSON. Validate before use. */ +interface Attachment { + readonly type: AttachmentType; + readonly content_type?: string; + readonly data?: string; // Base64-encoded content (inline upload, max 500 KB decoded) + readonly url?: string; // URL to fetch (url type) + readonly filename?: string; // Original filename + readonly expected_size_bytes?: number; // Declared size for presigned uploads (required for budget pre-check) +} +``` + +**Validated format (post-validation):** After `validateAttachments()` succeeds, the result is a discriminated union that makes illegal states unrepresentable: + +```typescript +/** Delivery mechanism — discriminant for the three upload paths. + * `type` alone cannot distinguish inline from presigned (both are 'image' | 'file'), + * so `delivery` provides the exhaustive three-way branch. */ +type AttachmentDelivery = 'inline' | 'presigned' | 'url_fetch'; + +interface BaseAttachment { + readonly filename: string; // Required after validation (generated if absent) + readonly content_type: string; // Required after validation (detected from magic bytes if absent) +} + +/** Inline image/file: data present, validated, decoded, magic-bytes checked */ +interface InlineAttachment extends BaseAttachment { + readonly delivery: 'inline'; + readonly type: 'image' | 'file'; + readonly data: string; // Validated base64 + readonly url?: never; + readonly decoded_size_bytes: number; +} + +/** Presigned upload: metadata only, no data, no url */ +interface PresignedAttachment extends BaseAttachment { + readonly delivery: 'presigned'; + readonly type: 'image' | 'file'; + readonly data?: never; + readonly url?: never; + readonly expected_size_bytes: number; // Declared by client for early budget validation +} + +/** URL to fetch during hydration */ +interface UrlAttachment extends BaseAttachment { + readonly delivery: 'url_fetch'; + readonly type: 'url'; + readonly url: string; + readonly data?: never; +} + +/** Output of validateAttachments() — illegal combinations are unrepresentable */ +type ValidatedAttachment = InlineAttachment | PresignedAttachment | UrlAttachment; +``` + +The `delivery` field is the primary discriminant for exhaustive `switch` statements: `switch (att.delivery)` branches into the three upload paths without nested `if ('data' in att)` checks. The `type` field (`'image' | 'file' | 'url'`) describes the content kind. The `never` fields prevent accidental inclusion of forbidden properties at compile time. Note: `never` only provides compile-time protection — the validation function must construct each variant explicitly (see [Validation changes](#validation-changes)) to guarantee forbidden fields are absent at runtime. + +### Attachment record (persisted metadata) + +After upload and screening, each attachment becomes an `AttachmentRecord` stored as part of the `TaskRecord` in DynamoDB. The `screening` field is a discriminated union that prevents nonsensical states (e.g., a `passed` record with blocking categories): + +```typescript +/** Screening outcome — discriminated union prevents invalid combinations. + * `categories` uses a non-empty tuple type to make empty-array states unrepresentable. */ +type ScreeningResult = + | { readonly status: 'pending' } + | { readonly status: 'passed'; readonly screened_at: string } + | { readonly status: 'blocked'; readonly screened_at: string; readonly categories: [string, ...string[]] }; + +interface AttachmentRecord { + readonly attachment_id: string; // ULID + readonly type: AttachmentType; + readonly content_type: string; // Resolved MIME type + readonly filename: string; // Original or generated filename + readonly s3_key?: string; // S3 object key — absent when pending (URL not yet fetched) + readonly s3_version_id?: string; // S3 object version — pinned at screening time (see S3 versioning) + readonly size_bytes?: number; // Decoded content size — absent when pending + readonly screening: ScreeningResult; + readonly source_url?: string; // Original URL (for url type or channel-sourced) + readonly checksum_sha256?: string; // Lowercase hex-encoded SHA-256 (64 chars) — required when screening.status === 'passed' (enforced by factory) + readonly token_estimate?: number; // Estimated token cost (images only) +} +``` + +**Runtime validation note:** The `ScreeningResult` discriminated union and the `[string, ...string[]]` non-empty tuple provide compile-time safety, but data read from DynamoDB is untyped at runtime. The `createAttachmentRecord` factory (below) validates these invariants at construction time. For records read back from DynamoDB, a `parseScreeningResult(raw: unknown): ScreeningResult` function must validate: (a) `status` is one of the three allowed values, (b) `screened_at` is present when status is `passed` or `blocked`, (c) `categories` is a non-empty array when status is `blocked`. This parser is called in the DynamoDB → `AttachmentRecord` mapper (same pattern as the existing `toTaskDetail` mapper). Invalid data throws rather than silently returning a malformed object. + +**Changes from prior version:** `s3_key` and `size_bytes` are now `undefined` when pending (not sentinel values `'pending'` and `0`). This is idiomatic TypeScript — the compiler forces null-checks on consumers instead of relying on documentation about magic values. + +The `TaskRecord` gains an `attachments?: AttachmentRecord[]` field. This stores metadata only — binary content lives in S3. + +**Construction (Phase 1 — ships with the type definitions):** A factory function `createAttachmentRecord(params)` centralizes construction validation, matching the codebase pattern of mapper functions like `toTaskDetail`. This ships in Phase 1 alongside the type definitions — not deferred — because cross-field invariants (`s3_key` required when screening passed, `checksum_sha256` required when screening passed, `token_estimate` required for images) are too dangerous to leave unenforced during 4 phases of development: + +```typescript +function createAttachmentRecord(params: CreateAttachmentRecordParams): AttachmentRecord { + // Validates invariants: + // - token_estimate required when type === 'image' + // - s3_key and s3_version_id required when screening.status === 'passed' + // - checksum_sha256 required when screening.status === 'passed' + // - size_bytes required when screening.status === 'passed' + // - categories non-empty when screening.status === 'blocked' + if (params.screening.status === 'passed') { + if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { + throw new Error('Passed screening requires s3_key, s3_version_id, checksum_sha256, and size_bytes'); + } + } + if (params.type === 'image' && params.screening.status === 'passed' && !params.token_estimate) { + throw new Error('Image attachments with passed screening must have token_estimate'); + } + return params as AttachmentRecord; +} +``` + +### Agent payload type + +A named TypeScript interface for the orchestrator → agent payload (not an anonymous `.map()` shape). This enables test contracts and prevents silent drift between the TypeScript producer and Python consumer: + +```typescript +/** Attachment descriptor sent to the agent runtime. Exported for test assertions. */ +export interface AgentAttachmentPayload { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly content_type: string; + readonly filename: string; + readonly s3_uri: string; // s3://bucket/attachments/user/task/att/file.png + readonly s3_version_id: string; // Pinned S3 object version — prevents TOCTOU between screening and download + readonly size_bytes: number; + readonly source_url?: string; // Original URL (for url type) + readonly token_estimate?: number; // Images only + readonly checksum_sha256: string; // Lowercase hex-encoded SHA-256 (64 chars) of screened content — agent verifies after download +} +``` + +A test should assert field-for-field parity between this interface and the Python `AttachmentConfig` Pydantic model. + +### S3 key layout + +``` +attachments//// +``` + +Example: `attachments/us-east-1:abc123/01J5X7.../01J5X8.../screenshot.png` + +The `` segment ensures uniqueness even if multiple attachments share the same filename. The `` suffix preserves the original name for human-readable S3 console browsing and for the agent (which receives the file under its original name). + +## Limits + +| Limit | Value | Rationale | +|---|---|---| +| Max attachments per task | 10 | Bounds screening cost and agent context size | +| Max size per attachment (decoded) | 10 MB | Bedrock image input limit; practical for screenshots/logs | +| Max inline data per attachment | 500 KB decoded | The Lambda synchronous invocation payload limit is **6 MB**. At 500 KB decoded (~667 KB base64) per attachment, even 5 inline attachments plus request JSON stays under 6 MB. The presigned path handles anything larger. | +| Max total inline data per request | 3 MB decoded | Hard cap on total base64-decoded bytes in a single request. Even with base64 overhead (~4 MB encoded) plus JSON fields, this stays under the 6 MB Lambda payload limit. | +| Max total size per task | 50 MB | Prevents abuse; bounds total screening and transfer time | +| Max task_description length | 10,000 chars | Increased from 2,000. **This is a standalone API change that affects all tasks** (not just attachment tasks). Rationale: (a) attachments need rich explanatory context ("implement this design per the attached mockup, paying attention to the header layout"), (b) multiple users have reported the 2K limit as a friction point for complex task descriptions even without attachments, (c) the guardrail screening cost increase is minimal (text screening is cheap), (d) DynamoDB item size impact is negligible (~8 KB vs ~2 KB for the description field). **Requires updating [API_CONTRACT.md](/architecture/api-contract) line 82 in tandem.** | +| Allowed image MIME types | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | Bedrock vision-supported formats | +| Allowed file MIME types | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | Useful for code/data context; no executables | +| Max URL fetch size | 10 MB | Same per-attachment limit for fetched content | +| URL fetch timeout | 10 seconds | Prevent SSRF-style long-poll attacks | +| URL scheme | `https` only | No `http`, `file`, `ftp`, or custom schemes | + +**Payload limit note:** The binding constraint for the inline path is **not** the API Gateway 10 MB body limit — it is the **Lambda synchronous invocation payload limit of 6 MB**. API Gateway REST API forwards the request body to Lambda as part of the invocation payload, which includes the body plus API Gateway metadata. The limits above are set conservatively to ensure the full payload stays under 6 MB. + +## Upload flows + +The design provides three upload paths, unified by a common screening + S3 storage backend. The presigned upload is the primary path for files > 500 KB; inline base64 is a convenience for small attachments; URL fetch handles remote resources. + +### Primary path: Presigned URL upload + +For attachments > 500 KB (or any attachment where the client prefers not to base64-encode), the client requests a presigned POST policy and uploads directly to S3. This bypasses the Lambda payload limit entirely. S3 enforces the `content-length-range` condition server-side, rejecting oversized uploads before storing them. + +```mermaid +sequenceDiagram + participant C as Client + participant GW as API Gateway + participant H as Create-Task Lambda + participant S3 as Attachments Bucket + participant SCR as Screening Pipeline + participant DB as TaskTable + + C->>GW: POST /v1/tasks { attachments: [{ type: 'image', filename: 'screenshot.png', content_type: 'image/png', expected_size_bytes: 2500000 }] } + GW->>H: Forward (metadata only, no binary) + H->>H: Validate metadata + declared sizes against limits + H->>S3: Generate presigned POST policy (10 min expiry, content-length-range enforced) + H->>DB: Write TaskRecord (status: PENDING_UPLOADS, attachments with screening: pending) + H-->>C: 202 Accepted { task_id, upload_urls: [...], upload_fields: [...], task_expires_at } + + C->>S3: POST multipart/form-data (binary content, policy enforces size) + S3-->>C: 200 OK + + C->>GW: POST /v1/tasks/{task_id}/confirm-uploads + GW->>H: Forward + H->>H: Check status == PENDING_UPLOADS (short-circuit if already SUBMITTED) + H->>S3: HeadObject per attachment (verify uploads exist, with retry for 404) + loop Each attachment (parallel, bounded concurrency 3) + H->>S3: GetObject (stream content for screening) + H->>SCR: Screen content (with retry: 3 attempts, exponential backoff) + SCR-->>H: Pass / Blocked + alt Screening blocked + H->>S3: DeleteObjects (all uploads for this task) + H->>DB: Update status → FAILED + H-->>C: 400 ATTACHMENT_BLOCKED { attachment_id, reason } + end + end + H->>H: Strip EXIF from images, re-upload cleaned version + H->>DB: Update TaskRecord (status: SUBMITTED, screening: passed) with condition: status = PENDING_UPLOADS + H->>H: Async-invoke Orchestrator + H-->>C: 200 OK { task_id, status: SUBMITTED } +``` + +**Key details:** + +- **New task status: `PENDING_UPLOADS`.** Tasks with presigned-upload attachments are created in `PENDING_UPLOADS` status. They do not enter the orchestration pipeline until uploads are confirmed. See [State machine changes](#state-machine-changes) for the full transition table. +- **Declared sizes for early budget validation.** Clients must include `expected_size_bytes` for presigned attachments. The create-task handler validates total declared size against the 50 MB limit immediately, preventing the user from uploading 100 MB only to be rejected at confirm-uploads. +- **Presigned POST policy generation.** S3 presigned POST policies (via `createPresignedPost` from `@aws-sdk/s3-presigned-post`) support **`content-length-range` conditions**, enforcing the 10 MB per-attachment limit at the S3 layer. This is preferred over presigned PUT URLs, which cannot enforce `Content-Length`. The presigned POST also fixes `Content-Type` and the S3 key. The 10-minute policy expiry bounds the upload window. Clients must use `multipart/form-data` POST (not PUT) to upload. +- **Confirm-uploads endpoint.** `POST /v1/tasks/{task_id}/confirm-uploads` triggers screening and transitions to `SUBMITTED`. See [Confirm-uploads concurrency](#confirm-uploads-concurrency) for the race-condition handling. +- **Parallel screening with bounded concurrency.** Attachments are screened in parallel (max 3 concurrent) to reduce wall time. Sequential screening of 10 large images can exceed the Lambda timeout; parallel processing with concurrency 3 keeps worst-case under the timeout budget (see [CDK construct changes](#cdk-construct-changes)). +- **Atomic failure.** If any attachment fails screening, the entire task fails. All uploaded objects are deleted. The user gets a clear error identifying which attachment was blocked and why. +- **HeadObject retry for incomplete uploads.** When `confirm-uploads` is called immediately after the client receives 200 from S3, there is a brief S3 eventual-consistency window where `HeadObject` may return 404. The handler retries `HeadObject` up to 3 times with 1-second delays before concluding the object is truly missing. After retries exhaust, the handler returns `400 ATTACHMENT_SIZE_MISMATCH` with message: "Upload for `{filename}` not found. Ensure the upload completed successfully before calling confirm-uploads." This differentiates "upload in progress" from "upload never happened" (the latter should not retry indefinitely). + +### Confirm-uploads concurrency + +Two concurrent `confirm-uploads` calls can race. The design prevents corruption through three mechanisms: + +1. **Early short-circuit:** The handler reads the task status first. If status is not `PENDING_UPLOADS`, return the current task status immediately (idempotent success for `SUBMITTED`, error for `FAILED`/`CANCELLED`). This avoids redundant screening work. +2. **Conditional DynamoDB write:** The final status transition uses `ConditionExpression: 'status = :pending_uploads'`. Only one caller wins the write. The loser gets `ConditionalCheckFailedException`, which the handler maps to an idempotent success response (re-read the task and return current status). +3. **Safe cleanup via conditional write result:** On screening failure, the handler attempts a conditional DynamoDB write to `FAILED` (`ConditionExpression: 'status = PENDING_UPLOADS'`). If the write **succeeds**, this caller owns the failure — proceed with S3 cleanup. If the write **fails** (`ConditionalCheckFailedException`), another caller already transitioned to `SUBMITTED` — skip cleanup entirely (those objects are needed by the running agent). The conditional write result is the authoritative signal, eliminating the TOCTOU window that would exist with a separate "read status then delete" approach. + +```typescript +// In confirm-uploads handler: +try { + await dynamoClient.send(new UpdateCommand({ + TableName: TASK_TABLE, + Key: { task_id }, + UpdateExpression: 'SET #s = :submitted, ...', + ConditionExpression: '#s = :pending_uploads', + ... + })); +} catch (err) { + if (err instanceof ConditionalCheckFailedException) { + // Another caller already transitioned — return current state (idempotent) + const current = await getTask(task_id); + return toTaskDetailResponse(current); + } + throw err; +} +``` + +### Idempotency key interaction with PENDING_UPLOADS + +When a client retries a task creation with the same idempotency key, the existing task may be in `PENDING_UPLOADS` with expired presigned POST policies. The idempotency check is special-cased: + +| Existing task status | Presigned URLs expired? | Behaviour | +|---|---|---| +| `PENDING_UPLOADS` | No (< 10 min old) | Return existing response with original upload instructions (true idempotency) | +| `PENDING_UPLOADS` | Yes (> 10 min old) | Generate new attachment IDs + new S3 keys, generate new presigned POST policies, update AttachmentRecords in DynamoDB (conditional write: `ConditionExpression: 'status = PENDING_UPLOADS'` — prevents clobbering a concurrent `confirm-uploads` that transitioned to `SUBMITTED`), return updated response. If the conditional write fails, re-read the task and return current status. | +| `SUBMITTED` or later | N/A | Return existing task (standard idempotency) | +| `FAILED` / `CANCELLED` | N/A | Return existing task (standard idempotency) | + +This prevents the deadlock where a client crash leaves an unreachable `PENDING_UPLOADS` task blocking retries for 30 minutes. + +**Presigned POST policy expiry and clock skew:** The 10-minute policy expiry uses the server's clock (AWS SigV4 signing time). Clients with clock skew > 5 minutes may find policies expire earlier than expected. The 10-minute window provides ~5 minutes of effective skew tolerance. Clients that consistently fail uploads should check their system clock (AWS SDK requests also fail with > 15 minutes skew). The CLI should log the server's `Date` header on upload failure to help users diagnose clock issues. + +**Why new S3 keys on retry:** Regenerating presigned POST policies for the same S3 keys creates a collision risk if the first client instance is still alive (e.g., network partition, not a crash). Both instances would upload to the same key, with one overwriting the other. Using new attachment IDs (and therefore new S3 key paths) ensures concurrent client instances cannot interfere. The orphaned objects from the original attempt are cleaned up by the auto-cancel rule or the 90-day lifecycle. + +### Convenience path: Inline base64 (small attachments) + +For attachments <= 500 KB decoded, clients can include base64-encoded content directly in the `POST /v1/tasks` body. This avoids the two-phase round trip for small files like cropped screenshots and short logs. + +```mermaid +sequenceDiagram + participant C as Client + participant GW as API Gateway + participant H as Create-Task Lambda + participant SCR as Screening Pipeline + participant S3 as Attachments Bucket + participant DB as TaskTable + + C->>GW: POST /v1/tasks { attachments: [{ type: 'image', data: 'base64...', content_type: 'image/png' }] } + GW->>H: Forward (body < 6 MB Lambda payload limit) + H->>H: Validate: decode base64, check size <= 500 KB, magic bytes, total inline <= 3 MB + loop Each inline attachment + H->>SCR: Screen content (with retry: 3 attempts, exponential backoff) + SCR-->>H: Pass / Blocked + alt Screening blocked + H->>H: Cleanup already-uploaded S3 objects + H-->>C: 400 ATTACHMENT_BLOCKED + end + H->>H: Strip EXIF (images) / sanitize; on sharp failure → ATTACHMENT_INVALID_CONTENT + H->>S3: PutObject (cleaned content) + H->>H: Build AttachmentRecord + end + H->>DB: Write TaskRecord (status: SUBMITTED, with attachment metadata) + H->>H: Async-invoke Orchestrator + H-->>C: 201 Created +``` + +**Limit rationale:** The binding constraint is the **6 MB Lambda synchronous invocation payload limit** (not the 10 MB API Gateway body limit). The API Gateway forwards the full request body to Lambda as part of the invocation payload. At 500 KB decoded per inline attachment (~667 KB base64), with 3 MB total decoded (~4 MB base64), plus JSON overhead and API Gateway metadata, the total stays safely under 6 MB. In practice, most inline attachments are small screenshots (100-300 KB) where one or two are included alongside a task description. + +### URL fetch (deferred download) + +For `type: 'url'` attachments, the content is fetched during context hydration (not at submission time), because: + +1. The URL may require the repo's GitHub token to access (e.g., private repo assets). +2. Fetching at submission time blocks the API response on external network calls. +3. The content should be fresh at agent execution time, not stale from hours-ago submission. + +```mermaid +sequenceDiagram + participant H as Create-Task Lambda + participant O as Orchestrator (Hydration) + participant SCR as Screening Pipeline + participant S3 as Attachments Bucket + + H->>H: Validate URL format, scheme (HTTPS only) + Note over H: SSRF pre-check is async — runs in a separate step after sync validation + H->>H: Store AttachmentRecord (s3_key: absent, screening: pending) + Note over O: During context hydration + O->>O: SSRF check (DNS resolve → reject private IPs → connect to resolved IP) + O->>O: Fetch URL content (with timeout + size limit) + O->>SCR: Screen fetched content (with retry) + SCR-->>O: Pass / Blocked + alt Screening passed + O->>O: Strip EXIF (images) / sanitize; on sharp failure → fail task + O->>S3: PutObject + O->>O: Update AttachmentRecord + else Screening blocked or fetch failed + O->>O: Throw AttachmentResolutionError → fails the task + end +``` + +**Failure semantics:** A blocked or unfetchable URL attachment **fails the task** — same as inline attachments. The user chose the URL; they expect it to work. The error message identifies the problematic URL and reason (blocked content, fetch timeout, DNS failure, SSRF violation). The user can re-submit without the problematic URL. + +**User notification for async failures:** URL fetch failures occur during hydration (minutes after submission). The user has already received a `201 Created` response. The failure surfaces through: +- Task status transitions to `FAILED` with `error_message` identifying the attachment and reason. +- A `task_failed` event is written to `TaskEventsTable`. +- Notifications (if configured): Slack reply, Linear comment, or webhook callback. +- The CLI `bgagent submit` with URL attachments should poll for the `HYDRATING → RUNNING` transition before returning, surfacing early failures inline. + +**SSRF protections** (applied at fetch-time in the orchestrator Lambda): + +1. **URL scheme:** Only `https` is allowed. No `http`, `file`, `ftp`, or custom schemes. Validated synchronously at submission time. +2. **DNS resolution pinning (prevents DNS rebinding):** The fetch implementation must: (a) resolve DNS manually (e.g., via `dns.resolve4`/`dns.resolve6`), (b) validate the resolved IP against private ranges (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `169.254.0.0/16`, `127.0.0.0/8`, `::1`, `fd00::/8`), (c) connect to the validated IP directly using `undici`'s `connect` option or equivalent — **pinning** the connection to the resolved IP and preventing the HTTP library from doing a second DNS lookup that could return a different (malicious) IP. This technique is called "DNS resolution pinning" and it mitigates the DNS rebinding attack, where an attacker's DNS server returns a public IP on the first lookup (passing validation) then a private IP on the second lookup (reaching internal services). +3. **Redirect limits:** Follow at most 2 redirects. Re-validate the target URL and resolved IP after each redirect. +4. **Timeout and size:** 10-second timeout, 10 MB max response body. Stream the response and abort if the size limit is exceeded. +5. **No auth headers to non-GitHub URLs:** Only GitHub URLs (matching the repo's installation) receive the GitHub token. All other URLs are fetched without credentials. + +**URL fetch policy (open fetch with SSRF protection):** The default policy allows fetching from any public HTTPS URL that passes SSRF validation. This supports real-world use cases (Figma exports, Google Docs, Confluence, etc.) without requiring per-deployment configuration. Repos can optionally restrict to a URL allowlist via blueprint config: + +```json +{ + "attachments": { + "url_allowlist": ["raw.githubusercontent.com", "*.figma.com", "docs.google.com"], + "url_denylist": ["*.internal.company.com"] + } +} +``` + +When `url_allowlist` is set, only matching URLs are allowed. When absent (default), any public HTTPS URL passing SSRF checks is permitted. `url_denylist` is always evaluated regardless of allowlist. + +## Security screening pipeline + +Every attachment passes through a type-specific screening pipeline before its binary content reaches durable S3 storage or the agent. The pipeline is fail-closed: if any screening step is unavailable after retries, the attachment (and the task) is rejected. + +### Retry strategy + +All Bedrock `ApplyGuardrailCommand` calls use exponential backoff before failing closed: + +- **Max retries:** 3 +- **Backoff:** 200ms, 400ms, 800ms +- **Retryable errors:** HTTP 429 (throttling), HTTP 5xx (transient service errors) +- **Non-retryable errors:** HTTP 4xx (except 429), validation errors, content policy violations (`INTERVENED`) + +This prevents a single transient Bedrock hiccup from failing an entire task after the user waited for 10 attachments to upload. + +### Image screening + +```mermaid +flowchart TD + I[Image attachment] --> MB[Magic bytes: verify image signature] + MB -->|Invalid| R[REJECTED: not a valid image] + MB -->|Valid| V[Validate: decodable, dimensions ≤ 8192x8192] + V --> G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] + G -->|INTERVENED| B[BLOCKED: content policy violation] + G -->|NONE| M[EXIF strip + re-encode via sharp] + M -->|sharp error| SE[REJECTED: ATTACHMENT_INVALID_CONTENT] + M -->|Success| P[PASSED] + G -->|Error after retries| F[FAIL CLOSED: 503] +``` + +**Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the image processing pipeline. + +**Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks. **Prerequisite:** Verify that the installed version of `@aws-sdk/client-bedrock-runtime` in `cdk/package.json` supports image content blocks in `ApplyGuardrail`. If it does not, a dependency upgrade is required before Phase 1. + +**Format limitation:** The Bedrock `GuardrailImageBlock` API **only supports `png` and `jpeg` formats**. GIF and WebP images must be converted to PNG via `sharp` before screening. This conversion happens before the `ApplyGuardrailCommand` call (not after — the screened content must be the same content that reaches the agent): + +```typescript +// Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) +let screeningBuffer: Buffer; +let screeningFormat: 'png' | 'jpeg'; + +if (contentType === 'image/jpeg') { + screeningBuffer = imageBuffer; + screeningFormat = 'jpeg'; +} else if (['image/gif', 'image/webp'].includes(contentType)) { + // GIF/WebP → PNG. For animated GIFs, extract first frame only (prevents OOM). + screeningBuffer = await sharp(imageBuffer, { animated: false }).png().toBuffer(); + screeningFormat = 'png'; + + // Post-conversion size check: PNG expansion of compressed GIF/WebP can exceed 10 MB. + // Fail with a clear error rather than letting a downstream size check reject it opaquely. + if (screeningBuffer.length > MAX_ATTACHMENT_SIZE_BYTES) { + throw new AttachmentResolutionError( + `Image "${filename}" is ${contentType} and its PNG conversion for screening ` + + `exceeds the ${MAX_ATTACHMENT_SIZE_BYTES / (1024 * 1024)} MB limit ` + + `(${(screeningBuffer.length / (1024 * 1024)).toFixed(1)} MB after conversion). ` + + `Please convert to JPEG or reduce image dimensions before uploading.` + ); + } +} else { + // PNG: use as-is + screeningBuffer = imageBuffer; + screeningFormat = 'png'; +} + +const result = await retryWithBackoff(() => + bedrockClient.send(new ApplyGuardrailCommand({ + guardrailIdentifier: GUARDRAIL_ID, + guardrailVersion: GUARDRAIL_VERSION, + source: 'INPUT', + content: [{ + image: { + format: screeningFormat, + source: { bytes: screeningBuffer }, + }, + }], + })), + { maxRetries: 3, baseDelayMs: 200, retryableErrors: [429, 500, 502, 503] }, +); +``` + +The GIF/WebP → PNG conversion uses `{ animated: false }` to extract only the first frame from animated GIFs, preventing unbounded memory usage. The post-conversion size check catches cases where a highly-compressed GIF/WebP expands beyond 10 MB as PNG — the user gets a clear error with remediation (convert to JPEG or reduce dimensions). The conversion error path (`ATTACHMENT_INVALID_CONTENT`) is shared with the EXIF stripping pipeline. + +**EXIF stripping + re-encoding:** After screening passes, the image is processed through `sharp`: +1. Strip all EXIF/IPTC/XMP metadata (GPS coordinates, device info, timestamps — prevents PII leakage). +2. Re-encode the image in the same format. This strips any non-image data that may have been appended to the file (steganography payloads, polyglot trailing data). +3. The re-encoded image is what gets stored in S3 and delivered to the agent. + +**sharp failure handling:** If `sharp` cannot process an image (corrupt image that passes magic bytes check, OOM on large image, library bug), the attachment is **rejected** with `ATTACHMENT_INVALID_CONTENT` and message: "Image could not be processed for security sanitization. Please re-export the image in a standard format and try again." This preserves the security guarantee — no un-sanitized images reach the agent. Fail-closed, not fail-open. + +### File screening + +```mermaid +flowchart TD + F[File attachment] --> MB[Magic bytes: verify against claimed MIME type] + MB -->|Mismatch| R[REJECTED: content does not match declared type] + MB -->|Match| T{Text-based?} + T -->|Yes| TG[Bedrock Guardrail: text content screening, retries] + T -->|No - PDF| PDF[Extract text via pdf-parse, max 50 pages / 1 MB output] + PDF --> TG + TG -->|INTERVENED| B[BLOCKED] + TG -->|NONE| P[PASSED] + TG -->|Error after retries| FC[FAIL CLOSED: 503] +``` + +**Magic bytes validation:** Don't trust `content_type` from the client. Validate the first bytes of the content against known signatures for allowed types: + +| MIME type | Expected magic bytes | +|---|---| +| `application/pdf` | `%PDF-` | +| `application/json` | Starts with `{`, `[`, or UTF-8 BOM + `{`/`[` | +| `text/*` | Valid UTF-8, no null bytes in first 8 KB | +| `image/png` | `\x89PNG\r\n\x1a\n` | +| `image/jpeg` | `\xFF\xD8\xFF` | +| `image/gif` | `GIF87a` or `GIF89a` | +| `image/webp` | `RIFF....WEBP` | + +A file claiming to be `text/plain` but starting with `MZ` (PE executable) or `PK` (ZIP) is rejected immediately. + +**Text content screening:** For text-based files (plain text, CSV, Markdown, JSON), the full content is screened through the same Bedrock Guardrail used for task descriptions. For PDFs, text is extracted first (using `pdf-parse`, capped at 50 pages and 1 MB extracted text output to prevent decompression bombs) and then screened. + +**PDF extraction failure handling:** `pdf-parse` is wrapped in a try-catch with a 15-second timeout. Corrupt PDFs, deeply nested objects, or excessive embedded fonts can cause OOM before the page limit takes effect. On any `pdf-parse` failure (exception, timeout, or OOM), the attachment is **rejected** with `ATTACHMENT_INVALID_CONTENT` and message: "PDF could not be processed. It may be corrupt or use unsupported features. Try exporting to a simpler PDF format." Consider running `pdf-parse` in a child process with a memory limit to prevent Lambda OOM when the confirm-uploads Lambda is processing multiple PDFs concurrently. + +**No executable content:** The MIME allowlist explicitly excludes executables, archives, and scripts. This is a hard boundary — there is no override. If a user needs to share a shell script, they should paste it as text in the task description or commit it to the repo. + +### Screening result caching + +Screening results are not cached. Each attachment is screened exactly once, at upload/fetch time. The `screening` field in `AttachmentRecord` records the outcome for audit purposes. Re-screening is not needed because: + +- Attachments are immutable once stored (no update API). +- Guardrail policies may change, but retroactive re-screening of existing attachments is a separate concern (batch job, not inline). + +## Storage: Attachments S3 bucket + +A new CDK construct, `AttachmentsBucket`, following the same pattern as `TraceArtifactsBucket`: + +```typescript +// cdk/src/constructs/attachments-bucket.ts + +/** Props interface mirrors TraceArtifactsBucketProps for consistency. */ +export interface AttachmentsBucketProps { + readonly removalPolicy?: RemovalPolicy; // Default: DESTROY (dev-friendly; override for prod) + readonly autoDeleteObjects?: boolean; // Default: true (matches removalPolicy: DESTROY) +} + +export class AttachmentsBucket extends Construct { + public readonly bucket: s3.Bucket; + + constructor(scope: Construct, id: string, props: AttachmentsBucketProps = {}) { + super(scope, id); + + this.bucket = new s3.Bucket(this, 'Bucket', { + encryption: s3.BucketEncryption.S3_MANAGED, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + versioned: true, // Required: pins object versions at screening time to prevent TOCTOU + lifecycleRules: [ + { + expiration: Duration.days(ATTACHMENT_TTL_DAYS), // 90 days — matches task retention + noncurrentVersionExpiration: Duration.days(7), // Noncurrent versions kept 7 days (see rationale below) + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, + autoDeleteObjects: props.autoDeleteObjects ?? true, + }); + } +} +``` + +**S3 versioning (TOCTOU prevention):** The bucket has versioning enabled. This prevents a class of attack where a client uploads benign content (passes screening), then replaces the S3 object with malicious content before the agent downloads it. The mitigation flow: + +1. At `confirm-uploads` time, `HeadObject` records the `VersionId`. +2. `GetObject` for screening specifies `VersionId` — screens exactly the uploaded version. +3. After screening passes, the `VersionId` is stored in `AttachmentRecord.s3_version_id`. +4. The `AgentAttachmentPayload` includes `s3_version_id`. The agent downloads with `VersionId` pinned. +5. Even if the presigned URL is still valid and the client uploads a second version, the agent always downloads the screened version. +6. `noncurrentVersionExpiration: 7 days` — old versions are cleaned up after a safe window. The 7-day retention ensures that even the longest-running tasks (max_turns=500 with slow models, potentially 24-48 hours) can always access the screened version. After 7 days, any task still referencing a noncurrent version is either completed or stuck in a terminal state. + +**Lifecycle:** 90 days, matching the task record TTL (confirmed: `TASK_RETENTION_DAYS` defaults to 90 in `task-api.ts`). When a task expires from DynamoDB, its attachments expire from S3 around the same time. No cross-resource cleanup needed. + +**IAM grants:** + +| Principal | Grant | Purpose | +|---|---|---| +| Create-task Lambda | `grantPut`, `grantDelete` | Upload inline attachments; delete on partial failure cleanup | +| Confirm-uploads Lambda | `grantRead`, `grantPut`, `grantDelete` | Read for screening; re-upload cleaned images; delete blocked content | +| Orchestrator Lambda | `grantReadWrite` | Fetch URL attachments during hydration; write screened content | +| Agent runtime (AgentCore / ECS) | `grantRead` | Download attachments into workspace via IAM role | + +**Presigned POST policy generation:** + +```typescript +import { createPresignedPost } from '@aws-sdk/s3-presigned-post'; + +const { url, fields } = await createPresignedPost(s3Client, { + Bucket: ATTACHMENTS_BUCKET, + Key: s3Key, + Conditions: [ + ['content-length-range', 1, MAX_ATTACHMENT_SIZE_BYTES], // 1 byte to 10 MB + ['eq', '$Content-Type', declaredMimeType], + ], + Fields: { + 'Content-Type': declaredMimeType, + }, + Expires: 600, // 10 minutes +}); +// Presigned POST policies enforce content-length-range at the S3 layer. +// S3 rejects uploads outside the declared range BEFORE storing the object. +// The Content-Type is fixed by the policy — mismatched uploads fail. +// Clients must POST with multipart/form-data (not PUT). +``` + +**Why presigned POST over presigned PUT:** Presigned PUT URLs (`getSignedUrl` with `PutObjectCommand`) **cannot enforce `Content-Length` conditions** — the only way to limit upload size with PUT is a bucket resource policy using `s3:content-length-range`, but this condition key is **not documented for `s3:PutObject` actions in bucket policies** and its behavior is unreliable. Presigned POST policies, in contrast, have `content-length-range` as a [documented, first-class condition](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html#sigv4-PolicyConditions) that S3 enforces server-side before writing the object. This provides reliable, documented size enforcement at the S3 layer. + +**Client upload format:** Clients upload via `POST` with `multipart/form-data`, including the policy fields returned in the creation response. The CLI and webhook documentation include code examples. This is a minor ergonomic tradeoff (form-data POST vs raw PUT) for guaranteed size enforcement. + +**Defense-in-depth:** Even with presigned POST size enforcement, the `confirm-uploads` handler still validates object sizes via `HeadObject` (defense-in-depth against implementation bugs). The auto-cancel cleanup Lambda also deletes ALL objects under a task's `attachments///` prefix (not just confirmed ones), catching objects from tasks that were never confirmed. + +## Orchestrator integration + +### Hydration error handling + +The existing `hydrateContext()` has a broad infrastructure-failure catch block (context-hydration.ts lines 1157-1189) that returns minimal context (task_description only) and sets `fallback_error` — allowing the task to proceed with degraded context. This is acceptable for optional context (e.g., a memory lookup failure), but **not acceptable for attachments**. + +If the user explicitly provided attachments, proceeding without them produces incorrect agent output. Attachment resolution errors must **propagate and fail the task**, not be caught by the generic fallback. + +**Implementation (Phase 1, step 10 — ships before any code throws these errors):** Create an `AttachmentError` base class. All attachment-related errors (`AttachmentResolutionError`, `AttachmentBudgetExceededError`, `AttachmentDownloadError`, `AttachmentIntegrityError`) extend this base class. A single `instanceof AttachmentError` check in the hydration catch block covers all current and future attachment error types, avoiding a growing allowlist. This is wired into the catch block in Phase 1 (alongside the class definitions), not deferred — ensuring no deployment window exists where typed errors are thrown but not re-thrown: + +```typescript +// Error class hierarchy: +export class AttachmentError extends Error { } +export class AttachmentResolutionError extends AttachmentError { } +export class AttachmentBudgetExceededError extends AttachmentError { } + +// In hydrateContext() catch block: +} catch (err) { + if ( + err instanceof GuardrailScreeningError || + err instanceof AttachmentError || // NEW: covers all attachment errors (budget, resolution, integrity) + err instanceof TypeError || + err instanceof RangeError || + err instanceof ReferenceError + ) { + throw err; // Do not swallow — these must fail the task + } + // Only non-critical infrastructure errors fall through to the fallback path + ... +} +``` + +Using a base class instead of individual `instanceof` checks prevents the bug where a new error type (e.g., `AttachmentBudgetExceededError`) is added to `resolveAttachments()` but not to the re-throw list — causing it to be silently swallowed by the infrastructure fallback. + +### Context hydration changes + +The `hydrateContext()` function gains an attachment resolution step that runs in parallel with issue/PR/memory fetching: + +```typescript +// In hydrateContext(), added to the parallel fetch block: +const resolvedAttachments = await resolveAttachments( + task.attachments, // AttachmentRecord[] from TaskRecord + attachmentsBucket, + githubToken, // For URL fetches requiring auth + bedrockClient, + guardrailConfig, +); +``` + +`resolveAttachments()` handles: + +1. **Inline/presigned attachments (already screened, already in S3):** Validate S3 key exists, compute `s3_uri` for the agent. +2. **URL attachments (not yet fetched):** Fetch (with SSRF protections), screen (with retry), sanitize (EXIF strip / re-encode), upload to S3, compute `s3_uri`. On any failure, throw `AttachmentResolutionError` which fails the task. +3. **Token budget accounting:** Estimate token cost of image attachments and deduct from the available prompt budget (see [Token budget](#token-budget-accounting)). + +### Payload changes + +The orchestrator payload to the agent gains a top-level `attachments` field using the named `AgentAttachmentPayload` interface: + +```typescript +const payload = { + // ... existing fields ... + attachments: resolvedAttachments.map(a => ({ + attachment_id: a.attachment_id, + type: a.type, + content_type: a.content_type, + filename: a.filename, + s3_uri: a.s3_uri, + s3_version_id: a.s3_version_id, + size_bytes: a.size_bytes, + source_url: a.source_url, + token_estimate: a.token_estimate, + checksum_sha256: a.checksum_sha256, + } satisfies AgentAttachmentPayload)), +}; +``` + +**Top-level placement rationale:** Attachments are placed at the top level of the agent payload (alongside `repo_url`, `task_type`, etc.), **not** inside `HydratedContext`. This avoids: + +- A `HydratedContext` version bump and the deployment ordering constraint it creates (agent image must deploy before orchestrator). +- Pydantic `extra="forbid"` rejection — `HydratedContext` uses `extra="forbid"` (models.py line 68), so adding a field there without updating the agent would crash it. The `TaskConfig` model (models.py line 94) uses only `validate_assignment=True` without `extra="forbid"`, so Pydantic v2's default behaviour (`extra="ignore"`) silently discards unrecognized top-level fields on old agents. +- Conflating raw inputs (attachments) with assembled prompt context (issue body, PR comments, memory). + +### Agent capability check + +While old agents silently ignore the `attachments` top-level field (due to Pydantic `extra="ignore"` default), this produces a bad user experience: the user's attachments have no effect, with no error or warning. To prevent this during incremental rollout: + +**The orchestrator checks the agent's deployment version before including attachments in the payload.** The mechanism: + +1. The agent container image is tagged with a version identifier (already tracked via `prompt_version` in the payload). +2. The orchestrator checks the `prompt_version` (or a new `agent_capabilities` field in the blueprint config) to determine attachment support. +3. If the agent does not support attachments AND the task has attachments, the orchestrator fails the task with: `"Agent runtime version does not support attachments. A deployment is required to enable this feature for repository {repo}."`. +4. If the task has no attachments, the orchestrator proceeds normally regardless of agent version. + +This ensures users never silently lose their attachments during the rollout window. + +### No HydratedContext version bump needed + +Because attachments live at the top level of the payload (not inside `hydrated_context`), no version bump is required. The `HydratedContext` Pydantic model with `extra="forbid"` and `version: 1` remains unchanged. This eliminates the deployment ordering constraint. + +### Token budget accounting + +Images consume tokens when sent as multimodal content blocks. The system's `USER_PROMPT_TOKEN_BUDGET` (default 100K tokens, configured via environment variable) must account for image token costs. + +**Image token estimation:** + +Claude resizes images before tokenizing. The estimation must apply the same resizing rules. Per Anthropic's documentation (as of May 2025): + +1. If either dimension exceeds 1568px, scale down proportionally to fit 1568px on the longest side. +2. Pad each dimension up to the next multiple of 28 pixels. +3. Apply: `ceil(resized_width * resized_height / 750)` tokens, capped at 1568 tokens. +4. Apply a **1.2x safety margin** to account for padding imprecision and API changes. + +**Note:** The documentation describes a single resize step (fit 1568px longest side) and a hard token cap at 1568. An earlier version of this design included a separate 1.15 megapixel step; that has been removed to match the documented behaviour. If targeting Claude Opus 4.7 (which supports 2576px / 4784 tokens max), the constants should be made configurable. + +```typescript +const MAX_IMAGE_SIDE = 1568; // Standard models; Opus 4.7 uses 2576 +const MAX_IMAGE_TOKENS = 1568; // Standard models; Opus 4.7 uses 4784 +const TOKEN_SAFETY_MARGIN = 1.2; +const TILE_SIZE = 28; + +function estimateImageTokens(width: number, height: number): number { + let w = width; + let h = height; + + // Step 1: Scale to fit MAX_IMAGE_SIDE on longest side + const maxSide = Math.max(w, h); + if (maxSide > MAX_IMAGE_SIDE) { + const scale = MAX_IMAGE_SIDE / maxSide; + w = Math.round(w * scale); + h = Math.round(h * scale); + } + + // Step 2: Pad to next multiple of 28 pixels (tile alignment) + w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; + h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; + + // Step 3: Token calculation with safety margin, capped + const rawTokens = Math.min(Math.ceil((w * h) / 750), MAX_IMAGE_TOKENS); + return Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN); +} +``` + +For standard image sizes (after resizing): + +| Original dimensions | Resized to (pre-pad) | Padded to | Estimated tokens (with margin) | +|---|---|---|---| +| 1920x1080 (full screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | +| 3840x2160 (4K screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | +| 800x600 (cropped screenshot) | 800x600 (no resize) | 812x616 | ~800 | +| 4096x4096 (max-size design) | 1568x1568 | 1568x1568 | ~1,882 (capped) | + +**Note:** The 4K screenshot and full HD screenshot produce the **same** token cost after resizing — both scale down to the same dimensions. The token cap at 1568 (+ safety margin) means very large or square images plateau rather than growing linearly. + +**Budget enforcement in attachment resolution:** + +```typescript +async function resolveAttachments(attachments, ...) { + let attachmentTokenBudget = 0; + + for (const att of attachments) { + if (att.type === 'image') { + // getImageDimensions uses sharp.metadata() on the S3 content. + // If dimensions cannot be determined (corrupt image, unsupported format variant), + // throw AttachmentResolutionError — never default to (0,0) or skip the estimate. + let width: number, height: number; + try { + ({ width, height } = await getImageDimensions(att)); + } catch (err) { + throw new AttachmentResolutionError( + `Cannot determine dimensions for image "${att.filename}". ` + + `The image may be corrupt or in an unsupported format variant. ` + + `Re-export the image and try again.`, + { cause: err }, + ); + } + const tokenCost = estimateImageTokens(width, height); + att.token_estimate = tokenCost; + attachmentTokenBudget += tokenCost; + } + } + + // Reserve tokens for attachments; reduce available budget for text context + const availableForText = USER_PROMPT_TOKEN_BUDGET - attachmentTokenBudget; + + if (availableForText < MIN_TEXT_TOKEN_BUDGET) { // MIN_TEXT_TOKEN_BUDGET = 20,000 + throw new AttachmentBudgetExceededError( + `Image attachments require ~${attachmentTokenBudget} tokens, ` + + `leaving insufficient budget for task context (minimum ${MIN_TEXT_TOKEN_BUDGET} required). ` + + `Reduce image count or dimensions.` + ); + } + + return { resolvedAttachments, attachmentTokenBudget, availableForText }; +} +``` + +**Policy:** If image attachments consume more than `USER_PROMPT_TOKEN_BUDGET - MIN_TEXT_TOKEN_BUDGET` tokens (i.e., they would leave fewer than 20K tokens for text context), the task fails with a clear error. The user can reduce image count or downscale images before resubmitting. + +**Token budget vs. payload size:** The token budget above measures **vision tokens** (based on pixel dimensions). This is separate from the **API payload size**, which is affected by base64 encoding overhead (~33% expansion). Image attachments are sent as multimodal content blocks with base64-encoded data, so a 10 MB image becomes ~13.3 MB in the API request. The Anthropic API has its own request size limits (separate from our Lambda payload limits). The `MAX_ATTACHMENT_SIZE_BYTES` (10 MB) is chosen to ensure that even after base64 expansion, individual images stay within the Anthropic API's per-image limits. For multiple large images, the total base64-encoded payload is bounded by the 50 MB total task limit (which produces ~67 MB base64), but in practice the vision token budget is the binding constraint — 10 full-resolution images would consume ~18,820 vision tokens (well within the 100K budget) but produce a very large API payload. The agent should stream images from local files rather than holding all base64 data in memory simultaneously. + +The `availableForText` budget is passed to the existing text trimming logic (`enforceTokenBudget`), which trims issue comments and PR threads to fit within the remaining allocation. + +## Agent consumption + +The agent pipeline (`pipeline.py`) gains an attachment preparation step that runs after repo clone and before the agent session: + +### Step 1: Download attachments from S3 with integrity verification + +The agent runtime has an IAM role with `s3:GetObject` permission on the attachments bucket. It downloads attachments using the AWS SDK — no presigned URLs, no expiry concerns. + +```python +async def prepare_attachments( + attachments: list[AttachmentConfig], + workspace_dir: Path, + s3_client: S3Client, +) -> list[PreparedAttachment]: + """Download attachments from S3 into the workspace with integrity checks.""" + attachments_dir = workspace_dir / ".attachments" + attachments_dir.mkdir(exist_ok=True) + + prepared = [] + for att in attachments: + local_path = attachments_dir / f"{att.attachment_id}_{att.filename}" + bucket, key = parse_s3_uri(att.s3_uri) + try: + # Download the pinned version — prevents TOCTOU between screening and download + await s3_client.download_file( + bucket, key, str(local_path), + ExtraArgs={"VersionId": att.s3_version_id}, + ) + except ClientError as e: + raise AttachmentDownloadError( + f"Failed to download attachment {att.filename}: {e}" + ) from e + + # Verify integrity via SHA-256 checksum (always present — required by factory) + # hexdigest() returns lowercase hex, matching the format enforced by AttachmentConfig validator + actual_hash = hashlib.sha256(local_path.read_bytes()).hexdigest() + if actual_hash != att.checksum_sha256: + raise AttachmentIntegrityError( + f"Checksum mismatch for {att.filename}: " + f"expected {att.checksum_sha256}, got {actual_hash}" + ) + + prepared.append(PreparedAttachment( + attachment_id=att.attachment_id, + type=att.type, + content_type=att.content_type, + filename=att.filename, + local_path=local_path, + token_estimate=att.token_estimate, + )) + return prepared +``` + +**Error handling:** `AttachmentDownloadError` and `AttachmentIntegrityError` are fatal — the task fails with a clear error. The agent does not proceed with missing or corrupted attachments. + +Attachments are downloaded to `.attachments/` in the workspace root. This directory is `.gitignore`d by the agent to prevent accidentally committing binary attachments. + +**No presigned URL expiry problem:** Because the agent downloads via IAM role credentials (which auto-rotate via the instance metadata service or ECS task role), there is no expiry window. Whether the task starts immediately or after a 4-hour approval wait (Change Manifest `AWAITING_APPROVAL` state), the download works identically. + +### Python models + +```python +class AttachmentConfig(BaseModel): + """Attachment descriptor from the orchestrator payload.""" + model_config = ConfigDict(frozen=True, extra="forbid") + + attachment_id: str + type: Literal["image", "file", "url"] + content_type: str + filename: str + s3_uri: str + s3_version_id: str # Pinned S3 object version — prevents TOCTOU + size_bytes: int + source_url: str | None = None + token_estimate: int | None = None + checksum_sha256: str # Required — lowercase hex-encoded SHA-256 (64 chars, e.g., "a1b2c3...") + + @model_validator(mode="after") + def _validate_invariants(self) -> Self: + if self.type == "image" and self.token_estimate is None: + raise ValueError("Image attachments must have token_estimate") + # checksum_sha256 must be lowercase hex, exactly 64 characters + import re + if not re.fullmatch(r"[0-9a-f]{64}", self.checksum_sha256): + raise ValueError( + f"checksum_sha256 must be 64 lowercase hex characters, got: {self.checksum_sha256!r}" + ) + return self + + +class PreparedAttachment(BaseModel): + """Attachment downloaded to the local workspace.""" + model_config = ConfigDict(frozen=True, extra="forbid") + + attachment_id: str + type: Literal["image", "file", "url"] + content_type: str + filename: str + local_path: Path + token_estimate: int | None = None + + @model_validator(mode="after") + def _validate_path_exists(self) -> Self: + if not self.local_path.exists(): + raise ValueError(f"Attachment file not found: {self.local_path}") + return self +``` + +Both models use `frozen=True` and `extra="forbid"`, matching existing patterns in `models.py` (`HydratedContext`, `GitHubIssue`, `MemoryContext`). + +`TaskConfig` gains an optional field: + +```python +class TaskConfig(BaseModel): + # ... existing fields ... + attachments: list[AttachmentConfig] = Field(default_factory=list) +``` + +### Step 2: Inject into agent prompt + +Attachments are referenced in the system prompt and optionally injected as multimodal content blocks in the user message: + +**Image attachments → multimodal content blocks:** + +```python +# In build_user_message(): +content_blocks = [{"type": "text", "text": user_prompt}] + +for att in prepared_attachments: + if att.type == "image": + image_data = att.local_path.read_bytes() + content_blocks.append({ + "type": "image", + "source": { + "type": "base64", + "media_type": att.content_type, + "data": base64.b64encode(image_data).decode(), + }, + }) + +# Append text listing all attachments +attachment_listing = build_attachment_listing(prepared_attachments) +content_blocks[0]["text"] += f"\n\n{attachment_listing}" +``` + +**File attachments → local file references:** + +``` +The following attachments are available in .attachments/: +- screenshot.png (image/png, 245 KB) — included as image content above +- error-log.txt (text/plain, 12 KB) — read with: Read .attachments/01J5X8_error-log.txt +- data.csv (text/csv, 1.2 MB) — read with: Read .attachments/01J5X9_data.csv +``` + +### Agent tool access + +No new tools are needed. The agent uses its existing `Read` tool to access file attachments and sees image attachments directly in the conversation (multimodal content blocks). The `.attachments/` directory is within the workspace and already covered by the agent's filesystem access policy. + +## Channel-specific behaviour + +### CLI (`bgagent submit`) + +New `--attachment` flag (repeatable): + +```bash +# Local files (auto-detects inline vs presigned based on size) +bgagent submit --repo org/app --description "Fix this bug" \ + --attachment screenshot.png \ + --attachment error.log + +# URL reference +bgagent submit --repo org/app --description "Implement this design" \ + --attachment https://figma.com/file/abc123/export.png +``` + +The CLI detects whether the argument is a local file path or URL: +- **Local file <= 500 KB:** Read, base64-encode, detect MIME type from extension/magic bytes, send inline as `{ type: 'image'|'file', data: '...', content_type: '...', filename: '...' }` +- **Local file > 500 KB:** Send metadata only (with `expected_size_bytes`) in task creation, receive presigned POST policy (URL + form fields), upload directly to S3 via multipart form POST, call confirm-uploads. +- **URL:** Send as `{ type: 'url', url: '...' }` + +The CLI enforces size limits client-side (fail fast with a clear error rather than uploading 10 MB only to get a rejection). Progress bars show upload status for large files. + +**URL attachment polling:** When the task includes URL attachments (fetched asynchronously during hydration), the CLI polls for the `HYDRATING → RUNNING` transition before returning. If the task fails during hydration (attachment fetch/screening failure), the error is surfaced inline to the user. + +### Webhook + +Same `attachments` array in the JSON body. For attachments > 500 KB, webhook callers must use the presigned upload flow (same two-phase pattern as CLI). The webhook documentation will include code examples in Python and Node.js. + +### Slack + +When a user mentions `@Shoof` in a message that contains file uploads or image attachments: + +1. The `slack-events.ts` handler extracts `event.files[]` from the Slack event. +2. For each file, it calls the Slack API (`files.info` or uses the `url_private_download`) to get the file content. +3. Files are validated against size limits (10 MB per file). Oversized files are rejected with a Slack reply: "Attachment `{filename}` is too large (max 10 MB). Please reduce the file size or link to it instead." +4. Files are uploaded to S3 directly (the Slack handler Lambda has `grantPut`), screened, and converted to `AttachmentRecord` entries. +5. The records are passed to `createTaskCore()`. + +**Slack error surface — atomic failure (consistent with design principle 3):** If any attachment fails validation, screening, or upload, the **entire task is rejected** — no partial attachment submission. This matches the behaviour of all other channels (CLI, webhook, API). The Slack reply lists all failures so the user can fix them in one re-submission: + +| Failure | Slack reply | +|---|---| +| File too large | "Task not created. `{filename}` is too large ({size} MB, max 10 MB). Please reduce the file size or remove it and try again." | +| Screening blocked | "Task not created. `{filename}` was blocked by content screening ({categories}). Please remove this file and try again." | +| Unsupported MIME type | "Task not created. `{filename}` has unsupported type `{mime}`. Supported: images (png, jpeg, gif, webp) and text files (txt, csv, json, md, pdf, log)." | +| S3 upload failure | "Task not created. Failed to process `{filename}`. Please try again." | +| Multiple failures | "Task not created. 2 attachment errors: `{file1}` (blocked by content screening), `{file2}` (too large, 15 MB > 10 MB limit). Fix or remove these files and try again." | + +**Note:** Slack files bypass the inline base64 path (they go directly from Slack's CDN to our S3 via the Lambda). This avoids the 500 KB inline limit. + +### Linear + +When a Linear issue triggers task creation and the issue body contains embedded images: + +1. The `linear-webhook-processor.ts` extracts image URLs from the issue body markdown (pattern: `![...](url)`). +2. Each image URL becomes `{ type: 'url', url: '...' }` in the attachments array. +3. Images are fetched and screened during context hydration, following the URL fetch flow. + +Linear issue attachments (non-inline files) are not supported in v1, as the Linear API requires separate API calls to list attachments per issue. This can be added later. + +## Validation changes + +Validation is split into two steps to preserve the existing codebase pattern where validation functions are synchronous pure functions: + +**Step 1 — Synchronous validation** (in `validation.ts`): + +```typescript +const MAX_ATTACHMENTS_PER_TASK = 10; +const MAX_INLINE_ATTACHMENT_SIZE_BYTES = 500 * 1024; // 500 KB +const MAX_TOTAL_INLINE_SIZE_BYTES = 3 * 1024 * 1024; // 3 MB +const MAX_ATTACHMENT_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB +const MAX_TOTAL_ATTACHMENT_SIZE_BYTES = 50 * 1024 * 1024; // 50 MB +const MAX_TASK_DESCRIPTION_LENGTH = 10_000; // Increased from 2,000 + +export function validateAttachments( + attachments: unknown[] | undefined, +): { valid: true; parsed: ValidatedAttachment[] } | { valid: false; error: string } { + if (!attachments) return { valid: true, parsed: [] }; + if (!Array.isArray(attachments)) return { valid: false, error: 'attachments must be an array' }; + if (attachments.length > MAX_ATTACHMENTS_PER_TASK) { + return { valid: false, error: `Maximum ${MAX_ATTACHMENTS_PER_TASK} attachments per task` }; + } + + let totalInlineSize = 0; + let totalDeclaredSize = 0; + const parsed: ValidatedAttachment[] = []; + + for (const [i, att] of attachments.entries()) { + // Type validation + if (!att.type || !['image', 'file', 'url'].includes(att.type)) { + return { valid: false, error: `attachments[${i}].type must be 'image', 'file', or 'url'` }; + } + + // Mutual exclusivity: data vs url + if (att.type === 'url') { + if (!att.url) return { valid: false, error: `attachments[${i}]: url required for type 'url'` }; + if (att.data) return { valid: false, error: `attachments[${i}]: data not allowed for type 'url'` }; + if (!isValidHttpsUrl(att.url)) return { valid: false, error: `attachments[${i}]: must be a valid HTTPS URL` }; + // NOTE: SSRF DNS check is async — runs in step 2, not here + } else { + if (att.data && att.url) { + return { valid: false, error: `attachments[${i}]: provide data or url, not both` }; + } + } + + // Decode inline data (hoisted to loop scope — used by size check, magic bytes, MIME detection, and variant construction) + let decoded: Buffer | undefined; + + // Size validation (for inline data) + if (att.data) { + decoded = Buffer.from(att.data, 'base64'); + if (decoded.length > MAX_INLINE_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `attachments[${i}]: inline data exceeds 500 KB limit. Use presigned upload for larger files.` }; + } + // Magic bytes validation + if (!validateMagicBytes(decoded, att.content_type ?? att.type)) { + return { valid: false, error: `attachments[${i}]: content does not match declared type` }; + } + totalInlineSize += decoded.length; + } + + // Declared size validation (for presigned uploads) + if (!att.data && !att.url && att.type !== 'url') { + // Presigned upload — expected_size_bytes required for early budget check + if (typeof att.expected_size_bytes !== 'number' || att.expected_size_bytes <= 0) { + return { valid: false, error: `attachments[${i}]: expected_size_bytes required for presigned uploads` }; + } + if (att.expected_size_bytes > MAX_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `attachments[${i}]: expected size exceeds 10 MB limit` }; + } + totalDeclaredSize += att.expected_size_bytes; + } + + // MIME type resolution and validation + // content_type is required after validation; detect from magic bytes if absent + let resolvedContentType: string; + if (att.content_type) { + if (!isAllowedMimeType(att.content_type, att.type)) { + return { valid: false, error: `attachments[${i}]: content_type '${att.content_type}' not allowed for type '${att.type}'` }; + } + resolvedContentType = att.content_type; + } else if (att.data && decoded) { + // Auto-detect from magic bytes for inline attachments (decoded is set above) + const detected = detectMimeTypeFromMagicBytes(decoded); + if (!detected) { + return { valid: false, error: `attachments[${i}]: could not determine file type. Please provide content_type explicitly.` }; + } + resolvedContentType = detected; + } else { + // Presigned uploads and URLs must declare content_type + return { valid: false, error: `attachments[${i}]: content_type is required for presigned uploads and URL attachments` }; + } + + // Filename resolution (required after validation; generate if absent) + const resolvedFilename = att.filename ?? generateFilename(att.type, resolvedContentType, i); + if (!isValidFilename(resolvedFilename)) { + return { valid: false, error: `attachments[${i}]: invalid filename` }; + } + + // Construct validated variant explicitly — "parse, don't validate" pattern. + // This ensures the discriminated union invariants hold at runtime, not just via a cast. + if (att.type === 'url') { + parsed.push({ + delivery: 'url_fetch', + type: 'url', + url: att.url, + filename: resolvedFilename, + content_type: resolvedContentType, + } satisfies UrlAttachment); + } else if (att.data && decoded) { + parsed.push({ + delivery: 'inline', + type: att.type, + data: att.data, + filename: resolvedFilename, + content_type: resolvedContentType, + decoded_size_bytes: decoded.length, // decoded is hoisted and set in the size validation block above + } satisfies InlineAttachment); + } else { + parsed.push({ + delivery: 'presigned', + type: att.type, + filename: resolvedFilename, + content_type: resolvedContentType, + expected_size_bytes: att.expected_size_bytes, + } satisfies PresignedAttachment); + } + } + + // Total inline size check + if (totalInlineSize > MAX_TOTAL_INLINE_SIZE_BYTES) { + return { valid: false, error: `Total inline attachment size exceeds 3 MB limit. Use presigned upload for larger files.` }; + } + + // Total declared size check (inline + presigned) + if (totalInlineSize + totalDeclaredSize > MAX_TOTAL_ATTACHMENT_SIZE_BYTES) { + return { valid: false, error: `Total attachment size exceeds 50 MB limit` }; + } + + return { valid: true, parsed }; +} + +/** Reject filenames with path traversal, null bytes, or unusual characters */ +function isValidFilename(filename: string): boolean { + if (filename.length > 255) return false; + if (filename.includes('/') || filename.includes('\\')) return false; + if (filename.includes('\0')) return false; + if (filename.startsWith('.') || filename.startsWith('-')) return false; + if (filename === '.' || filename === '..') return false; + return /^[a-zA-Z0-9][a-zA-Z0-9._\- ]{0,253}[a-zA-Z0-9._]$/.test(filename); +} +``` + +**Step 2 — Async pre-checks** (separate function, called after sync validation): + +```typescript +/** Async validation step: SSRF DNS resolution for URL attachments. */ +export async function validateAttachmentUrls( + attachments: ValidatedAttachment[], +): Promise<{ valid: true } | { valid: false; error: string }> { + for (const [i, att] of attachments.entries()) { + if (att.type === 'url') { + try { + const ssrfCheck = await checkSsrf(att.url, { timeoutMs: 5000 }); + if (!ssrfCheck.safe) { + return { valid: false, error: `attachments[${i}]: ${ssrfCheck.reason}` }; + } + } catch (err) { + // DNS lookup failure — fail closed. Differentiate transient vs security failures: + return { valid: false, error: `attachments[${i}]: DNS resolution failed for the provided URL (transient network error, not a security block). Please check the URL is correct and try again.` }; + } + } + } + return { valid: true }; +} +``` + +This split preserves the existing pattern where `validation.ts` functions are synchronous pure functions (no I/O), while keeping the async SSRF check fail-closed with a 5-second timeout. + +## Partial failure cleanup + +When task creation fails after some attachments have already been uploaded to S3, orphaned objects must be cleaned up: + +```typescript +async function cleanupOrphanedAttachments(uploadedKeys: string[]): Promise { + if (uploadedKeys.length === 0) return; + try { + const result = await s3Client.send(new DeleteObjectsCommand({ + Bucket: ATTACHMENTS_BUCKET, + Delete: { Objects: uploadedKeys.map(Key => ({ Key })) }, + })); + // DeleteObjectsCommand does NOT throw on individual object failures — + // check the Errors array explicitly + if (result.Errors && result.Errors.length > 0) { + logger.error('Partial cleanup failure — some orphaned objects remain', { + failedKeys: result.Errors.map(e => e.Key), + errorCodes: result.Errors.map(e => e.Code), + }); + emitMetric('OrphanedAttachmentCleanupPartialFailure', result.Errors.length); + } + } catch (err) { + // Entire cleanup failed — log and emit metric, do NOT re-throw + // (we are already in an error handler; the 90-day lifecycle is the safety net) + logger.error('Cleanup failed entirely — all objects orphaned', { + keys: uploadedKeys, + error: String(err), + }); + emitMetric('OrphanedAttachmentCleanupFailure', uploadedKeys.length); + } +} +``` + +**Strategy:** The inline upload path accumulates S3 keys as it processes attachments. If any attachment fails screening (or any other error occurs), the error handler calls `cleanupOrphanedAttachments` before returning the error response. The function handles both partial and total cleanup failures gracefully — it never re-throws (we're already in an error handler), instead emitting metrics for operational visibility. + +For the presigned upload path, cleanup happens in `confirm-uploads`: if screening fails for any attachment, the handler attempts a conditional DynamoDB write to `FAILED` (`ConditionExpression: 'status = PENDING_UPLOADS'`). Only if the write succeeds does the handler delete S3 objects. If the write fails (concurrent `confirm-uploads` already transitioned to `SUBMITTED`), cleanup is skipped — those objects are needed by the running agent. + +**Lifecycle as safety net:** Even if cleanup fails, the 90-day lifecycle expiration ensures objects are eventually removed. The `OrphanedAttachmentCleanupFailure` metric tracks these cases for alerting. + +**Pre-DynamoDB-write orphans:** If the inline upload path fails BEFORE the TaskRecord is written to DynamoDB (e.g., S3 PutObject succeeds for attachment 1, then screening crashes for attachment 2, and the task was never persisted), the orphaned S3 objects have no task record to correlate them with. The cleanup function still runs and emits `OrphanedAttachmentNoTask` (distinct from `OrphanedAttachmentCleanupFailure`). Structured logs include the S3 keys so operators can identify these objects. The 90-day lifecycle is the final safety net. + +## State machine changes + +Adding `PENDING_UPLOADS` requires updating `task-status.ts`. The full impact: + +### New status + +```typescript +// In cdk/src/constructs/task-status.ts: +export const TaskStatus = { + // ... existing statuses ... + PENDING_UPLOADS: 'PENDING_UPLOADS', +} as const; +``` + +### Valid transitions + +```typescript +export const VALID_TRANSITIONS: Record = { + // ... existing transitions ... + PENDING_UPLOADS: ['SUBMITTED', 'FAILED', 'CANCELLED'], +}; +``` + +### Status classification + +`PENDING_UPLOADS` is **neither active nor terminal**. It is a new category: **pre-active**. It does not count against user concurrency limits (no agent resources allocated), but it is visible in task lists and can be cancelled. + +```typescript +export const ACTIVE_STATUSES: TaskStatusType[] = [ + // PENDING_UPLOADS is NOT here — does not count against concurrency + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING', +]; + +export const PRE_ACTIVE_STATUSES: TaskStatusType[] = [ + 'PENDING_UPLOADS', +]; + +export const TERMINAL_STATUSES: TaskStatusType[] = [ + 'COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT', +]; +``` + +### State diagram + +```mermaid +stateDiagram-v2 + [*] --> PENDING_UPLOADS : Presigned upload task + [*] --> SUBMITTED : Inline/no-attachment task + PENDING_UPLOADS --> SUBMITTED : confirm-uploads succeeds + PENDING_UPLOADS --> FAILED : confirm-uploads screening fails + PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel + SUBMITTED --> HYDRATING : Admission passes + HYDRATING --> RUNNING : Context assembled + RUNNING --> FINALIZING : Session ends + FINALIZING --> COMPLETED : Success + FINALIZING --> FAILED : Agent failure +``` + +### Auto-cancel mechanism + +Tasks in `PENDING_UPLOADS` for > 30 minutes are auto-cancelled via an **EventBridge scheduled rule** (not DynamoDB TTL — TTL would delete the record, losing audit trail). The rule: + +1. Runs every 5 minutes. +2. Queries `TaskTable` for tasks with `status = PENDING_UPLOADS` and `created_at < (now - 30 minutes)`. +3. For each expired task, attempts a **conditional DynamoDB write**: `ConditionExpression: 'status = PENDING_UPLOADS'`. If the condition fails (task already transitioned to `SUBMITTED` by a concurrent `confirm-uploads` call), the Lambda skips that task — no cleanup, no overwrite. +4. On successful conditional write: transitions to `CANCELLED` with `error_message: "Upload window expired (30 minutes). Please re-submit the task."`. +5. Cleans up ALL S3 objects under the task's `attachments///` prefix (not just confirmed ones — catches oversized abuse uploads too). +6. Writes a `pending_upload_expired` event to `TaskEventsTable`. + +**Race safety:** The conditional write ensures the auto-cancel Lambda and `confirm-uploads` cannot both succeed. If `confirm-uploads` wins the race (transitions to `SUBMITTED` first), the auto-cancel Lambda's conditional write fails harmlessly. If the auto-cancel Lambda wins, `confirm-uploads`'s conditional write fails and returns the current `CANCELLED` status to the client. There is no window where S3 objects are deleted from under a running task. + +The task record is preserved with status `CANCELLED` — `bgagent status ` returns a clear explanation, not a 404. + +### Impact on ORCHESTRATOR.md + +`ORCHESTRATOR.md` must be updated to include `PENDING_UPLOADS` in the state diagram and describe the confirm-uploads → SUBMITTED transition. This is a documentation-only change (the orchestrator Lambda itself does not handle `PENDING_UPLOADS` — it only sees tasks that have already reached `SUBMITTED`). + +### Concurrency tracking changes + +The existing codebase increments user concurrency at task creation time (in `create-task-core.ts`), since tasks currently always start in `SUBMITTED`. With `PENDING_UPLOADS`, this must change: + +- **Create-task handler (PENDING_UPLOADS path):** Skip concurrency increment. No agent resources are allocated; the task may never be confirmed. +- **Confirm-uploads handler (PENDING_UPLOADS → SUBMITTED):** Increment user concurrency. If the concurrency limit is reached at this point, the confirm-uploads call fails with `CONCURRENCY_LIMIT_EXCEEDED` (same as if the user tried to create a new task). The task remains in `PENDING_UPLOADS` and the user must wait for a slot or cancel another running task. +- **Auto-cancel Lambda (PENDING_UPLOADS → CANCELLED):** No concurrency decrement needed (was never incremented). + +This ensures `PENDING_UPLOADS` tasks never count against the user's concurrency limit, while still enforcing the limit at the point where agent resources would actually be allocated. + +### Impact on existing code that assumes binary status classification + +The existing codebase uses `ACTIVE_STATUSES` and `TERMINAL_STATUSES` for filtering, concurrency counting, and dashboard queries. With the new `PRE_ACTIVE_STATUSES` category, code paths that assume `!isActive → isTerminal` must be updated: + +| Code path | Current assumption | Required change | +|---|---|---| +| Concurrency counting | All non-terminal tasks are counted | Count only `ACTIVE_STATUSES` (excludes `PENDING_UPLOADS`) | +| `bgagent list` status filter | Filters by active vs terminal | Add `PRE_ACTIVE_STATUSES` to "all non-terminal" filter | +| Dashboard "active tasks" widget | Uses `ACTIVE_STATUSES` | No change needed (already correct) | +| Task cleanup / retention | Applies to `TERMINAL_STATUSES` | No change needed (PENDING_UPLOADS auto-cancels to CANCELLED first) | + +### Impact on CLI + +The CLI's status display and any status-based filtering must recognize `PENDING_UPLOADS`. The `bgagent list` command should show these tasks with a "[pending upload]" indicator. + +## DynamoDB schema changes + +The `TaskRecord` gains one new field: + +```typescript +interface TaskRecord { + // ... existing 39 fields ... + attachments?: AttachmentRecord[]; // Array of attachment metadata +} +``` + +The `status` field gains the new `PENDING_UPLOADS` value (see [State machine changes](#state-machine-changes)). + +No new DynamoDB tables are needed. The `AttachmentRecord[]` is stored as a nested attribute in the existing `TaskTable`. This is appropriate because: + +- Attachments are always accessed with their parent task (never queried independently). +- The metadata is small (< 1 KB per attachment, max 10 attachments = < 10 KB total). +- No secondary index is needed on attachment fields. +- The TaskRecord with 39 fields + 10 attachment records stays well under DynamoDB's 400 KB item size limit. + +## CDK construct changes + +### New: `AttachmentsBucket` + +New construct at `cdk/src/constructs/attachments-bucket.ts`. Follows `TraceArtifactsBucket` patterns (see [Storage](#storage-attachments-s3-bucket) above). + +### New: `ConfirmUploadsFunction` + +A separate Lambda for the `confirm-uploads` endpoint, with: +- **Memory:** 2048 MB (must hold up to 10 MB raw image + decompressed pixel buffer for `sharp` re-encoding; `sharp` decompresses a 4096x4096 RGBA image to ~64 MB in memory) +- **Timeout:** 180 seconds (3 minutes). Attachments are screened in **parallel with bounded concurrency of 3**. Worst-case: 4 batches of ~45s each (S3 read + Bedrock screen with retries + sharp re-encode + S3 write). The 180s budget accommodates Bedrock retry delays. +- **Internal deadline timer:** The handler sets a deadline at `Lambda timeout - 15 seconds` (165s). If the screening loop has not completed by this deadline, remaining unscreened attachments are aborted and the handler returns a 503 with `Retry-After: 30` header and body: "Attachment screening did not complete within the time limit. Reduce the number or size of attachments and try again, or retry after 30 seconds (already-screened attachments will be skipped on retry)." The `Retry-After` header enables clients to implement automatic backoff. On retry, the per-attachment screening state (above) ensures only unscreened attachments are re-processed, so retries make forward progress. This prevents opaque Lambda timeout errors. +- **Per-attachment screening state with atomic DynamoDB + S3 ordering:** Each attachment's screening pipeline follows a strict order: (1) screen content, (2) sanitize/re-encode via sharp, (3) PutObject to S3 (the cleaned version), (4) update the attachment's `screening` status to `passed` in DynamoDB (with `s3_version_id` from the PutObject response). The DynamoDB write is the **commit point** — if any prior step fails, the attachment remains in `pending` status. On retry (after a timeout or Lambda restart), the handler skips attachments with `screening.status === 'passed'` (already committed to both S3 and DynamoDB). Attachments still in `pending` are re-processed from step 1 — this is safe because S3 PutObject is idempotent and the version ID from the new put supersedes any orphaned partial upload. This ordering ensures no attachment is marked as `passed` in DynamoDB without the corresponding cleaned content being in S3. +- **Bundled dependencies:** `sharp` (for EXIF stripping + re-encoding), `pdf-parse` (for PDF text extraction) +- **Cold-start validation:** On module initialization, the handler verifies `sharp` loads correctly (e.g., `sharp(Buffer.alloc(1)).metadata()` wrapped in try-catch). If the native module fails to load (architecture mismatch, missing binary), the handler short-circuits all requests with 503: "Attachment processing is temporarily unavailable. Please try again later." This prevents opaque `Runtime.ImportModuleError` failures from reaching users. + +Separating this from the create-task Lambda keeps the common path (task creation without attachments) lean and fast. + +### New: `PendingUploadCleanupRule` + +EventBridge scheduled rule for auto-cancelling stale `PENDING_UPLOADS` tasks. Runs every 5 minutes, backed by a lightweight Lambda (256 MB, 30s timeout) that queries and transitions expired tasks. + +### Modified: `CreateTaskFunction` + +- **Memory:** 256 MB (increased from CDK default 128 MB — needed for base64 decode of inline attachments + screening + S3 upload) +- **Timeout:** 15 seconds (increased from CDK default 3s — inline screening of small attachments plus S3 operations) +- Pass the attachments bucket as `ATTACHMENTS_BUCKET` environment variable. +- Grant `grantPut` and `grantDelete` on the attachments bucket. + +### Modified: `TaskOrchestrator` + +- Grant `grantReadWrite` on the attachments bucket (for URL attachment fetch/screen/upload). +- Pass the bucket name as `ATTACHMENTS_BUCKET` environment variable. + +### Modified: `TaskApi` (API Gateway) + +- New resource: `POST /v1/tasks/{task_id}/confirm-uploads` → `ConfirmUploadsFunction` +- No body size limit changes needed (presigned uploads bypass the gateway entirely; inline bodies stay under the default limit). +- The `confirm-uploads` endpoint uses the same Cognito authorizer as other task endpoints. + +## API response changes + +The `TaskDetail` response gains an `attachments` summary: + +```typescript +interface TaskDetail { + // ... existing fields ... + attachments?: AttachmentSummary[]; +} + +interface AttachmentSummary { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly filename: string; + readonly content_type: string; + readonly size_bytes: number; + readonly screening_status: 'passed' | 'blocked' | 'pending'; +} +``` + +Binary content is never returned in API responses. The `AttachmentSummary` is metadata only. + +**New type for presigned upload response:** + +```typescript +/** Returned in the creation response for PENDING_UPLOADS tasks only. */ +interface AttachmentUploadInstruction { + readonly attachment_id: string; + readonly filename: string; + readonly upload_url: string; // Presigned POST URL + readonly upload_fields: Record; // Form fields to include in multipart POST + readonly upload_expires_at: string; // Presigned POST policy expiry (10 min) +} +``` + +**New response for presigned upload tasks:** + +```json +{ + "data": { + "task_id": "01HYX...", + "status": "PENDING_UPLOADS", + "task_expires_at": "2025-03-15T11:00:00Z", + "attachments": [ + { + "attachment_id": "01HYX...", + "filename": "screenshot.png", + "upload_url": "https://s3.amazonaws.com/...", + "upload_expires_at": "2025-03-15T10:40:00Z" + } + ] + } +} +``` + +Note: `task_expires_at` (30-minute auto-cancel window) is distinct from `upload_expires_at` (10-minute presigned POST policy expiry). Both are communicated to the client. + +The `upload_url` and `upload_expires_at` fields are only present in the initial creation response. They are not returned on subsequent `GET /v1/tasks/{task_id}` calls. + +## API contract sync + +This design introduces changes that conflict with the current [API_CONTRACT.md](/architecture/api-contract). The following updates must be made to API_CONTRACT.md in tandem with implementation: + +| Section | Current value | New value | +|---|---|---| +| Conventions: max body | "max 1 MB body" | "max 1 MB body (6 MB for task creation with inline attachments)" | +| Create task: `data` field | "max 10 MB decoded" | "max 500 KB decoded (inline); use presigned upload for larger files (up to 10 MB)" | +| Create task: `task_description` | "max 2,000 chars" | "max 10,000 chars" (standalone API change — benefits all tasks, not just attachment tasks) | +| Endpoints table | — | Add `POST /v1/tasks/{task_id}/confirm-uploads` | +| Error codes | — | Add all `ATTACHMENT_*` error codes | + +## CLI type sync + +The CLI `types.ts` must be updated to match the server types. The CDK `CreateTaskRequest` already includes `attachments?: Attachment[]` (types.ts line 275), but the CLI mirror at `cli/src/types.ts` lacks it — a pre-existing sync gap that this implementation must close. + +1. Add `attachments?: Attachment[]` to `CreateTaskRequest`. +2. Add `AttachmentType`, `AttachmentSummary`, `AttachmentUploadInstruction` types. +3. Add `attachments?: AttachmentSummary[]` to `TaskDetail`. +4. Add `PresignedUploadResponse` type for the two-phase flow. +5. Update `MAX_TASK_DESCRIPTION_LENGTH` to 10,000. + +## Error codes + +New error codes for attachment-related failures: + +| Code | Status | Description | +|---|---|---| +| `ATTACHMENT_BLOCKED` | 400 | Attachment content blocked by security screening | +| `ATTACHMENT_TOO_LARGE` | 400 | Individual attachment exceeds 10 MB size limit | +| `ATTACHMENT_INLINE_TOO_LARGE` | 400 | Inline attachment exceeds 500 KB limit (use presigned upload) | +| `ATTACHMENTS_INLINE_TOTAL_TOO_LARGE` | 400 | Total inline attachment size exceeds 3 MB limit | +| `ATTACHMENTS_TOTAL_TOO_LARGE` | 400 | Total attachment size exceeds 50 MB limit | +| `ATTACHMENT_INVALID_TYPE` | 400 | MIME type not in allowlist | +| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or could not be sanitized (sharp failure) | +| `ATTACHMENT_INVALID_FILENAME` | 400 | Filename contains invalid characters or path traversal | +| `ATTACHMENT_SIZE_MISMATCH` | 400 | Uploaded file size does not match declared `expected_size_bytes` (> 10% deviation) | +| `ATTACHMENT_FETCH_FAILED` | 422 | URL attachment could not be fetched (timeout, DNS, SSRF blocked) | +| `ATTACHMENT_BUDGET_EXCEEDED` | 422 | Image attachments exceed token budget (insufficient room for text context) | +| `ATTACHMENT_DOWNLOAD_FAILED` | 500 | Agent could not download attachment from S3 | +| `ATTACHMENT_INTEGRITY_FAILED` | 500 | Attachment checksum mismatch after download | +| `ATTACHMENT_UNSUPPORTED_AGENT` | 422 | Agent runtime version does not support attachments | +| `ATTACHMENT_SCREENING_UNAVAILABLE` | 503 | Screening service unavailable after retries (fail-closed) | +| `UPLOADS_NOT_CONFIRMED` | 409 | Task is in PENDING_UPLOADS but confirm-uploads not yet called | +| `UPLOADS_EXPIRED` | 410 | Upload window expired (> 30 minutes); re-submit the task | + +## Observability + +### New CloudWatch metrics + +| Metric | Dimensions | Purpose | +|---|---|---| +| `AttachmentUploadCount` | Type (`image`/`file`/`url`), Path (`inline`/`presigned`/`url_fetch`) | Track attachment usage patterns | +| `AttachmentUploadSize` | Type | Size distribution | +| `AttachmentScreeningDuration` | Type, Outcome | Screening latency | +| `AttachmentScreeningOutcome` | Type, Outcome (`passed`/`blocked`) | Block rate | +| `AttachmentScreeningRetry` | RetryCount | How often retries are needed | +| `AttachmentFetchDuration` | — | URL fetch latency | +| `AttachmentFetchFailure` | Reason (`timeout`/`ssrf_blocked`/`too_large`/`dns_error`/`http_error`) | URL fetch failure breakdown | +| `AttachmentTokenBudgetUsed` | — | How much of the token budget images consume | +| `AttachmentBudgetExceeded` | — | Tasks failed due to image token budget overflow | +| `OrphanedAttachmentCleanupFailure` | — | Cleanup failures (orphaned S3 objects) | +| `OrphanedAttachmentCleanupPartialFailure` | — | Partial cleanup failures | +| `OrphanedAttachmentNoTask` | — | Objects uploaded before task was persisted to DynamoDB (pre-write failures); no task record to correlate | +| `PendingUploadExpired` | — | Tasks that expired in PENDING_UPLOADS (never confirmed) | +| `ConfirmUploadsRace` | — | Concurrent confirm-uploads detected (condition check failed) | +| `SharpColdStartFailure` | — | `sharp` native module failed to load on Lambda cold start | +| `ScreeningDeadlineExceeded` | — | Confirm-uploads hit internal deadline timer before all attachments screened | + +### Task events + +New event types in `TaskEventsTable`: + +| Event type | When | Metadata | +|---|---|---| +| `attachments_uploaded` | After inline attachments are stored in S3 | Count, total size, types | +| `uploads_confirmed` | After confirm-uploads succeeds | Count, total size, screening duration | +| `attachment_blocked` | Screening rejects an attachment | Attachment ID, screening categories | +| `attachment_fetch_failed` | URL attachment fetch fails | Attachment ID, URL (redacted), reason | +| `attachments_resolved` | All attachments ready for agent | Count, total size, token budget used | +| `pending_upload_expired` | Task auto-cancelled due to upload timeout | Task ID, pending attachment count | +| `attachment_unsupported` | Agent version does not support attachments | Agent version, attachment count | + +## Security considerations + +### Threat model + +| Threat | Vector | Mitigation | +|---|---|---| +| Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; Bedrock image screening; EXIF stripping; image re-encoding through `sharp` strips embedded payloads; sharp failure → reject | +| Prompt injection via file content | Text file containing adversarial instructions | Magic bytes validation; Bedrock Guardrail text screening with retry (same as task descriptions); content trust tagging as `untrusted-external` | +| SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time | +| Data exfiltration via URL attachment | URL pointing to attacker-controlled server (leaks request headers/IP) | No auth headers sent to non-GitHub URLs; minimal request headers; no cookies | +| Denial of service via large attachments | Many large base64 payloads | 500 KB inline limit; 3 MB total inline; 10 MB per-attachment; 50 MB total; 10 count limit; 6 MB Lambda payload limit | +| Path traversal via filename | `filename: "../../etc/passwd"` | Filename sanitization regex; reject path separators, dots-prefix, null bytes; use `attachment_id` as primary path component | +| Zip bomb / decompression bomb | Compressed content that expands massively | No archive types in MIME allowlist; PDF text extraction capped at 50 pages and 1 MB output | +| Polyglot files | File with valid image header + appended executable | Magic bytes validation at upload; image re-encoding strips trailing data | +| Presigned URL abuse | Leaked presigned POST policy used to upload different content | Content-Type fixed in presigned POST policy; `content-length-range` enforced by S3 (rejects > 10 MB before writing); 10-minute expiry; screening runs after upload regardless; size verified via HeadObject (defense-in-depth) | +| S3 object replacement (TOCTOU) | Client uploads benign content (passes screening), then replaces object with malicious content before agent downloads | S3 versioning enabled; `VersionId` pinned at screening time and stored in `AttachmentRecord`; agent downloads with pinned `VersionId`; `noncurrentVersionExpiration: 7 days` prevents storage bloat while allowing long-running tasks to complete | +| Upload slot exhaustion | Create many tasks in PENDING_UPLOADS, never confirm | 30-minute EventBridge auto-cancel with S3 cleanup; PENDING_UPLOADS does not count against concurrency; rate limiting on task creation already exists | +| Confirm-uploads race | Two concurrent confirm-uploads corrupt state | Early status check short-circuit; DynamoDB conditional write; safe cleanup skips if already SUBMITTED | +| DNS rebinding | DNS returns public IP at first lookup (passes validation), private IP at second lookup (reaches internal services) | DNS resolution pinning: resolve DNS manually, validate IP, connect directly to the validated IP (preventing a second DNS lookup); re-validate resolved IP after each redirect | + +### Content trust + +Attachment content inherits the `untrusted-external` trust level in the content trust framework. The agent's system prompt labels attachments accordingly: + +``` +Content trust levels: +- task_description: trusted (from authenticated user) +- issue_body: untrusted-external (from GitHub) +- attachments: untrusted-external (user-provided binary/text content, security-screened) +``` + +This is correct even for inline uploads from authenticated users. The content of a file is fundamentally different from the task description: a user might upload a file they received from an untrusted source (e.g., a customer's screenshot, a downloaded log file). Screening catches known-bad content; trust tagging handles the residual risk by ensuring the agent treats attachment content as context, not as instructions. + +## Cost impact + +| Component | Additional cost per task (with attachments) | Notes | +|---|---|---| +| S3 storage | ~$0.001 | 50 MB * $0.023/GB, 90-day retention | +| S3 PUT/GET | ~$0.00001 | 10 PUTs + 10 GETs | +| Bedrock Guardrail (image) | ~$0.01-0.05 per image | Depends on image size and guardrail config | +| Bedrock Guardrail (text) | ~$0.001-0.01 per file | Same as existing text screening | +| Lambda compute (confirm-uploads) | ~$0.01-0.05 | 30-180s at 2048 MB for screening + re-encoding | +| Lambda compute (create-task, inline) | ~$0.001 | Additional 1-3s at 256 MB for small inline attachments | +| Lambda compute (auto-cancel rule) | ~$0.0001 | 5-minute schedule, mostly no-op | +| Data transfer (URL fetch) | ~$0.001 | Outbound fetch within region is free; cross-region is negligible | +| Claude vision (image in prompt) | ~$0.01-0.05 per image | Multimodal input token cost | + +**Total additional cost per task with attachments:** ~$0.05-0.35, heavily dependent on attachment count and types. Tasks without attachments have zero additional cost. + +## Implementation plan + +The implementation is ordered to deliver value incrementally while maintaining system safety. Security protections ship with the attack vectors they defend against — not deferred to a later phase. + +### Phase 1: Storage + validation + inline upload (small attachments) + +1. Verify `@aws-sdk/client-bedrock-runtime` supports image content blocks in `ApplyGuardrail` (specifically `GuardrailImageBlock` with `format: 'png' | 'jpeg'`); upgrade if needed +2. Create `AttachmentsBucket` construct (with versioning enabled, props interface) +3. Extract `AttachmentType` shared type in `types.ts`; add `AttachmentDelivery` type +4. Add `ValidatedAttachment` discriminated union types (with `delivery` discriminant) +5. Add `AttachmentRecord` with discriminated `ScreeningResult` union (non-empty `categories` tuple, `s3_version_id` field) +6. Add `createAttachmentRecord` factory function (enforces cross-field invariants from day one) +7. Add `AttachmentSummary` and `AttachmentUploadInstruction` response types +8. Add `AgentAttachmentPayload` named interface (with `checksum_sha256` and `s3_version_id`) +9. Add `AttachmentError` base class hierarchy (`AttachmentResolutionError`, `AttachmentBudgetExceededError`) +10. Add `AttachmentError` base class to hydration catch block re-throw list (single `instanceof` check — ships with the error classes so the re-throw is in place before any code throws these errors in Phase 3) +11. Add attachment validation to `validation.ts` (sync: schema, limits, magic bytes, content_type detection, filename generation; async: SSRF DNS pre-check with differentiated error messages) +12. Increase `MAX_TASK_DESCRIPTION_LENGTH` to 10,000 (standalone API change — update API_CONTRACT.md) +13. Add Bedrock screening retry logic (3 retries, exponential backoff) +14. Add GIF/WebP → PNG conversion before Bedrock screening (Bedrock only supports png|jpeg), with post-conversion size check +15. Add inline upload path to `create-task-core.ts` (base64 → magic bytes → screen with retry → EXIF strip/re-encode → S3 with versioning; sharp failure → ATTACHMENT_INVALID_CONTENT) +16. Add SHA-256 checksum computation at upload time (stored in AttachmentRecord, required by factory) +17. Add partial failure cleanup with proper `DeleteObjects` error handling and metrics +18. Add `attachments` field to `TaskRecord` +19. Add `AttachmentSummary` to `TaskDetail` response +20. Sync CLI types (close the pre-existing sync gap) +21. Increase create-task Lambda memory to 256 MB (from CDK default 128 MB), timeout to 15s (from CDK default 3s) +22. Update `API_CONTRACT.md` (inline limit, task_description limit, note Bedrock format constraints) + +**Security included in Phase 1:** magic bytes validation, EXIF stripping, image re-encoding, GIF/WebP conversion (with post-conversion size check), sharp fail-closed, filename sanitization, partial failure cleanup, Bedrock retry, S3 versioning (TOCTOU prevention), SHA-256 integrity. + +### Phase 2: Presigned upload + state machine (large attachments) + +23. Add `PENDING_UPLOADS` to `task-status.ts` (status, transitions, `PRE_ACTIVE_STATUSES` classification) +24. Update all code paths that assume binary status classification (active vs terminal) — see [Impact on existing code](#impact-on-existing-code-that-assumes-binary-status-classification) +25. Add `confirm-uploads` Lambda (2048 MB, 180s timeout) with parallel screening (concurrency 3), internal deadline timer, per-attachment screening state, and sharp cold-start validation +26. Add `POST /v1/tasks/{task_id}/confirm-uploads` API endpoint with concurrent-call safety (early short-circuit, conditional DynamoDB write for both success and failure paths) +27. Move concurrency increment from create-task to confirm-uploads for presigned-upload tasks +28. Add presigned POST policy generation in create-task handler (with `content-length-range` enforcement, `expected_size_bytes` validation, S3 versioning) +29. Add idempotency key special-casing for `PENDING_UPLOADS` (new S3 keys + new attachment IDs on retry to prevent collision, conditional DynamoDB write) +30. Add `PendingUploadCleanupRule` EventBridge rule (5-minute schedule, conditional DynamoDB write for race safety, prefix-level S3 cleanup) +31. Add CLI two-phase upload flow for files > 500 KB (with progress bar, multipart form POST) +32. Update `ORCHESTRATOR.md` state diagram + +### Phase 3: Agent delivery + token budget + +33. Add `attachments` to top-level agent payload (in orchestrator) using `AgentAttachmentPayload` (includes `s3_version_id` and `checksum_sha256`) +34. Add `AttachmentConfig` and `PreparedAttachment` Pydantic models to agent `models.py` (with validators, `s3_version_id` required, `checksum_sha256` required as lowercase hex) +35. Add attachment download from S3 with pinned `VersionId` (via IAM role) and mandatory SHA-256 integrity verification +36. Add multimodal content blocks for image attachments in agent prompt +37. Add token budget accounting with resize-aware formula matching Anthropic docs (1568px cap, 28px tile padding, 1568 token cap, 1.2x safety margin), with explicit error path for `getImageDimensions` failures +38. Add `AttachmentBudgetExceededError` (extends `AttachmentError` base class — already caught by hydration re-throw list from Phase 1 step 10) +39. Add agent capability check in orchestrator (fail task if agent doesn't support attachments) +40. Add parity test: `AgentAttachmentPayload` fields match `AttachmentConfig` fields (including `s3_version_id` and `checksum_sha256`) + +### Phase 4: URL fetch + channels + +41. Add URL fetch in orchestrator hydration with full SSRF suite (manual DNS resolve → IP check → connect to resolved IP via undici; differentiated error messages: "DNS failed" vs "SSRF blocked") +42. Add `--attachment` flag to `bgagent submit` (auto-detect file vs URL, size-based routing to inline vs presigned, URL attachment polling) +43. Add Slack file extraction in `slack-events.ts` with atomic failure semantics (consistent with design principle 3 — all-or-nothing) +44. Add Linear image URL extraction in `linear-webhook-processor.ts` + +**Security included in Phase 4:** Full SSRF protection suite with DNS resolution pinning (TOCTOU mitigation). + +### Phase 5: Observability + hardening + +45. Add all CloudWatch metrics (including retry, race detection, cleanup failure, `OrphanedAttachmentNoTask` for pre-DynamoDB-write failures) +46. Add all task events +47. Add alerting on `OrphanedAttachmentCleanupFailure` and `PendingUploadExpired` +48. Add comprehensive integration tests (all upload paths, all failure modes, all channels, concurrent confirm-uploads, auto-cancel racing with confirm-uploads) +49. Add load testing for screening pipeline (sustained 10-attachment tasks at peak rate) +50. Add monitoring for `sharp` native module health (cold-start validation failures) +51. Add monitoring for format conversion size expansion (GIF/WebP → PNG rejection rate) From 76a37badc94b19b2e632e9b2aa9924bdb9bba534 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 19 May 2026 15:25:07 -0500 Subject: [PATCH 03/19] chore(main): merge main --- cdk/src/handlers/shared/memory.ts | 8 ++++---- cdk/test/handlers/shared/memory.test.ts | 19 +++++-------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/cdk/src/handlers/shared/memory.ts b/cdk/src/handlers/shared/memory.ts index 587c47e8..81136f9c 100644 --- a/cdk/src/handlers/shared/memory.ts +++ b/cdk/src/handlers/shared/memory.ts @@ -166,8 +166,8 @@ function getClient(): BedrockAgentCoreClient { * - Semantic: `/{actorId}/knowledge/` (actorId = repo) * - Episodic: `/{actorId}/episodes/` (covers all sessions and reflections) * - * Both calls use `namespacePath` for hierarchical retrieval — episodic per-task - * records live at `/{actorId}/episodes/{sessionId}/`, which is below the read path. + * Both calls pass a `namespace` prefix for hierarchical retrieval — episodic + * per-task records live at `/{actorId}/episodes/{sessionId}/`, below the read path. * * Results are trimmed to a 2000-token budget (knowledge is prioritized before episodes; * entries beyond the budget are dropped). @@ -199,7 +199,7 @@ export async function loadMemoryContext( taskDescription ? client.send(new RetrieveMemoryRecordsCommand({ memoryId, - namespacePath: semanticNamespacePath, + namespace: semanticNamespacePath, searchCriteria: { searchQuery: taskDescription, topK: 5, @@ -212,7 +212,7 @@ export async function loadMemoryContext( // Episodic search — recent task episodes (prefix matches all sessions) client.send(new RetrieveMemoryRecordsCommand({ memoryId, - namespacePath: episodicNamespacePath, + namespace: episodicNamespacePath, searchCriteria: { searchQuery: 'recent task episodes', topK: 3, diff --git a/cdk/test/handlers/shared/memory.test.ts b/cdk/test/handlers/shared/memory.test.ts index 701a5427..854ecc0d 100644 --- a/cdk/test/handlers/shared/memory.test.ts +++ b/cdk/test/handlers/shared/memory.test.ts @@ -69,7 +69,7 @@ describe('loadMemoryContext', () => { expect(result!.repo_knowledge[0]).toContain('Jest'); }); - test('uses namespacePath (hierarchical retrieval) for both queries', async () => { + test('uses namespace prefix (hierarchical retrieval) for both queries', async () => { const { RetrieveMemoryRecordsCommand } = jest.requireMock('@aws-sdk/client-bedrock-agentcore'); mockAgentCoreSend .mockResolvedValueOnce({ memoryRecordSummaries: [] }) @@ -77,34 +77,25 @@ describe('loadMemoryContext', () => { await loadMemoryContext('mem-123', 'owner/repo', 'Fix the build'); - // Semantic search uses /{repo}/knowledge/ as namespacePath. The legacy - // `namespace` field switched from prefix-match to exact-match in the - // AgentCore Memory API; namespacePath preserves the hierarchical (prefix) - // semantics this code depends on for episodic per-task records nested - // under /{repo}/episodes/{sessionId}/. + // Semantic search uses /{repo}/knowledge/ as a namespace prefix. expect(RetrieveMemoryRecordsCommand).toHaveBeenCalledWith( expect.objectContaining({ - namespacePath: '/owner/repo/knowledge/', + namespace: '/owner/repo/knowledge/', searchCriteria: expect.objectContaining({ searchQuery: 'Fix the build', }), }), ); - // Episodic search uses /{repo}/episodes/ namespacePath to scoop up records + // Episodic search uses /{repo}/episodes/ prefix to scoop up records // under all task sessions plus the cross-task reflection records. expect(RetrieveMemoryRecordsCommand).toHaveBeenCalledWith( expect.objectContaining({ - namespacePath: '/owner/repo/episodes/', + namespace: '/owner/repo/episodes/', searchCriteria: expect.objectContaining({ searchQuery: 'recent task episodes', }), }), ); - // Confirm the legacy `namespace` field is NOT being passed — we don't - // want to send both fields (the API rejects that) or the wrong one. - expect(RetrieveMemoryRecordsCommand).not.toHaveBeenCalledWith( - expect.objectContaining({ namespace: expect.anything() }), - ); }); test('returns undefined when no results are found', async () => { From 6c8fa89cdcb1b329913fbc5c1b99677bd99a6345 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 19 May 2026 17:25:02 -0500 Subject: [PATCH 04/19] chore(test): update tests --- cli/test/format-status-snapshot.test.ts | 1 + cli/test/format.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/cli/test/format-status-snapshot.test.ts b/cli/test/format-status-snapshot.test.ts index 5cae4085..ae0f20fb 100644 --- a/cli/test/format-status-snapshot.test.ts +++ b/cli/test/format-status-snapshot.test.ts @@ -55,6 +55,7 @@ function buildTask(overrides: Partial = {}): TaskDetail { turns_completed: null, trace: false, trace_s3_uri: null, + attachments: null, approval_gate_count: 0, approval_gate_cap: 50, awaiting_approval_request_id: null, diff --git a/cli/test/format.test.ts b/cli/test/format.test.ts index a285a2e9..57ea32d4 100644 --- a/cli/test/format.test.ts +++ b/cli/test/format.test.ts @@ -49,6 +49,7 @@ describe('format', () => { turns_completed: null, trace: false, trace_s3_uri: null, + attachments: null, approval_gate_count: 0, approval_gate_cap: 50, awaiting_approval_request_id: null, From bc16362cd4bb879830de31ea1f9f98105c043c84 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 19 May 2026 18:21:38 -0500 Subject: [PATCH 05/19] chore(attachments): add phase 2 --- cdk/package.json | 3 + cdk/src/constructs/task-status.ts | 29 +- .../handlers/shared/attachment-screening.ts | 388 ++++++++++++++++++ cdk/src/handlers/shared/create-task-core.ts | 217 ++++++---- cdk/src/handlers/slack-interactions.ts | 14 +- cdk/src/types/pdf-parse.d.ts | 9 + cdk/test/constructs/task-status.test.ts | 71 +++- cli/src/types.ts | 1 + docs/design/ATTACHMENTS.md | 7 +- docs/design/ORCHESTRATOR.md | 25 +- .../content/docs/architecture/Attachments.md | 7 +- .../content/docs/architecture/Orchestrator.md | 25 +- 12 files changed, 700 insertions(+), 96 deletions(-) create mode 100644 cdk/src/handlers/shared/attachment-screening.ts create mode 100644 cdk/src/types/pdf-parse.d.ts diff --git a/cdk/package.json b/cdk/package.json index e3d71a2e..54f95ab1 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -31,6 +31,8 @@ "aws-cdk-lib": "^2.238.0", "cdk-nag": "^2.37.55", "constructs": "^10.3.0", + "pdf-parse": "^1.1.1", + "sharp": "^0.33.5", "ulid": "^3.0.2" }, "devDependencies": { @@ -39,6 +41,7 @@ "@types/aws-lambda": "^8.10.161", "@types/jest": "^30.0.0", "@types/node": "^20", + "@types/pdf-parse": "^1.1.4", "@typescript-eslint/eslint-plugin": "^8", "@typescript-eslint/parser": "^8", "aws-cdk": "^2", diff --git a/cdk/src/constructs/task-status.ts b/cdk/src/constructs/task-status.ts index ef3dcc89..a5db1e33 100644 --- a/cdk/src/constructs/task-status.ts +++ b/cdk/src/constructs/task-status.ts @@ -20,10 +20,15 @@ /** * Valid task states in the task lifecycle state machine. * - * States progress through the lifecycle: SUBMITTED -> HYDRATING -> - * RUNNING -> FINALIZING -> terminal (COMPLETED / FAILED / CANCELLED / TIMED_OUT). + * States progress through the lifecycle: + * [PENDING_UPLOADS ->] SUBMITTED -> HYDRATING -> + * RUNNING -> FINALIZING -> terminal (COMPLETED / FAILED / CANCELLED / TIMED_OUT). * See ORCHESTRATOR.md for the full state transition table. * + * PENDING_UPLOADS is a pre-active state for tasks with presigned-upload + * attachments: no compute allocated, no concurrency slot consumed. The + * task transitions to SUBMITTED once uploads are confirmed and screened. + * * AWAITING_APPROVAL is the Cedar-HITL soft-deny gate surface: the * task is alive but paused on a human decision. See * `docs/design/CEDAR_HITL_GATES.md` §10.3 for the joint @@ -31,6 +36,7 @@ * must preserve when transitioning in or out of this state. */ export const TaskStatus = { + PENDING_UPLOADS: 'PENDING_UPLOADS', SUBMITTED: 'SUBMITTED', HYDRATING: 'HYDRATING', RUNNING: 'RUNNING', @@ -57,10 +63,20 @@ export const TERMINAL_STATUSES: readonly TaskStatusType[] = [ TaskStatus.TIMED_OUT, ]; +/** + * Pre-active states where the task exists but has not entered the + * orchestration pipeline. No compute resources are allocated and no + * concurrency slot is consumed. + */ +export const PRE_ACTIVE_STATUSES: readonly TaskStatusType[] = [ + TaskStatus.PENDING_UPLOADS, +]; + /** * Active (non-terminal) states that indicate a task is still in progress. * AWAITING_APPROVAL counts as active — the task is alive, just paused - * waiting on a human decision. + * waiting on a human decision. PENDING_UPLOADS is NOT here — it is + * pre-active and does not consume a concurrency slot. */ export const ACTIVE_STATUSES: readonly TaskStatusType[] = [ TaskStatus.SUBMITTED, @@ -73,9 +89,14 @@ export const ACTIVE_STATUSES: readonly TaskStatusType[] = [ /** * Valid state transitions. Maps each state to the set of states it can transition to. * Derived from the transition table in ORCHESTRATOR.md + §10.3 of the - * Cedar HITL gates design (AWAITING_APPROVAL entries). + * Cedar HITL gates design (AWAITING_APPROVAL entries) + ATTACHMENTS.md + * (PENDING_UPLOADS entries). */ export const VALID_TRANSITIONS: Readonly> = { + // PENDING_UPLOADS: presigned-upload task awaiting client file uploads. + // Transitions to SUBMITTED on confirm-uploads success, FAILED on + // screening failure, CANCELLED on user cancel or 30-min auto-cancel. + [TaskStatus.PENDING_UPLOADS]: [TaskStatus.SUBMITTED, TaskStatus.FAILED, TaskStatus.CANCELLED], [TaskStatus.SUBMITTED]: [TaskStatus.HYDRATING, TaskStatus.FAILED, TaskStatus.CANCELLED], [TaskStatus.HYDRATING]: [ TaskStatus.RUNNING, diff --git a/cdk/src/handlers/shared/attachment-screening.ts b/cdk/src/handlers/shared/attachment-screening.ts new file mode 100644 index 00000000..6b1fc8aa --- /dev/null +++ b/cdk/src/handlers/shared/attachment-screening.ts @@ -0,0 +1,388 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { ApplyGuardrailCommand, type BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import { createHash } from 'crypto'; +import sharp from 'sharp'; +import { logger } from './logger'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const MAX_RETRIES = 3; +const BASE_DELAY_MS = 200; +const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503]); + +export const MAX_ATTACHMENT_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface ScreeningConfig { + readonly guardrailId: string; + readonly guardrailVersion: string; + readonly bedrockClient: BedrockRuntimeClient; +} + +export type ScreeningOutcome = + | { readonly status: 'passed' } + | { readonly status: 'blocked'; readonly categories: [string, ...string[]] }; + +export interface ScreenedAttachment { + readonly content: Buffer; + readonly contentType: string; + readonly checksum: string; + readonly screening: ScreeningOutcome; +} + +// --------------------------------------------------------------------------- +// Retry utility +// --------------------------------------------------------------------------- + +/** + * Retry with exponential backoff for transient Bedrock errors. + * Non-retryable errors (4xx except 429, validation errors) propagate immediately. + */ +async function retryWithBackoff( + fn: () => Promise, + opts: { maxRetries: number; baseDelayMs: number; context: string }, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt <= opts.maxRetries; attempt++) { + try { + return await fn(); + } catch (err: any) { + lastError = err; + const statusCode = err?.$metadata?.httpStatusCode ?? err?.statusCode; + const isRetryable = RETRYABLE_STATUS_CODES.has(statusCode); + if (!isRetryable || attempt === opts.maxRetries) { + if (isRetryable && attempt === opts.maxRetries) { + logger.error('All retries exhausted for Bedrock screening', { + context: opts.context, + total_attempts: attempt + 1, + status_code: statusCode, + error: err instanceof Error ? err.message : String(err), + }); + } + throw err; + } + const delay = opts.baseDelayMs * Math.pow(2, attempt); + logger.warn('Retrying after transient error', { + context: opts.context, + attempt: attempt + 1, + max_retries: opts.maxRetries, + status_code: statusCode, + delay_ms: delay, + error: err instanceof Error ? err.message : String(err), + }); + await sleep(delay); + } + } + throw lastError; +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +// --------------------------------------------------------------------------- +// Image screening +// --------------------------------------------------------------------------- + +/** + * Screen an image attachment through the Bedrock Guardrail. + * + * Flow: validate → convert GIF/WebP to PNG (Bedrock only accepts png|jpeg) → + * screen → strip EXIF / re-encode on pass. + * + * @returns ScreenedAttachment with cleaned content (EXIF-stripped, re-encoded) and checksum. + * @throws Error on sharp failure or guardrail unavailability (fail-closed). + */ +export async function screenImage( + content: Buffer, + contentType: string, + filename: string, + config: ScreeningConfig, +): Promise { + // Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) + let screeningBuffer: Buffer; + let screeningFormat: 'png' | 'jpeg'; + + if (contentType === 'image/jpeg') { + screeningBuffer = content; + screeningFormat = 'jpeg'; + } else if (contentType === 'image/gif' || contentType === 'image/webp') { + // GIF/WebP → PNG. For animated GIFs, extract first frame only. + try { + screeningBuffer = await sharp(content, { animated: false }).png().toBuffer(); + } catch (convErr) { + throw new AttachmentScreeningError( + `Image "${filename}" could not be converted from ${contentType} for screening. ` + + `The file may be corrupt. Please re-export or use a PNG/JPEG format.`, + { cause: convErr }, + ); + } + screeningFormat = 'png'; + + // Post-conversion size check: PNG expansion of compressed GIF/WebP can exceed limit. + if (screeningBuffer.length > MAX_ATTACHMENT_SIZE_BYTES) { + throw new AttachmentScreeningError( + `Image "${filename}" is ${contentType} and its PNG conversion for screening ` + + `exceeds the ${MAX_ATTACHMENT_SIZE_BYTES / (1024 * 1024)} MB limit ` + + `(${(screeningBuffer.length / (1024 * 1024)).toFixed(1)} MB after conversion). ` + + `Please convert to JPEG or reduce image dimensions before uploading.`, + ); + } + } else { + // PNG: use as-is + screeningBuffer = content; + screeningFormat = 'png'; + } + + // Screen through Bedrock Guardrail with retry + const result = await retryWithBackoff( + () => config.bedrockClient.send(new ApplyGuardrailCommand({ + guardrailIdentifier: config.guardrailId, + guardrailVersion: config.guardrailVersion, + source: 'INPUT', + content: [{ + image: { + format: screeningFormat, + source: { bytes: screeningBuffer }, + }, + }], + })), + { maxRetries: MAX_RETRIES, baseDelayMs: BASE_DELAY_MS, context: `image_screening:${filename}` }, + ); + + if (result.action === 'GUARDRAIL_INTERVENED') { + const categories = extractBlockedCategories(result.assessments); + return { + content: screeningBuffer, + contentType, + checksum: computeSha256(screeningBuffer), + screening: { status: 'blocked', categories }, + }; + } + + // Screening passed — strip EXIF and re-encode. + // Note: NOT calling .withMetadata() — sharp strips all metadata by default + // when withMetadata is omitted. Calling .withMetadata({}) would opt INTO + // metadata preservation, which is the opposite of what we want. + let cleanedContent: Buffer; + try { + cleanedContent = await sharp(content) + .rotate() // Apply EXIF orientation before stripping + .toBuffer(); + } catch (sharpErr) { + throw new AttachmentScreeningError( + `Image "${filename}" could not be processed for security sanitization. ` + + `Please re-export the image in a standard format and try again.`, + { cause: sharpErr }, + ); + } + + const checksum = computeSha256(cleanedContent); + return { + content: cleanedContent, + contentType, + checksum, + screening: { status: 'passed' }, + }; +} + +// --------------------------------------------------------------------------- +// Text/file screening +// --------------------------------------------------------------------------- + +/** + * Screen a text-based file attachment through the Bedrock Guardrail. + * Supports plain text, CSV, Markdown, JSON, and log files directly. + * PDFs have their text extracted first. + * + * @returns ScreenedAttachment with the original content (text files are not re-encoded). + * @throws Error on guardrail unavailability (fail-closed). + */ +export async function screenTextFile( + content: Buffer, + contentType: string, + filename: string, + config: ScreeningConfig, +): Promise { + let textToScreen: string; + + if (contentType === 'application/pdf') { + textToScreen = await extractPdfText(content, filename); + if (textToScreen.trim().length === 0) { + throw new AttachmentScreeningError( + `PDF "${filename}" contains no extractable text (it may be image-only or encrypted). ` + + `Please use an OCR tool to add a text layer, or convert to an image attachment.`, + ); + } + } else { + textToScreen = content.toString('utf-8'); + } + + // Screen through Bedrock Guardrail with retry + const result = await retryWithBackoff( + () => config.bedrockClient.send(new ApplyGuardrailCommand({ + guardrailIdentifier: config.guardrailId, + guardrailVersion: config.guardrailVersion, + source: 'INPUT', + content: [{ text: { text: textToScreen } }], + })), + { maxRetries: MAX_RETRIES, baseDelayMs: BASE_DELAY_MS, context: `text_screening:${filename}` }, + ); + + const checksum = computeSha256(content); + + if (result.action === 'GUARDRAIL_INTERVENED') { + const categories = extractBlockedCategories(result.assessments); + return { + content, + contentType, + checksum, + screening: { status: 'blocked', categories }, + }; + } + + return { + content, + contentType, + checksum, + screening: { status: 'passed' }, + }; +} + +// --------------------------------------------------------------------------- +// PDF text extraction +// --------------------------------------------------------------------------- + +const PDF_MAX_PAGES = 50; +const PDF_MAX_TEXT_BYTES = 1024 * 1024; // 1 MB extracted text cap +const PDF_EXTRACT_TIMEOUT_MS = 15_000; + +async function extractPdfText(content: Buffer, filename: string): Promise { + // Dynamic import — pdf-parse is only used for PDF attachments. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let pdfParseFn: (data: Buffer, options?: { max?: number }) => Promise<{ text: string }>; + try { + // pdf-parse uses a default export; handle both CJS and ESM module shapes. + const mod = await import(/* webpackIgnore: true */ 'pdf-parse'); + pdfParseFn = (mod as any).default ?? mod; + } catch (importErr) { + logger.error('pdf-parse module could not be imported — PDF screening unavailable', { + error: importErr instanceof Error ? importErr.message : String(importErr), + metric_type: 'pdf_parse_import_failure', + }); + throw new AttachmentScreeningError( + `PDF processing is unavailable. Cannot screen "${filename}".`, + { cause: importErr }, + ); + } + + let timeoutId: ReturnType; + try { + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => reject(new Error('PDF extraction timed out')), PDF_EXTRACT_TIMEOUT_MS); + }); + + const result = await Promise.race([ + pdfParseFn(content, { max: PDF_MAX_PAGES }), + timeoutPromise, + ]); + + let text: string = result.text ?? ''; + if (Buffer.byteLength(text, 'utf-8') > PDF_MAX_TEXT_BYTES) { + text = text.slice(0, PDF_MAX_TEXT_BYTES); + } + return text; + } catch (err) { + throw new AttachmentScreeningError( + `PDF "${filename}" could not be processed. It may be corrupt or use unsupported features. ` + + `Try exporting to a simpler PDF format.`, + { cause: err }, + ); + } finally { + clearTimeout(timeoutId!); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function computeSha256(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + +function extractBlockedCategories( + assessments: any[] | undefined, +): [string, ...string[]] { + const categories: string[] = []; + if (assessments) { + for (const assessment of assessments) { + // Extract topic/content/word/sensitive-info policy categories + for (const policyResult of Object.values(assessment) as any[]) { + if (Array.isArray(policyResult?.topics)) { + for (const t of policyResult.topics) { + if (t.name) categories.push(t.name); + } + } + if (Array.isArray(policyResult?.filters)) { + for (const f of policyResult.filters) { + if (f.type) categories.push(f.type); + } + } + if (Array.isArray(policyResult?.managedWordLists)) { + for (const w of policyResult.managedWordLists) { + if (w.match) categories.push(`word:${w.match}`); + } + } + if (Array.isArray(policyResult?.piiEntities)) { + for (const p of policyResult.piiEntities) { + if (p.type) categories.push(`pii:${p.type}`); + } + } + } + } + } + if (categories.length === 0) { + logger.warn('Could not extract specific categories from guardrail assessment — using generic fallback', { + has_assessments: !!assessments, + assessment_count: assessments?.length ?? 0, + assessment_keys: assessments?.[0] ? Object.keys(assessments[0]) : [], + }); + categories.push('content_policy_violation'); + } + return categories as [string, ...string[]]; +} + +// --------------------------------------------------------------------------- +// Error class +// --------------------------------------------------------------------------- + +export class AttachmentScreeningError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'AttachmentScreeningError'; + } +} diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index e88e11e1..bddb63e3 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -21,7 +21,6 @@ // Idempotent replay: same user + same Idempotency-Key → 200 + TaskDetail (no duplicate write, no orchestrator re-invoke). // Tests: cdk/test/handlers/shared/create-task-core.test.ts, cdk/test/handlers/create-task.test.ts -import { createHash } from 'crypto'; import { BedrockRuntimeClient, ApplyGuardrailCommand } from '@aws-sdk/client-bedrock-runtime'; import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; @@ -30,6 +29,7 @@ import { DynamoDBDocumentClient, PutCommand, QueryCommand, GetCommand } from '@a import type { APIGatewayProxyResult } from 'aws-lambda'; import { ulid } from 'ulid'; import { isDegeneratePattern, parseApprovalScope } from './approval-scope'; +import { screenImage, screenTextFile, AttachmentScreeningError, type ScreeningConfig } from './attachment-screening'; import { generateBranchName } from './gateway'; import { logger } from './logger'; import { lookupRepo } from './repo-config'; @@ -331,16 +331,34 @@ export async function createTaskCore( // Generate task ID early so attachment S3 keys use the correct task ID const taskId = ulid(); - // 2b. Process inline attachments: screen, upload to S3, build records + // 2b. Process inline attachments: screen (with retry + EXIF strip), upload to S3, build records. + // Presigned attachments are deferred to confirm-uploads; URL attachments are resolved during hydration. const attachmentRecords: AttachmentRecord[] = []; const uploadedS3Keys: string[] = []; if (validatedAttachments.length > 0 && s3Client && ATTACHMENTS_BUCKET) { + // Build screening config — fail-closed if guardrail is not configured + if (!bedrockClient || !process.env.GUARDRAIL_ID || !process.env.GUARDRAIL_VERSION) { + const hasInline = validatedAttachments.some(a => a.delivery === 'inline'); + if (hasInline) { + logger.error('Inline attachment submitted but guardrail is not configured (fail-closed)', { + request_id: requestId, + }); + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is not configured. Please contact your administrator.', requestId); + } + } + + const screeningConfig: ScreeningConfig | undefined = bedrockClient && process.env.GUARDRAIL_ID && process.env.GUARDRAIL_VERSION + ? { bedrockClient, guardrailId: process.env.GUARDRAIL_ID, guardrailVersion: process.env.GUARDRAIL_VERSION } + : undefined; + for (const att of validatedAttachments) { if (att.delivery !== 'inline') continue; const inlineAtt = att as InlineAttachment; // Validate base64 encoding before decode if (!isValidBase64(inlineAtt.data)) { + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); return errorResponse(400, ErrorCode.ATTACHMENT_INVALID_CONTENT, `Attachment '${inlineAtt.filename}' has invalid base64 encoding.`, requestId); } @@ -348,94 +366,102 @@ export async function createTaskCore( const decoded = Buffer.from(inlineAtt.data, 'base64'); const attachmentId = ulid(); - // Screen inline attachment content via Bedrock Guardrail (fail-closed) - if (!bedrockClient) { + // Screen content via Bedrock with retry, EXIF stripping, and format conversion + let screenResult; + try { + const isImage = inlineAtt.type === 'image'; + screenResult = isImage + ? await screenImage(decoded, inlineAtt.content_type, inlineAtt.filename, screeningConfig!) + : await screenTextFile(decoded, inlineAtt.content_type, inlineAtt.filename, screeningConfig!); + } catch (screenErr) { await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - logger.error('Inline attachment submitted but guardrail is not configured (fail-closed)', { - request_id: requestId, - attachment_filename: inlineAtt.filename, - }); - return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, - 'Attachment content screening is not configured. Please contact your administrator.', requestId); - } - { - try { - const isImage = inlineAtt.type === 'image'; - const guardrailContent = isImage - ? [{ image: { format: mimeToGuardrailFormat(inlineAtt.content_type), source: { bytes: decoded } } }] - : [{ text: { text: decoded.toString('utf-8') } }]; - - const screenResult = await bedrockClient.send(new ApplyGuardrailCommand({ - guardrailIdentifier: process.env.GUARDRAIL_ID!, - guardrailVersion: process.env.GUARDRAIL_VERSION!, - source: 'INPUT', - content: guardrailContent, - })); - - if (screenResult.action === 'GUARDRAIL_INTERVENED') { - // Clean up any already-uploaded attachments - await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, - `Attachment '${inlineAtt.filename}' was blocked by content policy.`, requestId); - } - } catch (screenErr) { - await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - logger.error('Attachment screening failed (fail-closed)', { - error: String(screenErr), + if (screenErr instanceof AttachmentScreeningError) { + logger.warn('Attachment screening rejected content', { attachment_filename: inlineAtt.filename, request_id: requestId, + error: screenErr.message, }); - return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, - 'Attachment content screening is temporarily unavailable. Please try again later.', requestId); + return errorResponse(400, ErrorCode.ATTACHMENT_INVALID_CONTENT, screenErr.message, requestId); } + logger.error('Attachment screening failed (fail-closed)', { + error: screenErr instanceof Error ? screenErr.message : String(screenErr), + attachment_filename: inlineAtt.filename, + request_id: requestId, + metric_type: 'attachment_screening_failure', + }); + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is temporarily unavailable. Please try again later.', requestId); + } + + if (screenResult.screening.status === 'blocked') { + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); + const categories = screenResult.screening.categories.join(', '); + return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, + `Attachment '${inlineAtt.filename}' was blocked by content policy (${categories}).`, requestId); } - // Upload to S3 + // Upload cleaned content to S3 (images are EXIF-stripped and re-encoded) const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${context.userId}/${taskId}/${attachmentId}/${inlineAtt.filename}`; + let putResult; try { - const putResult = await s3Client.send(new PutObjectCommand({ + putResult = await s3Client.send(new PutObjectCommand({ Bucket: ATTACHMENTS_BUCKET, Key: s3Key, - Body: decoded, + Body: screenResult.content, ContentType: inlineAtt.content_type, })); - - uploadedS3Keys.push(s3Key); - const checksum = createHash('sha256').update(decoded).digest('hex'); - - attachmentRecords.push(createAttachmentRecord({ - attachment_id: attachmentId, - type: inlineAtt.type, - content_type: inlineAtt.content_type, - filename: inlineAtt.filename, - s3_key: s3Key, - s3_version_id: putResult.VersionId ?? 'unversioned', - size_bytes: decoded.length, - screening: { status: 'passed', screened_at: new Date().toISOString() }, - checksum_sha256: checksum, - })); } catch (s3Err) { await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - logger.error('S3 upload failed for inline attachment', { - error: String(s3Err), + logger.error('S3 upload failed for attachment', { + error: s3Err instanceof Error ? s3Err.message : String(s3Err), attachment_filename: inlineAtt.filename, + s3_key: s3Key, request_id: requestId, + metric_type: 'attachment_upload_failure', }); return errorResponse(500, ErrorCode.INTERNAL_ERROR, - `Failed to upload attachment '${inlineAtt.filename}'.`, requestId); + `Failed to store attachment '${inlineAtt.filename}'. Please try again.`, requestId); } + + uploadedS3Keys.push(s3Key); + + // Estimate image token cost (best-effort, non-blocking) + const tokenEstimate = inlineAtt.type === 'image' + ? await estimateImageTokensFromBuffer(screenResult.content) + : undefined; + + attachmentRecords.push(createAttachmentRecord({ + attachment_id: attachmentId, + type: inlineAtt.type, + content_type: inlineAtt.content_type, + filename: inlineAtt.filename, + s3_key: s3Key, + s3_version_id: putResult.VersionId ?? 'unversioned', + size_bytes: screenResult.content.length, + screening: { status: 'passed', screened_at: new Date().toISOString() }, + checksum_sha256: screenResult.checksum, + ...(tokenEstimate !== undefined && { token_estimate: tokenEstimate }), + })); } - // URL attachments get pending records (resolved during hydration) + // URL attachments: reject until hydration-phase screening is implemented. + // This prevents unscreened content from reaching the agent runtime (fail-closed). for (const att of validatedAttachments) { if (att.delivery !== 'url_fetch') continue; + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); + return errorResponse(400, ErrorCode.ATTACHMENT_INVALID_CONTENT, + `URL attachments are not yet supported. Please download the file and submit it as an inline attachment.`, requestId); + } + + // Presigned upload attachments get pending records (confirmed via confirm-uploads endpoint) + for (const att of validatedAttachments) { + if (att.delivery !== 'presigned') continue; attachmentRecords.push(createAttachmentRecord({ attachment_id: ulid(), type: att.type, content_type: att.content_type, filename: att.filename, screening: { status: 'pending' }, - source_url: att.url, })); } } @@ -500,11 +526,16 @@ export async function createTaskCore( ? 'pending:pr_resolution' : generateBranchName(taskId, body.task_description ?? body.repo); + // Determine initial status: PENDING_UPLOADS if any presigned attachments need uploading, + // otherwise SUBMITTED (inline/url/no attachments go straight to the pipeline). + const hasPresignedAttachments = validatedAttachments.some(a => a.delivery === 'presigned'); + const initialStatus = hasPresignedAttachments ? TaskStatus.PENDING_UPLOADS : TaskStatus.SUBMITTED; + // 5. Build task record const taskRecord: TaskRecord = { task_id: taskId, user_id: context.userId, - status: TaskStatus.SUBMITTED, + status: initialStatus, repo: body.repo, ...(body.issue_number !== undefined && { issue_number: body.issue_number }), task_type: taskType, @@ -518,7 +549,7 @@ export async function createTaskCore( channel_source: context.channelSource, channel_metadata: context.channelMetadata, ...(attachmentRecords.length > 0 && { attachments: attachmentRecords }), - status_created_at: `${TaskStatus.SUBMITTED}#${now}`, + status_created_at: `${initialStatus}#${now}`, created_at: now, updated_at: now, // Cedar HITL extensions (§10.2). Only written when the submit @@ -584,8 +615,10 @@ export async function createTaskCore( approval_gate_cap_source: blueprintCap !== undefined ? 'blueprint' : 'platform_default', }); - // 8. Async-invoke the orchestrator (fire-and-forget) - if (lambdaClient && process.env.ORCHESTRATOR_FUNCTION_ARN) { + // 8. Async-invoke the orchestrator (fire-and-forget). + // Skip for PENDING_UPLOADS — the orchestrator is invoked from confirm-uploads + // once all attachments are uploaded and screened. + if (initialStatus === TaskStatus.SUBMITTED && lambdaClient && process.env.ORCHESTRATOR_FUNCTION_ARN) { try { await lambdaClient.send(new InvokeCommand({ FunctionName: process.env.ORCHESTRATOR_FUNCTION_ARN, @@ -607,18 +640,58 @@ export async function createTaskCore( } // 9. Return created task - return successResponse(201, toTaskDetail(taskRecord), requestId); + const statusCode = hasPresignedAttachments ? 202 : 201; + return successResponse(statusCode, toTaskDetail(taskRecord), requestId); +} + +// --------------------------------------------------------------------------- +// Image token estimation (matches Anthropic's documented resize rules) +// --------------------------------------------------------------------------- + +const MAX_IMAGE_SIDE = 1568; +const MAX_IMAGE_TOKENS = 1568; +const TOKEN_SAFETY_MARGIN = 1.2; +const TILE_SIZE = 28; + +function estimateImageTokens(width: number, height: number): number { + let w = width; + let h = height; + + // Scale to fit MAX_IMAGE_SIDE on longest side + const maxSide = Math.max(w, h); + if (maxSide > MAX_IMAGE_SIDE) { + const scale = MAX_IMAGE_SIDE / maxSide; + w = Math.round(w * scale); + h = Math.round(h * scale); + } + + // Pad to next multiple of tile size + w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; + h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; + + // Token calculation with safety margin, then capped to hard ceiling + const rawTokens = Math.ceil((w * h) / 750); + return Math.min(Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN), MAX_IMAGE_TOKENS); } /** - * Map MIME type to Bedrock GuardrailImageFormat. - * The SDK currently supports 'png' | 'jpeg' — GIF and WebP are mapped - * to 'png' (lossless container) since the guardrail inspects visual - * content, not codec fidelity. + * Estimate image tokens from a buffer by reading dimensions via sharp. + * Returns undefined if dimensions cannot be determined (non-fatal for inline path). */ -function mimeToGuardrailFormat(contentType: string): 'png' | 'jpeg' { - if (contentType === 'image/jpeg') return 'jpeg'; - return 'png'; +async function estimateImageTokensFromBuffer(content: Buffer): Promise { + try { + const sharp = (await import('sharp')).default; + const metadata = await sharp(content).metadata(); + if (metadata.width && metadata.height) { + return estimateImageTokens(metadata.width, metadata.height); + } + } catch (err) { + logger.warn('Failed to estimate image tokens (non-fatal)', { + error: err instanceof Error ? err.message : String(err), + content_length: content.length, + }); + } + return undefined; } const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/; diff --git a/cdk/src/handlers/slack-interactions.ts b/cdk/src/handlers/slack-interactions.ts index 59b68d1f..ac25ccff 100644 --- a/cdk/src/handlers/slack-interactions.ts +++ b/cdk/src/handlers/slack-interactions.ts @@ -132,21 +132,23 @@ async function handleCancelAction(payload: SlackInteractionPayload, actionId: st } // Attempt to cancel. - const ACTIVE_STATUSES = ['SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING']; + const CANCELLABLE_STATUSES = ['PENDING_UPLOADS', 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING']; try { await ddb.send(new UpdateCommand({ TableName: TASK_TABLE, Key: { task_id: taskId }, UpdateExpression: 'SET #s = :cancelled, updated_at = :now', - ConditionExpression: '#s IN (:s1, :s2, :s3, :s4)', + ConditionExpression: '#s IN (:s1, :s2, :s3, :s4, :s5, :s6)', ExpressionAttributeNames: { '#s': 'status' }, ExpressionAttributeValues: { ':cancelled': 'CANCELLED', ':now': new Date().toISOString(), - ':s1': ACTIVE_STATUSES[0], - ':s2': ACTIVE_STATUSES[1], - ':s3': ACTIVE_STATUSES[2], - ':s4': ACTIVE_STATUSES[3], + ':s1': CANCELLABLE_STATUSES[0], + ':s2': CANCELLABLE_STATUSES[1], + ':s3': CANCELLABLE_STATUSES[2], + ':s4': CANCELLABLE_STATUSES[3], + ':s5': CANCELLABLE_STATUSES[4], + ':s6': CANCELLABLE_STATUSES[5], }, })); diff --git a/cdk/src/types/pdf-parse.d.ts b/cdk/src/types/pdf-parse.d.ts new file mode 100644 index 00000000..82ec15de --- /dev/null +++ b/cdk/src/types/pdf-parse.d.ts @@ -0,0 +1,9 @@ +declare module 'pdf-parse' { + interface PdfParseResult { + text: string; + numpages: number; + info: Record; + } + function pdfParse(data: Buffer, options?: { max?: number }): Promise; + export = pdfParse; +} diff --git a/cdk/test/constructs/task-status.test.ts b/cdk/test/constructs/task-status.test.ts index 2a183fe9..cf01ee8c 100644 --- a/cdk/test/constructs/task-status.test.ts +++ b/cdk/test/constructs/task-status.test.ts @@ -17,19 +17,19 @@ * SOFTWARE. */ -import { ACTIVE_STATUSES, TaskStatus, TaskStatusType, TERMINAL_STATUSES, VALID_TRANSITIONS } from '../../src/constructs/task-status'; +import { ACTIVE_STATUSES, PRE_ACTIVE_STATUSES, TaskStatus, TaskStatusType, TERMINAL_STATUSES, VALID_TRANSITIONS } from '../../src/constructs/task-status'; const ALL_STATUSES: TaskStatusType[] = Object.values(TaskStatus); describe('TaskStatus', () => { - test('defines exactly 9 states', () => { - // 8 original + AWAITING_APPROVAL (Cedar HITL gates, §10.3). - expect(ALL_STATUSES).toHaveLength(9); + test('defines exactly 10 states', () => { + // 8 original + AWAITING_APPROVAL (Cedar HITL gates, §10.3) + PENDING_UPLOADS (attachments). + expect(ALL_STATUSES).toHaveLength(10); }); test('contains all expected states', () => { expect(ALL_STATUSES).toEqual(expect.arrayContaining([ - 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', + 'PENDING_UPLOADS', 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', 'COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT', ])); }); @@ -38,6 +38,11 @@ describe('TaskStatus', () => { expect(TaskStatus.AWAITING_APPROVAL).toBe('AWAITING_APPROVAL'); expect(ALL_STATUSES).toContain('AWAITING_APPROVAL'); }); + + test('PENDING_UPLOADS is included as a distinct state', () => { + expect(TaskStatus.PENDING_UPLOADS).toBe('PENDING_UPLOADS'); + expect(ALL_STATUSES).toContain('PENDING_UPLOADS'); + }); }); describe('TERMINAL_STATUSES', () => { @@ -73,14 +78,36 @@ describe('ACTIVE_STATUSES', () => { }); }); -describe('TERMINAL_STATUSES and ACTIVE_STATUSES', () => { - test('are disjoint (no overlap)', () => { - const overlap = TERMINAL_STATUSES.filter(s => ACTIVE_STATUSES.includes(s)); - expect(overlap).toHaveLength(0); +describe('PRE_ACTIVE_STATUSES', () => { + test('contains exactly 1 pre-active state', () => { + expect(PRE_ACTIVE_STATUSES).toHaveLength(1); + }); + + test('contains PENDING_UPLOADS', () => { + expect(PRE_ACTIVE_STATUSES).toContain(TaskStatus.PENDING_UPLOADS); + }); + + test('PENDING_UPLOADS is NOT in ACTIVE_STATUSES (no concurrency slot consumed)', () => { + expect(ACTIVE_STATUSES).not.toContain(TaskStatus.PENDING_UPLOADS); + }); + + test('PENDING_UPLOADS is NOT terminal', () => { + expect(TERMINAL_STATUSES).not.toContain(TaskStatus.PENDING_UPLOADS); + }); +}); + +describe('TERMINAL_STATUSES, ACTIVE_STATUSES, and PRE_ACTIVE_STATUSES', () => { + test('are pairwise disjoint (no overlap)', () => { + const overlapTA = TERMINAL_STATUSES.filter(s => ACTIVE_STATUSES.includes(s)); + const overlapTP = TERMINAL_STATUSES.filter(s => PRE_ACTIVE_STATUSES.includes(s)); + const overlapAP = ACTIVE_STATUSES.filter(s => PRE_ACTIVE_STATUSES.includes(s)); + expect(overlapTA).toHaveLength(0); + expect(overlapTP).toHaveLength(0); + expect(overlapAP).toHaveLength(0); }); test('together cover all states', () => { - const combined = [...TERMINAL_STATUSES, ...ACTIVE_STATUSES]; + const combined = [...TERMINAL_STATUSES, ...ACTIVE_STATUSES, ...PRE_ACTIVE_STATUSES]; expect(combined).toHaveLength(ALL_STATUSES.length); expect(combined).toEqual(expect.arrayContaining(ALL_STATUSES)); }); @@ -150,4 +177,28 @@ describe('VALID_TRANSITIONS', () => { // gate can still fire; §10.3 explicitly lists it. expect(VALID_TRANSITIONS[TaskStatus.HYDRATING]).toContain(TaskStatus.AWAITING_APPROVAL); }); + + test('PENDING_UPLOADS can transition to SUBMITTED (confirm-uploads success)', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).toContain(TaskStatus.SUBMITTED); + }); + + test('PENDING_UPLOADS can transition to FAILED (screening blocked)', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).toContain(TaskStatus.FAILED); + }); + + test('PENDING_UPLOADS can transition to CANCELLED (user cancel or auto-cancel)', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).toContain(TaskStatus.CANCELLED); + }); + + test('PENDING_UPLOADS cannot skip to RUNNING or later states', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).not.toContain(TaskStatus.RUNNING); + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).not.toContain(TaskStatus.HYDRATING); + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).not.toContain(TaskStatus.FINALIZING); + }); + + test('pre-active states have at least one outgoing transition', () => { + for (const status of PRE_ACTIVE_STATUSES) { + expect(VALID_TRANSITIONS[status].length).toBeGreaterThan(0); + } + }); }); diff --git a/cli/src/types.ts b/cli/src/types.ts index 5450711b..94792390 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -30,6 +30,7 @@ export type AttachmentType = 'image' | 'file' | 'url'; * surface stays portable. */ export type TaskStatusType = + | 'PENDING_UPLOADS' | 'SUBMITTED' | 'HYDRATING' | 'RUNNING' diff --git a/docs/design/ATTACHMENTS.md b/docs/design/ATTACHMENTS.md index cf744f8c..6ee5dcd2 100644 --- a/docs/design/ATTACHMENTS.md +++ b/docs/design/ATTACHMENTS.md @@ -1293,7 +1293,7 @@ export const VALID_TRANSITIONS: Record = { ```typescript export const ACTIVE_STATUSES: TaskStatusType[] = [ // PENDING_UPLOADS is NOT here — does not count against concurrency - 'SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING', + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', ]; export const PRE_ACTIVE_STATUSES: TaskStatusType[] = [ @@ -1316,11 +1316,16 @@ stateDiagram-v2 PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel SUBMITTED --> HYDRATING : Admission passes HYDRATING --> RUNNING : Context assembled + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate + AWAITING_APPROVAL --> RUNNING : Approved / denied (resume) RUNNING --> FINALIZING : Session ends FINALIZING --> COMPLETED : Success FINALIZING --> FAILED : Agent failure ``` +**Relationship to AWAITING_APPROVAL:** `PENDING_UPLOADS` and `AWAITING_APPROVAL` (Cedar HITL) are independent lifecycle stages with no transition path between them. `PENDING_UPLOADS` is pre-pipeline (no compute allocated, no concurrency slot consumed). `AWAITING_APPROVAL` is mid-pipeline (container alive, concurrency slot held, paused on a human decision). A task with presigned attachments may later hit an approval gate: `PENDING_UPLOADS → SUBMITTED → ... → RUNNING → AWAITING_APPROVAL → RUNNING → ...`. The agent's IAM-based attachment downloads are unaffected by approval wait time (no presigned URL expiry). + ### Auto-cancel mechanism Tasks in `PENDING_UPLOADS` for > 30 minutes are auto-cancelled via an **EventBridge scheduled rule** (not DynamoDB TTL — TTL would delete the record, losing audit trail). The rule: diff --git a/docs/design/ORCHESTRATOR.md b/docs/design/ORCHESTRATOR.md index 920a67c0..6f8089df 100644 --- a/docs/design/ORCHESTRATOR.md +++ b/docs/design/ORCHESTRATOR.md @@ -52,9 +52,11 @@ Every task moves through a finite set of states from creation to a terminal outc | State | Description | Duration | |---|---|---| +| `PENDING_UPLOADS` | Presigned-upload task awaiting client file uploads | Minutes (30-min auto-cancel) | | `SUBMITTED` | Task accepted, awaiting orchestration | Milliseconds | | `HYDRATING` | Fetching GitHub data, querying memory, assembling prompt | Seconds | | `RUNNING` | Agent session active in compute environment | Minutes to hours | +| `AWAITING_APPROVAL` | Cedar HITL soft-deny gate fired; paused on human decision | Minutes to hours | | `FINALIZING` | Result inference and cleanup in progress | Seconds | | `COMPLETED` | Terminal. Task finished successfully | - | | `FAILED` | Terminal. Task could not complete | - | @@ -65,20 +67,31 @@ Every task moves through a finite set of states from creation to a terminal outc ```mermaid stateDiagram-v2 - [*] --> SUBMITTED + [*] --> PENDING_UPLOADS : Presigned upload task + [*] --> SUBMITTED : Inline/no-attachment task + PENDING_UPLOADS --> SUBMITTED : confirm-uploads succeeds + PENDING_UPLOADS --> FAILED : Screening blocked + PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel + SUBMITTED --> HYDRATING : Admission passes SUBMITTED --> FAILED : Admission rejected SUBMITTED --> CANCELLED : User cancels HYDRATING --> RUNNING : Session started + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate HYDRATING --> FAILED : Hydration error HYDRATING --> CANCELLED : User cancels + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate RUNNING --> FINALIZING : Session ends RUNNING --> CANCELLED : User cancels RUNNING --> TIMED_OUT : Duration exceeded RUNNING --> FAILED : Session crash + AWAITING_APPROVAL --> RUNNING : Approved or denied (resume) + AWAITING_APPROVAL --> CANCELLED : User cancels mid-approval + AWAITING_APPROVAL --> FAILED : Stranded-approval reconciler + FINALIZING --> COMPLETED : PR or commits found FINALIZING --> FAILED : No useful work FINALIZING --> TIMED_OUT : Idle timeout detected @@ -88,13 +101,21 @@ stateDiagram-v2 | From | To | Trigger | Condition | |---|---|---|---| +| `PENDING_UPLOADS` | `SUBMITTED` | `confirm-uploads` succeeds | All attachments screened and passed | +| `PENDING_UPLOADS` | `FAILED` | Screening blocked | Any attachment fails security screening | +| `PENDING_UPLOADS` | `CANCELLED` | User cancels or auto-cancel | Upload window expired (30 min) or explicit cancel | | `SUBMITTED` | `HYDRATING` | Admission passes | Concurrency slot acquired | | `SUBMITTED` | `FAILED` | Admission rejected | Repo not onboarded, rate/concurrency limit, validation error | | `HYDRATING` | `RUNNING` | Hydration complete | `invoke_agent_runtime` returns session ID | +| `HYDRATING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during hydration | | `HYDRATING` | `FAILED` | Hydration error | GitHub API failure, guardrail blocks content, Bedrock unavailable | +| `RUNNING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during execution | | `RUNNING` | `FINALIZING` | Session ends | Response received or session terminated | | `RUNNING` | `TIMED_OUT` | Max duration exceeded | Wall-clock timer (default 8h, matching AgentCore max) | | `RUNNING` | `FAILED` | Session crash | Heartbeat lost (see Liveness monitoring) | +| `AWAITING_APPROVAL` | `RUNNING` | Approved or denied | Human decision received; agent resumes | +| `AWAITING_APPROVAL` | `CANCELLED` | User cancels | Explicit cancel while awaiting approval | +| `AWAITING_APPROVAL` | `FAILED` | Stranded reconciler | Approval request orphaned (agent died mid-wait) | | `FINALIZING` | `COMPLETED` | Success inferred | PR exists or commits on branch | | `FINALIZING` | `FAILED` | Failure inferred | No commits, no PR, or agent reported error | @@ -104,9 +125,11 @@ Users can cancel a task at any point. The orchestrator's response depends on how | State when cancel arrives | Action | |---|---| +| `PENDING_UPLOADS` | Transition to `CANCELLED`. Clean up S3 objects under the task's attachment prefix. No concurrency slot to release. | | `SUBMITTED` | Transition to `CANCELLED`. No cleanup needed. | | `HYDRATING` | Abort hydration, release concurrency slot, transition to `CANCELLED`. | | `RUNNING` | Call `stop_runtime_session`, wait for confirmation, release concurrency, transition to `CANCELLED`. Partial work on GitHub remains for the user to inspect. | +| `AWAITING_APPROVAL` | Call `stop_runtime_session`, release concurrency slot, transition to `CANCELLED`. The pending approval row transitions to `STRANDED`. | | `FINALIZING` | Let finalization complete. Mark `CANCELLED` only if the terminal state was not yet written. | | Terminal | Reject the cancel request. | diff --git a/docs/src/content/docs/architecture/Attachments.md b/docs/src/content/docs/architecture/Attachments.md index 9c0c651a..de2ecba9 100644 --- a/docs/src/content/docs/architecture/Attachments.md +++ b/docs/src/content/docs/architecture/Attachments.md @@ -1297,7 +1297,7 @@ export const VALID_TRANSITIONS: Record = { ```typescript export const ACTIVE_STATUSES: TaskStatusType[] = [ // PENDING_UPLOADS is NOT here — does not count against concurrency - 'SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING', + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', ]; export const PRE_ACTIVE_STATUSES: TaskStatusType[] = [ @@ -1320,11 +1320,16 @@ stateDiagram-v2 PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel SUBMITTED --> HYDRATING : Admission passes HYDRATING --> RUNNING : Context assembled + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate + AWAITING_APPROVAL --> RUNNING : Approved / denied (resume) RUNNING --> FINALIZING : Session ends FINALIZING --> COMPLETED : Success FINALIZING --> FAILED : Agent failure ``` +**Relationship to AWAITING_APPROVAL:** `PENDING_UPLOADS` and `AWAITING_APPROVAL` (Cedar HITL) are independent lifecycle stages with no transition path between them. `PENDING_UPLOADS` is pre-pipeline (no compute allocated, no concurrency slot consumed). `AWAITING_APPROVAL` is mid-pipeline (container alive, concurrency slot held, paused on a human decision). A task with presigned attachments may later hit an approval gate: `PENDING_UPLOADS → SUBMITTED → ... → RUNNING → AWAITING_APPROVAL → RUNNING → ...`. The agent's IAM-based attachment downloads are unaffected by approval wait time (no presigned URL expiry). + ### Auto-cancel mechanism Tasks in `PENDING_UPLOADS` for > 30 minutes are auto-cancelled via an **EventBridge scheduled rule** (not DynamoDB TTL — TTL would delete the record, losing audit trail). The rule: diff --git a/docs/src/content/docs/architecture/Orchestrator.md b/docs/src/content/docs/architecture/Orchestrator.md index 77b96b6b..15d931ba 100644 --- a/docs/src/content/docs/architecture/Orchestrator.md +++ b/docs/src/content/docs/architecture/Orchestrator.md @@ -56,9 +56,11 @@ Every task moves through a finite set of states from creation to a terminal outc | State | Description | Duration | |---|---|---| +| `PENDING_UPLOADS` | Presigned-upload task awaiting client file uploads | Minutes (30-min auto-cancel) | | `SUBMITTED` | Task accepted, awaiting orchestration | Milliseconds | | `HYDRATING` | Fetching GitHub data, querying memory, assembling prompt | Seconds | | `RUNNING` | Agent session active in compute environment | Minutes to hours | +| `AWAITING_APPROVAL` | Cedar HITL soft-deny gate fired; paused on human decision | Minutes to hours | | `FINALIZING` | Result inference and cleanup in progress | Seconds | | `COMPLETED` | Terminal. Task finished successfully | - | | `FAILED` | Terminal. Task could not complete | - | @@ -69,20 +71,31 @@ Every task moves through a finite set of states from creation to a terminal outc ```mermaid stateDiagram-v2 - [*] --> SUBMITTED + [*] --> PENDING_UPLOADS : Presigned upload task + [*] --> SUBMITTED : Inline/no-attachment task + PENDING_UPLOADS --> SUBMITTED : confirm-uploads succeeds + PENDING_UPLOADS --> FAILED : Screening blocked + PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel + SUBMITTED --> HYDRATING : Admission passes SUBMITTED --> FAILED : Admission rejected SUBMITTED --> CANCELLED : User cancels HYDRATING --> RUNNING : Session started + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate HYDRATING --> FAILED : Hydration error HYDRATING --> CANCELLED : User cancels + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate RUNNING --> FINALIZING : Session ends RUNNING --> CANCELLED : User cancels RUNNING --> TIMED_OUT : Duration exceeded RUNNING --> FAILED : Session crash + AWAITING_APPROVAL --> RUNNING : Approved or denied (resume) + AWAITING_APPROVAL --> CANCELLED : User cancels mid-approval + AWAITING_APPROVAL --> FAILED : Stranded-approval reconciler + FINALIZING --> COMPLETED : PR or commits found FINALIZING --> FAILED : No useful work FINALIZING --> TIMED_OUT : Idle timeout detected @@ -92,13 +105,21 @@ stateDiagram-v2 | From | To | Trigger | Condition | |---|---|---|---| +| `PENDING_UPLOADS` | `SUBMITTED` | `confirm-uploads` succeeds | All attachments screened and passed | +| `PENDING_UPLOADS` | `FAILED` | Screening blocked | Any attachment fails security screening | +| `PENDING_UPLOADS` | `CANCELLED` | User cancels or auto-cancel | Upload window expired (30 min) or explicit cancel | | `SUBMITTED` | `HYDRATING` | Admission passes | Concurrency slot acquired | | `SUBMITTED` | `FAILED` | Admission rejected | Repo not onboarded, rate/concurrency limit, validation error | | `HYDRATING` | `RUNNING` | Hydration complete | `invoke_agent_runtime` returns session ID | +| `HYDRATING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during hydration | | `HYDRATING` | `FAILED` | Hydration error | GitHub API failure, guardrail blocks content, Bedrock unavailable | +| `RUNNING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during execution | | `RUNNING` | `FINALIZING` | Session ends | Response received or session terminated | | `RUNNING` | `TIMED_OUT` | Max duration exceeded | Wall-clock timer (default 8h, matching AgentCore max) | | `RUNNING` | `FAILED` | Session crash | Heartbeat lost (see Liveness monitoring) | +| `AWAITING_APPROVAL` | `RUNNING` | Approved or denied | Human decision received; agent resumes | +| `AWAITING_APPROVAL` | `CANCELLED` | User cancels | Explicit cancel while awaiting approval | +| `AWAITING_APPROVAL` | `FAILED` | Stranded reconciler | Approval request orphaned (agent died mid-wait) | | `FINALIZING` | `COMPLETED` | Success inferred | PR exists or commits on branch | | `FINALIZING` | `FAILED` | Failure inferred | No commits, no PR, or agent reported error | @@ -108,9 +129,11 @@ Users can cancel a task at any point. The orchestrator's response depends on how | State when cancel arrives | Action | |---|---| +| `PENDING_UPLOADS` | Transition to `CANCELLED`. Clean up S3 objects under the task's attachment prefix. No concurrency slot to release. | | `SUBMITTED` | Transition to `CANCELLED`. No cleanup needed. | | `HYDRATING` | Abort hydration, release concurrency slot, transition to `CANCELLED`. | | `RUNNING` | Call `stop_runtime_session`, wait for confirmation, release concurrency, transition to `CANCELLED`. Partial work on GitHub remains for the user to inspect. | +| `AWAITING_APPROVAL` | Call `stop_runtime_session`, release concurrency slot, transition to `CANCELLED`. The pending approval row transitions to `STRANDED`. | | `FINALIZING` | Let finalization complete. Mark `CANCELLED` only if the terminal state was not yet written. | | Terminal | Reject the cancel request. | From 485b3796f0d88271ea0e8d9b2aff4867623782a9 Mon Sep 17 00:00:00 2001 From: bgagent Date: Wed, 20 May 2026 20:06:35 -0500 Subject: [PATCH 06/19] chore(attachments): finish implementaiton --- agent/src/attachments.py | 129 +++ agent/src/config.py | 15 +- agent/src/models.py | 36 + agent/src/pipeline.py | 47 ++ agent/src/server.py | 4 + cdk/package.json | 6 +- cdk/src/constructs/pending-upload-cleanup.ts | 130 +++ cdk/src/constructs/task-api.ts | 66 ++ cdk/src/constructs/task-status.ts | 29 +- cdk/src/handlers/cleanup-pending-uploads.ts | 276 +++++++ cdk/src/handlers/confirm-uploads.ts | 752 ++++++++++++++++++ cdk/src/handlers/linear-webhook-processor.ts | 35 + .../handlers/shared/attachment-screening.ts | 388 +++++++++ cdk/src/handlers/shared/create-task-core.ts | 252 ++++-- cdk/src/handlers/shared/image-tokens.ts | 73 ++ cdk/src/handlers/shared/orchestrator.ts | 112 ++- .../shared/resolve-url-attachments.ts | 447 +++++++++++ cdk/src/handlers/shared/response.ts | 4 + cdk/src/handlers/shared/types.ts | 6 + cdk/src/handlers/slack-command-processor.ts | 123 +++ cdk/src/handlers/slack-events.ts | 15 +- cdk/src/handlers/slack-interactions.ts | 14 +- cdk/src/stacks/agent.ts | 12 + cdk/src/types/pdf-parse.d.ts | 9 + cdk/test/constructs/task-status.test.ts | 71 +- cdk/test/handlers/confirm-uploads.test.ts | 199 +++++ .../handlers/linear-webhook-processor.test.ts | 74 ++ .../handlers/slack-command-processor.test.ts | 110 +++ cli/src/api-client.ts | 11 +- cli/src/commands/submit.ts | 182 ++++- cli/src/types.ts | 7 + docs/design/ATTACHMENTS.md | 7 +- docs/design/ORCHESTRATOR.md | 25 +- .../content/docs/architecture/Attachments.md | 7 +- .../content/docs/architecture/Orchestrator.md | 25 +- yarn.lock | 296 ++++++- 36 files changed, 3859 insertions(+), 135 deletions(-) create mode 100644 agent/src/attachments.py create mode 100644 cdk/src/constructs/pending-upload-cleanup.ts create mode 100644 cdk/src/handlers/cleanup-pending-uploads.ts create mode 100644 cdk/src/handlers/confirm-uploads.ts create mode 100644 cdk/src/handlers/shared/attachment-screening.ts create mode 100644 cdk/src/handlers/shared/image-tokens.ts create mode 100644 cdk/src/handlers/shared/resolve-url-attachments.ts create mode 100644 cdk/src/types/pdf-parse.d.ts create mode 100644 cdk/test/handlers/confirm-uploads.test.ts diff --git a/agent/src/attachments.py b/agent/src/attachments.py new file mode 100644 index 00000000..05bb0d6b --- /dev/null +++ b/agent/src/attachments.py @@ -0,0 +1,129 @@ +"""Attachment download and integrity verification (Phase 3). + +Downloads attachments from S3 using version-pinned reads and verifies +SHA-256 checksums against the orchestrator-provided values. Files are +placed in a workspace subdirectory for the agent to reference. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +from typing import Literal +from urllib.parse import urlparse + +from pydantic import BaseModel, ConfigDict + +from shell import log + +ATTACHMENTS_DIR = ".attachments" + + +class PreparedAttachment(BaseModel): + """An attachment downloaded to the local filesystem and verified.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + attachment_id: str + type: Literal["image", "file", "url"] + content_type: str + filename: str + local_path: str + size_bytes: int + token_estimate: int | None = None + + +def download_attachments( + attachments: list, + workspace: str, +) -> list[PreparedAttachment]: + """Download all attachments from S3 and verify integrity. + + Args: + attachments: List of AttachmentConfig models from TaskConfig. + workspace: The agent workspace root (e.g., /workspace). + + Returns: + List of PreparedAttachment with local file paths. + + Raises: + RuntimeError: If any attachment fails download or integrity check. + """ + if not attachments: + return [] + + import boto3 + + attachments_dir = Path(workspace) / ATTACHMENTS_DIR + attachments_dir.mkdir(parents=True, exist_ok=True) + + s3_client = boto3.client("s3") + prepared: list[PreparedAttachment] = [] + + for att in attachments: + local_path = _download_single(att, attachments_dir, s3_client) + prepared.append( + PreparedAttachment( + attachment_id=att.attachment_id, + type=att.type, + content_type=att.content_type, + filename=att.filename, + local_path=str(local_path), + size_bytes=att.size_bytes, + token_estimate=att.token_estimate, + ) + ) + + log("TASK", f"Downloaded {len(prepared)} attachment(s) to {attachments_dir}") + return prepared + + +def _download_single(att, attachments_dir: Path, s3_client) -> Path: + """Download a single attachment and verify its SHA-256 checksum.""" + # Parse s3_uri (s3://bucket/key) + parsed = urlparse(att.s3_uri) + bucket = parsed.netloc + key = parsed.path.lstrip("/") + + # Unique subdirectory per attachment to avoid filename collisions + dest_dir = attachments_dir / att.attachment_id + dest_dir.mkdir(parents=True, exist_ok=True) + local_path = dest_dir / att.filename + + log( + "TASK", + f"Downloading attachment '{att.filename}' " + f"(s3://{bucket}/{key}, version={att.s3_version_id})", + ) + + # Download with pinned VersionId to prevent TOCTOU + response = s3_client.get_object( + Bucket=bucket, + Key=key, + VersionId=att.s3_version_id, + ) + content = response["Body"].read() + + # Verify SHA-256 integrity + actual_checksum = hashlib.sha256(content).hexdigest() + if actual_checksum != att.checksum_sha256: + raise RuntimeError( + f"Attachment '{att.filename}' integrity check failed: " + f"expected SHA-256 {att.checksum_sha256}, got {actual_checksum}. " + f"The file may have been tampered with." + ) + + # Verify size matches + if len(content) != att.size_bytes: + raise RuntimeError( + f"Attachment '{att.filename}' size mismatch: " + f"expected {att.size_bytes} bytes, got {len(content)} bytes." + ) + + # Write to local filesystem + local_path.write_bytes(content) + os.chmod(str(local_path), 0o444) # Read-only + + log("TASK", f" Verified: {att.filename} ({len(content)} bytes, SHA-256 OK)") + return local_path diff --git a/agent/src/config.py b/agent/src/config.py index 5f0934d2..06e8731a 100644 --- a/agent/src/config.py +++ b/agent/src/config.py @@ -4,7 +4,7 @@ import sys import uuid -from models import TaskConfig, TaskType +from models import AttachmentConfig, TaskConfig, TaskType from shell import log AGENT_WORKSPACE = os.environ.get("AGENT_WORKSPACE", "/workspace") @@ -114,6 +114,7 @@ def build_config( initial_approvals: list[str] | None = None, initial_approval_gate_count: int = 0, approval_gate_cap: int | None = None, + attachments: list[dict] | None = None, ) -> TaskConfig: """Build and validate configuration from explicit parameters. @@ -149,6 +150,17 @@ def build_config( if errors: raise ValueError("; ".join(errors)) + # Validate attachment descriptors into typed models (Pydantic validation + # surfaces schema mismatches between the orchestrator and agent early). + validated_attachments: list[AttachmentConfig] = [] + if attachments: + for i, raw_att in enumerate(attachments): + try: + validated_attachments.append(AttachmentConfig.model_validate(raw_att)) + except Exception as e: + log("ERROR", f"Attachment[{i}] validation failed: {e}") + raise ValueError(f"Attachment[{i}] validation failed: {e}") from e + return TaskConfig( repo_url=resolved_repo_url, issue_number=resolved_issue_number, @@ -172,6 +184,7 @@ def build_config( initial_approvals=initial_approvals or [], initial_approval_gate_count=initial_approval_gate_count, approval_gate_cap=approval_gate_cap, + attachments=validated_attachments, ) diff --git a/agent/src/models.py b/agent/src/models.py index bda8fe13..007b94bc 100644 --- a/agent/src/models.py +++ b/agent/src/models.py @@ -61,6 +61,39 @@ class MemoryContext(BaseModel): # (see cdk/src/handlers/shared/context-hydration.ts). SUPPORTED_HYDRATED_CONTEXT_VERSION = 1 +# Attachment types — mirrors AttachmentType in cdk/src/handlers/shared/types.ts. +AttachmentType = Literal["image", "file", "url"] + + +class AttachmentConfig(BaseModel): + """Attachment descriptor from the orchestrator — mirrors AgentAttachmentPayload in types.ts.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + attachment_id: str + type: AttachmentType + content_type: str + filename: str + s3_uri: str + s3_version_id: str + size_bytes: int + source_url: str | None = None + token_estimate: int | None = None + checksum_sha256: str + + @model_validator(mode="after") + def _validate_integrity_fields(self) -> Self: + if not self.s3_version_id: + raise ValueError("s3_version_id is required for integrity verification") + if not self.checksum_sha256: + raise ValueError("checksum_sha256 is required for integrity verification") + # checksum must be lowercase hex (SHA-256 = 64 hex chars) + if len(self.checksum_sha256) != 64 or not all( + c in "0123456789abcdef" for c in self.checksum_sha256 + ): + raise ValueError("checksum_sha256 must be a 64-character lowercase hex string") + return self + class HydratedContext(BaseModel): """Orchestrator context JSON — keep in sync with HydratedContext in context-hydration.ts.""" @@ -150,6 +183,9 @@ class TaskConfig(BaseModel): approval_gate_cap: int | None = None issue: GitHubIssue | None = None base_branch: str | None = None + # Attachments from the orchestrator payload (Phase 3). Validated as + # AttachmentConfig models. Empty list for tasks without attachments. + attachments: list[AttachmentConfig] = Field(default_factory=list) @model_validator(mode="after") def _validate_trace_requires_user_id(self) -> Self: diff --git a/agent/src/pipeline.py b/agent/src/pipeline.py index 5b1cd5b9..2845afac 100644 --- a/agent/src/pipeline.py +++ b/agent/src/pipeline.py @@ -57,6 +57,32 @@ def _chain_prior_agent_error(agent_result: AgentResult | None, exc: BaseExceptio return tail +def _inject_attachment_context(prompt: str, prepared_attachments: list) -> str: + """Append attachment file references to the user prompt. + + Images are referenced by absolute path so the agent can view them + with the Read tool (which supports multimodal image reading). + File attachments are similarly referenced by path. + """ + lines = ["\n\n---\n\n**Attachments provided with this task:**\n"] + for att in prepared_attachments: + size_kb = att.size_bytes / 1024 + if att.type == "image": + lines.append( + f"- **Image:** `{att.filename}` ({size_kb:.1f} KB, {att.content_type}) " + f"— View with: `Read {att.local_path}`" + ) + else: + lines.append( + f"- **File:** `{att.filename}` ({size_kb:.1f} KB, {att.content_type}) " + f"— Read with: `Read {att.local_path}`" + ) + lines.append( + "\nUse the Read tool to view these files. Image files will be displayed visually when read." + ) + return prompt + "\n".join(lines) + + def _maybe_upload_trace( config: TaskConfig, trajectory, @@ -252,6 +278,7 @@ def run_task( channel_metadata: dict[str, str] | None = None, trace: bool = False, user_id: str = "", + attachments: list[dict] | None = None, ) -> dict: """Run the full agent pipeline and return a serialized result dict. @@ -290,6 +317,7 @@ def run_task( initial_approvals=initial_approvals, initial_approval_gate_count=initial_approval_gate_count, approval_gate_cap=approval_gate_cap, + attachments=attachments, ) # Inject Cedar policies into config for the PolicyEngine in runner.py @@ -440,6 +468,18 @@ def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None: config.channel_metadata, ) + # Download attachments from S3 (version-pinned, integrity-verified) + prepared_attachments: list = [] + if config.attachments: + from attachments import download_attachments + + with task_span("task.attachment_download"): + prepared_attachments = download_attachments(config.attachments, setup.repo_dir) + progress.write_agent_milestone( + "attachments_downloaded", + f"count={len(prepared_attachments)}", + ) + # Log discovered repo-level project configuration # (all files loaded by setting_sources=["project"]) repo_dir = setup.repo_dir @@ -449,6 +489,13 @@ def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None: else: log("TASK", "No repo-level project configuration found") + # Inject attachment references into the prompt so the agent knows + # about available files. Images are read natively by the agent's + # Read tool (multimodal support). File attachments are referenced + # by path for the agent to read as needed. + if prepared_attachments: + prompt = _inject_attachment_context(prompt, prepared_attachments) + # Run agent disk_before = get_disk_usage(AGENT_WORKSPACE) start_time = time.time() diff --git a/agent/src/server.py b/agent/src/server.py index 041df5a5..c2879368 100644 --- a/agent/src/server.py +++ b/agent/src/server.py @@ -352,6 +352,7 @@ def _run_task_background( channel_metadata: dict[str, str] | None = None, trace: bool = False, user_id: str = "", + attachments: list[dict] | None = None, ) -> None: """Run the agent task in a background thread.""" global _background_pipeline_failed @@ -405,6 +406,7 @@ def _run_task_background( channel_metadata=channel_metadata, trace=trace, user_id=user_id, + attachments=attachments, ) _background_pipeline_failed = False except Exception as e: @@ -492,6 +494,7 @@ def _extract_invocation_params(inp: dict, request: Request) -> dict: approval_gate_cap = None channel_source = inp.get("channel_source", "") or "" channel_metadata = inp.get("channel_metadata") or {} + attachments = inp.get("attachments") or [] # ``trace`` is strictly opt-in (design §10.1). Accept only real # booleans from the orchestrator — a string "false" would otherwise # flip the flag on. @@ -556,6 +559,7 @@ def _extract_invocation_params(inp: dict, request: Request) -> dict: "channel_metadata": channel_metadata, "trace": trace, "user_id": user_id, + "attachments": attachments, } diff --git a/cdk/package.json b/cdk/package.json index e3d71a2e..72f00348 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -19,18 +19,21 @@ "@aws-cdk/mixins-preview": "2.238.0-alpha.0", "@aws-sdk/client-bedrock-agentcore": "^3.1046.0", "@aws-sdk/client-bedrock-runtime": "^3.1021.0", - "@aws-sdk/client-ecs": "^3.1021.0", "@aws-sdk/client-dynamodb": "^3.1021.0", + "@aws-sdk/client-ecs": "^3.1021.0", "@aws-sdk/client-lambda": "^3.1021.0", "@aws-sdk/client-s3": "^3.1021.0", "@aws-sdk/client-secrets-manager": "^3.1021.0", "@aws-sdk/lib-dynamodb": "^3.1021.0", + "@aws-sdk/s3-presigned-post": "3.1040.0", "@aws-sdk/s3-request-presigner": "^3.1021.0", "@aws/durable-execution-sdk-js": "^1.1.0", "@cedar-policy/cedar-wasm": "4.10.0", "aws-cdk-lib": "^2.238.0", "cdk-nag": "^2.37.55", "constructs": "^10.3.0", + "pdf-parse": "^1.1.1", + "sharp": "^0.33.5", "ulid": "^3.0.2" }, "devDependencies": { @@ -39,6 +42,7 @@ "@types/aws-lambda": "^8.10.161", "@types/jest": "^30.0.0", "@types/node": "^20", + "@types/pdf-parse": "^1.1.4", "@typescript-eslint/eslint-plugin": "^8", "@typescript-eslint/parser": "^8", "aws-cdk": "^2", diff --git a/cdk/src/constructs/pending-upload-cleanup.ts b/cdk/src/constructs/pending-upload-cleanup.ts new file mode 100644 index 00000000..2743b1de --- /dev/null +++ b/cdk/src/constructs/pending-upload-cleanup.ts @@ -0,0 +1,130 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import * as path from 'path'; +import { Duration } from 'aws-cdk-lib'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as targets from 'aws-cdk-lib/aws-events-targets'; +import { Architecture, Runtime } from 'aws-cdk-lib/aws-lambda'; +import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import { NagSuppressions } from 'cdk-nag'; +import { Construct } from 'constructs'; + +/** + * Properties for PendingUploadCleanup construct. + */ +export interface PendingUploadCleanupProps { + /** TaskTable (has StatusIndex GSI used by the query). */ + readonly taskTable: dynamodb.ITable; + + /** TaskEventsTable (handler writes pending_upload_expired events). */ + readonly taskEventsTable: dynamodb.ITable; + + /** Attachments S3 bucket (handler deletes orphaned objects). */ + readonly attachmentsBucket: s3.IBucket; + + /** + * How often to run the cleanup. Defaults to 5 minutes. + * @default Duration.minutes(5) + */ + readonly schedule?: Duration; + + /** + * Time (seconds) before a PENDING_UPLOADS task is auto-cancelled. + * @default 1800 (30 minutes) + */ + readonly pendingUploadTimeoutSeconds?: number; + + /** Task retention days for event TTL. @default 90 */ + readonly taskRetentionDays?: number; +} + +/** + * Scheduled Lambda that auto-cancels stale PENDING_UPLOADS tasks. + * + * Tasks with presigned-upload attachments are created in PENDING_UPLOADS. + * If the client never calls confirm-uploads (crash, abandoned session, + * network issue), these tasks sit indefinitely. This construct runs a + * scheduled Lambda that transitions expired tasks to CANCELLED and + * deletes their orphaned S3 objects. + * + * Race safety: The Lambda uses conditional DynamoDB writes so it cannot + * conflict with a concurrent confirm-uploads call. + */ +export class PendingUploadCleanup extends Construct { + public readonly fn: lambda.NodejsFunction; + + constructor(scope: Construct, id: string, props: PendingUploadCleanupProps) { + super(scope, id); + + const handlersDir = path.join(__dirname, '..', 'handlers'); + + const timeoutSeconds = props.pendingUploadTimeoutSeconds ?? 1800; + const retentionDays = props.taskRetentionDays ?? 90; + + this.fn = new lambda.NodejsFunction(this, 'CleanupFn', { + entry: path.join(handlersDir, 'cleanup-pending-uploads.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + timeout: Duration.seconds(30), + memorySize: 256, + environment: { + TASK_TABLE_NAME: props.taskTable.tableName, + TASK_EVENTS_TABLE_NAME: props.taskEventsTable.tableName, + ATTACHMENTS_BUCKET_NAME: props.attachmentsBucket.bucketName, + PENDING_UPLOAD_TIMEOUT_SECONDS: String(timeoutSeconds), + TASK_RETENTION_DAYS: String(retentionDays), + }, + bundling: { + externalModules: ['@aws-sdk/*'], + }, + }); + + // TaskTable: read (query by StatusIndex) + conditional UpdateItem. + props.taskTable.grantReadWriteData(this.fn); + // TaskEvents: write pending_upload_expired events. + props.taskEventsTable.grantWriteData(this.fn); + // Attachments bucket: list + delete orphaned objects. + props.attachmentsBucket.grantRead(this.fn); + props.attachmentsBucket.grantDelete(this.fn); + + const schedule = props.schedule ?? Duration.minutes(5); + const rule = new events.Rule(this, 'CleanupSchedule', { + schedule: events.Schedule.rate(schedule), + }); + rule.addTarget(new targets.LambdaFunction(this.fn)); + + NagSuppressions.addResourceSuppressions(this.fn, [ + { + id: 'AwsSolutions-IAM4', + reason: 'AWSLambdaBasicExecutionRole is required for CloudWatch Logs access', + }, + { + id: 'AwsSolutions-IAM5', + reason: + 'DynamoDB index/* wildcards generated by CDK grantReadWriteData for ' + + 'StatusIndex query access + Item update path. ' + + 'S3 wildcards from grantRead/grantDelete for prefix-based listing and deletion.', + }, + ], true); + } +} diff --git a/cdk/src/constructs/task-api.ts b/cdk/src/constructs/task-api.ts index 5f3f55cb..a6429e35 100644 --- a/cdk/src/constructs/task-api.ts +++ b/cdk/src/constructs/task-api.ts @@ -149,6 +149,12 @@ export interface TaskApiProps { * gets PutObject/DeleteObject grants and the bucket name as env var. */ readonly attachmentsBucket?: s3.IBucket; + + /** + * User concurrency table for admission control during confirm-uploads. + * Required when attachmentsBucket is provided. + */ + readonly userConcurrencyTable?: dynamodb.ITable; } /** @@ -517,8 +523,62 @@ export class TaskApi extends Construct { props.attachmentsBucket.grantDelete(createTaskFn); } + // --- Confirm-uploads Lambda (presigned upload flow) --- + let confirmUploadsFn: lambda.NodejsFunction | undefined; + if (props.attachmentsBucket && props.userConcurrencyTable) { + const confirmUploadsEnv: Record = { ...commonEnv }; + confirmUploadsEnv.ATTACHMENTS_BUCKET_NAME = props.attachmentsBucket.bucketName; + confirmUploadsEnv.USER_CONCURRENCY_TABLE_NAME = props.userConcurrencyTable.tableName; + if (props.orchestratorFunctionArn) { + confirmUploadsEnv.ORCHESTRATOR_FUNCTION_ARN = props.orchestratorFunctionArn; + } + if (props.guardrailId && props.guardrailVersion) { + confirmUploadsEnv.GUARDRAIL_ID = props.guardrailId; + confirmUploadsEnv.GUARDRAIL_VERSION = props.guardrailVersion; + } + + confirmUploadsFn = new lambda.NodejsFunction(this, 'ConfirmUploadsFn', { + entry: path.join(handlersDir, 'confirm-uploads.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + environment: confirmUploadsEnv, + bundling: commonBundling, + memorySize: 2048, + timeout: Duration.seconds(180), + }); + + // Grants: DDB read-write, S3 read-write-delete, orchestrator invoke, guardrail + props.taskTable.grantReadWriteData(confirmUploadsFn); + props.taskEventsTable.grantReadWriteData(confirmUploadsFn); + props.attachmentsBucket.grantReadWrite(confirmUploadsFn); + props.attachmentsBucket.grantDelete(confirmUploadsFn); + props.userConcurrencyTable.grantReadWriteData(confirmUploadsFn); + + if (props.orchestratorFunctionArn) { + confirmUploadsFn.addToRolePolicy(new iam.PolicyStatement({ + actions: ['lambda:InvokeFunction'], + resources: [props.orchestratorFunctionArn], + })); + } + + if (props.guardrailId) { + confirmUploadsFn.addToRolePolicy(new iam.PolicyStatement({ + actions: ['bedrock:ApplyGuardrail'], + resources: [ + Stack.of(this).formatArn({ + service: 'bedrock', + resource: 'guardrail', + resourceName: props.guardrailId, + }), + ], + })); + } + } + // Collect all Lambda functions for cdk-nag suppressions const allFunctions: lambda.NodejsFunction[] = [createTaskFn, getTaskFn, listTasksFn, cancelTaskFn, getTaskEventsFn]; + if (confirmUploadsFn) allFunctions.push(confirmUploadsFn); // --- API resource tree: /tasks --- const tasks = this.api.root.addResource('tasks'); @@ -532,6 +592,12 @@ export class TaskApi extends Construct { const events = taskById.addResource('events'); events.addMethod('GET', new apigw.LambdaIntegration(getTaskEventsFn), cognitoAuthOptions); + // --- Confirm-uploads endpoint: POST /tasks/{task_id}/confirm-uploads --- + if (confirmUploadsFn) { + const confirmUploads = taskById.addResource('confirm-uploads'); + confirmUploads.addMethod('POST', new apigw.LambdaIntegration(confirmUploadsFn), cognitoAuthOptions); + } + // --- Trace URL endpoint (design §10.1): GET /tasks/{task_id}/trace --- if (props.traceArtifactsBucket) { const traceBucket = props.traceArtifactsBucket; diff --git a/cdk/src/constructs/task-status.ts b/cdk/src/constructs/task-status.ts index ef3dcc89..a5db1e33 100644 --- a/cdk/src/constructs/task-status.ts +++ b/cdk/src/constructs/task-status.ts @@ -20,10 +20,15 @@ /** * Valid task states in the task lifecycle state machine. * - * States progress through the lifecycle: SUBMITTED -> HYDRATING -> - * RUNNING -> FINALIZING -> terminal (COMPLETED / FAILED / CANCELLED / TIMED_OUT). + * States progress through the lifecycle: + * [PENDING_UPLOADS ->] SUBMITTED -> HYDRATING -> + * RUNNING -> FINALIZING -> terminal (COMPLETED / FAILED / CANCELLED / TIMED_OUT). * See ORCHESTRATOR.md for the full state transition table. * + * PENDING_UPLOADS is a pre-active state for tasks with presigned-upload + * attachments: no compute allocated, no concurrency slot consumed. The + * task transitions to SUBMITTED once uploads are confirmed and screened. + * * AWAITING_APPROVAL is the Cedar-HITL soft-deny gate surface: the * task is alive but paused on a human decision. See * `docs/design/CEDAR_HITL_GATES.md` §10.3 for the joint @@ -31,6 +36,7 @@ * must preserve when transitioning in or out of this state. */ export const TaskStatus = { + PENDING_UPLOADS: 'PENDING_UPLOADS', SUBMITTED: 'SUBMITTED', HYDRATING: 'HYDRATING', RUNNING: 'RUNNING', @@ -57,10 +63,20 @@ export const TERMINAL_STATUSES: readonly TaskStatusType[] = [ TaskStatus.TIMED_OUT, ]; +/** + * Pre-active states where the task exists but has not entered the + * orchestration pipeline. No compute resources are allocated and no + * concurrency slot is consumed. + */ +export const PRE_ACTIVE_STATUSES: readonly TaskStatusType[] = [ + TaskStatus.PENDING_UPLOADS, +]; + /** * Active (non-terminal) states that indicate a task is still in progress. * AWAITING_APPROVAL counts as active — the task is alive, just paused - * waiting on a human decision. + * waiting on a human decision. PENDING_UPLOADS is NOT here — it is + * pre-active and does not consume a concurrency slot. */ export const ACTIVE_STATUSES: readonly TaskStatusType[] = [ TaskStatus.SUBMITTED, @@ -73,9 +89,14 @@ export const ACTIVE_STATUSES: readonly TaskStatusType[] = [ /** * Valid state transitions. Maps each state to the set of states it can transition to. * Derived from the transition table in ORCHESTRATOR.md + §10.3 of the - * Cedar HITL gates design (AWAITING_APPROVAL entries). + * Cedar HITL gates design (AWAITING_APPROVAL entries) + ATTACHMENTS.md + * (PENDING_UPLOADS entries). */ export const VALID_TRANSITIONS: Readonly> = { + // PENDING_UPLOADS: presigned-upload task awaiting client file uploads. + // Transitions to SUBMITTED on confirm-uploads success, FAILED on + // screening failure, CANCELLED on user cancel or 30-min auto-cancel. + [TaskStatus.PENDING_UPLOADS]: [TaskStatus.SUBMITTED, TaskStatus.FAILED, TaskStatus.CANCELLED], [TaskStatus.SUBMITTED]: [TaskStatus.HYDRATING, TaskStatus.FAILED, TaskStatus.CANCELLED], [TaskStatus.HYDRATING]: [ TaskStatus.RUNNING, diff --git a/cdk/src/handlers/cleanup-pending-uploads.ts b/cdk/src/handlers/cleanup-pending-uploads.ts new file mode 100644 index 00000000..23a3f7fe --- /dev/null +++ b/cdk/src/handlers/cleanup-pending-uploads.ts @@ -0,0 +1,276 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Scheduled handler: auto-cancel stale PENDING_UPLOADS tasks. + * + * Tasks in PENDING_UPLOADS that have not received a confirm-uploads call + * within the configured timeout (default 30 minutes) are transitioned to + * CANCELLED. This prevents abandoned upload sessions from accumulating + * orphaned S3 objects and cluttering the user's task list. + * + * Race safety: Uses conditional DynamoDB writes so this Lambda and a + * concurrent confirm-uploads call cannot both succeed. If confirm-uploads + * wins (transitions to SUBMITTED), the conditional write here fails + * harmlessly. If this Lambda wins, confirm-uploads' conditional write + * fails and returns the CANCELLED status idempotently. + * + * Tests: cdk/test/handlers/cleanup-pending-uploads.test.ts + */ + +import { + DynamoDBClient, + QueryCommand, + UpdateItemCommand, + PutItemCommand, +} from '@aws-sdk/client-dynamodb'; +import { DeleteObjectsCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'; +import { ulid } from 'ulid'; +import { ATTACHMENT_OBJECT_KEY_PREFIX } from '../constructs/attachments-bucket'; +import { logger } from './shared/logger'; + +const ddb = new DynamoDBClient({}); +const s3 = new S3Client({}); + +const TASK_TABLE = process.env.TASK_TABLE_NAME!; +const EVENTS_TABLE = process.env.TASK_EVENTS_TABLE_NAME!; +const ATTACHMENTS_BUCKET = process.env.ATTACHMENTS_BUCKET_NAME!; + +/** Timeout in seconds before a PENDING_UPLOADS task is auto-cancelled. */ +const PENDING_UPLOAD_TIMEOUT_SECONDS = Number( + process.env.PENDING_UPLOAD_TIMEOUT_SECONDS ?? '1800', +); + +const TASK_RETENTION_DAYS = Number(process.env.TASK_RETENTION_DAYS ?? '90'); + +interface ExpiredTask { + readonly task_id: string; + readonly user_id: string; + readonly created_at: string; + readonly age_seconds: number; +} + +/** + * Query TaskTable's StatusIndex GSI for PENDING_UPLOADS tasks + * older than the timeout threshold. + */ +async function findExpiredPendingUploads(now: Date): Promise { + const cutoff = new Date(now.getTime() - PENDING_UPLOAD_TIMEOUT_SECONDS * 1000); + const matches: ExpiredTask[] = []; + let lastKey: Record | undefined; + + do { + const resp = await ddb.send(new QueryCommand({ + TableName: TASK_TABLE, + IndexName: 'StatusIndex', + KeyConditionExpression: '#s = :status AND created_at < :cutoff', + ExpressionAttributeNames: { '#s': 'status' }, + ExpressionAttributeValues: { + ':status': { S: 'PENDING_UPLOADS' }, + ':cutoff': { S: cutoff.toISOString() }, + }, + ExclusiveStartKey: lastKey as Record | undefined, + })); + + for (const item of resp.Items ?? []) { + const taskId = item.task_id?.S; + const userId = item.user_id?.S; + const createdAt = item.created_at?.S; + if (!taskId || !userId || !createdAt) continue; + + const createdMs = Date.parse(createdAt); + const ageSec = Math.floor((now.getTime() - createdMs) / 1000); + + matches.push({ + task_id: taskId, + user_id: userId, + created_at: createdAt, + age_seconds: ageSec, + }); + } + + lastKey = resp.LastEvaluatedKey; + } while (lastKey); + + return matches; +} + +/** + * Transition a PENDING_UPLOADS task to CANCELLED with a conditional write. + * Returns true if the transition succeeded, false if another caller + * already transitioned (confirm-uploads won the race). + */ +async function cancelExpiredTask(task: ExpiredTask): Promise { + const now = new Date().toISOString(); + const errorMessage = + `Upload window expired (${Math.floor(task.age_seconds / 60)} minutes). ` + + 'Please re-submit the task.'; + + try { + await ddb.send(new UpdateItemCommand({ + TableName: TASK_TABLE, + Key: { task_id: { S: task.task_id } }, + UpdateExpression: + 'SET #s = :cancelled, updated_at = :now, completed_at = :now, ' + + 'error_message = :err, status_created_at = :sca', + ConditionExpression: '#s = :expected', + ExpressionAttributeNames: { '#s': 'status' }, + ExpressionAttributeValues: { + ':cancelled': { S: 'CANCELLED' }, + ':expected': { S: 'PENDING_UPLOADS' }, + ':now': { S: now }, + ':err': { S: errorMessage }, + ':sca': { S: `CANCELLED#${now}` }, + }, + })); + } catch (err: any) { + if (err.name === 'ConditionalCheckFailedException') { + logger.info('Task already transitioned during pending-upload cleanup (race with confirm-uploads)', { + task_id: task.task_id, + }); + return false; + } + throw err; + } + + // Write pending_upload_expired event (best-effort) + const ttl = Math.floor(Date.now() / 1000) + TASK_RETENTION_DAYS * 86400; + try { + await ddb.send(new PutItemCommand({ + TableName: EVENTS_TABLE, + Item: { + task_id: { S: task.task_id }, + event_id: { S: ulid() }, + event_type: { S: 'pending_upload_expired' }, + timestamp: { S: now }, + ttl: { N: String(ttl) }, + metadata: { + M: { + age_seconds: { N: String(task.age_seconds) }, + timeout_seconds: { N: String(PENDING_UPLOAD_TIMEOUT_SECONDS) }, + }, + }, + }, + })); + } catch (eventErr) { + logger.error('Failed to write pending_upload_expired event (best-effort)', { + task_id: task.task_id, + error: String(eventErr), + }); + } + + return true; +} + +/** + * Delete all S3 objects under a task's attachment prefix. + * Uses prefix listing to catch any objects (including oversized abuse + * uploads that were never confirmed). + */ +async function cleanupTaskAttachments(task: ExpiredTask): Promise { + const prefix = `${ATTACHMENT_OBJECT_KEY_PREFIX}${task.user_id}/${task.task_id}/`; + + let continuationToken: string | undefined; + let totalDeleted = 0; + + do { + const listResp = await s3.send(new ListObjectsV2Command({ + Bucket: ATTACHMENTS_BUCKET, + Prefix: prefix, + ContinuationToken: continuationToken, + })); + + const objects = listResp.Contents; + if (!objects || objects.length === 0) break; + + const keys = objects + .map(obj => obj.Key) + .filter((key): key is string => key !== undefined); + + if (keys.length > 0) { + const deleteResp = await s3.send(new DeleteObjectsCommand({ + Bucket: ATTACHMENTS_BUCKET, + Delete: { Objects: keys.map(Key => ({ Key })) }, + })); + + if (deleteResp.Errors && deleteResp.Errors.length > 0) { + logger.error('Partial S3 cleanup failure for expired pending-upload task', { + task_id: task.task_id, + failedKeys: deleteResp.Errors.map(e => e.Key), + }); + } + + totalDeleted += keys.length - (deleteResp.Errors?.length ?? 0); + } + + continuationToken = listResp.NextContinuationToken; + } while (continuationToken); + + if (totalDeleted > 0) { + logger.info('Cleaned up S3 objects for expired pending-upload task', { + task_id: task.task_id, + objects_deleted: totalDeleted, + }); + } +} + +/** + * EventBridge scheduled handler entry point. + */ +export async function handler(): Promise { + const now = new Date(); + + const expired = await findExpiredPendingUploads(now); + + if (expired.length === 0) { + logger.info('No expired PENDING_UPLOADS tasks found'); + return; + } + + logger.info('Found expired PENDING_UPLOADS tasks', { count: expired.length }); + + let cancelled = 0; + let raced = 0; + let errored = 0; + + for (const task of expired) { + try { + const transitioned = await cancelExpiredTask(task); + if (transitioned) { + cancelled++; + await cleanupTaskAttachments(task); + } else { + raced++; + } + } catch (err) { + errored++; + logger.error('Failed to process expired PENDING_UPLOADS task — continuing with remaining tasks', { + task_id: task.task_id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + logger.info('Pending-upload cleanup complete', { + expired: expired.length, + cancelled, + raced, + errored, + }); +} diff --git a/cdk/src/handlers/confirm-uploads.ts b/cdk/src/handlers/confirm-uploads.ts new file mode 100644 index 00000000..38fe27c0 --- /dev/null +++ b/cdk/src/handlers/confirm-uploads.ts @@ -0,0 +1,752 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// POST /v1/tasks/{task_id}/confirm-uploads — confirms presigned uploads, screens +// attachments, and transitions the task from PENDING_UPLOADS to SUBMITTED. +// Tests: cdk/test/handlers/confirm-uploads.test.ts + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; +import { DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { DynamoDBDocumentClient, GetCommand, PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; +import type { APIGatewayProxyEvent, APIGatewayProxyResult, Context } from 'aws-lambda'; +import { ulid } from 'ulid'; +import { ATTACHMENT_OBJECT_KEY_PREFIX } from '../constructs/attachments-bucket'; +import { TaskStatus } from '../constructs/task-status'; +import { screenImage, screenTextFile, AttachmentScreeningError, type ScreeningConfig } from './shared/attachment-screening'; +import { extractUserId } from './shared/gateway'; +import { estimateImageTokensFromBuffer } from './shared/image-tokens'; +import { logger } from './shared/logger'; +import { ErrorCode, errorResponse, successResponse } from './shared/response'; +import { type AttachmentRecord, createAttachmentRecord, type TaskRecord, toTaskDetail } from './shared/types'; +import { computeTtlEpoch } from './shared/validation'; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const s3Client = new S3Client({}); +const lambdaClient = process.env.ORCHESTRATOR_FUNCTION_ARN ? new LambdaClient({}) : undefined; + +const TABLE_NAME = process.env.TASK_TABLE_NAME!; +const EVENTS_TABLE_NAME = process.env.TASK_EVENTS_TABLE_NAME!; +const ATTACHMENTS_BUCKET = process.env.ATTACHMENTS_BUCKET_NAME!; +const CONCURRENCY_TABLE_NAME = process.env.USER_CONCURRENCY_TABLE_NAME!; +const MAX_CONCURRENT = Number(process.env.MAX_CONCURRENT_TASKS_PER_USER ?? '3'); +const TASK_RETENTION_DAYS = Number(process.env.TASK_RETENTION_DAYS ?? '90'); + +if (!TABLE_NAME || !EVENTS_TABLE_NAME || !ATTACHMENTS_BUCKET || !CONCURRENCY_TABLE_NAME) { + throw new Error( + 'confirm-uploads handler requires TASK_TABLE_NAME, TASK_EVENTS_TABLE_NAME, ATTACHMENTS_BUCKET_NAME, and USER_CONCURRENCY_TABLE_NAME env vars', + ); +} + +const SCREENING_CONCURRENCY = 3; +const HEAD_OBJECT_RETRIES = 3; +const HEAD_OBJECT_RETRY_DELAY_MS = 1000; +/** Safety margin before Lambda timeout: abort screening loop at this threshold. */ +const DEADLINE_MARGIN_MS = 15_000; + +/** Transient metadata from HeadObject, keyed by attachment_id. */ +interface S3ObjectMeta { + readonly s3Key: string; + readonly versionId?: string; + readonly sizeBytes: number; +} + +/** + * POST /v1/tasks/{task_id}/confirm-uploads + * + * Flow: + * 1. Auth — verify caller owns the task. + * 2. Short-circuit — if task is not PENDING_UPLOADS, return current state (idempotent). + * 3. HeadObject per attachment — verify uploads exist in S3 (with retry for eventual consistency). + * 4. Screen each attachment in parallel (bounded concurrency of 3). + * 5. On all pass: conditional DDB write (status → SUBMITTED), invoke orchestrator. + * 6. On any block: conditional DDB write (status → FAILED), cleanup S3. + */ +export async function handler(event: APIGatewayProxyEvent, context: Context): Promise { + const requestId = ulid(); + + try { + // 1. Auth + const callerUserId = extractUserId(event); + if (!callerUserId) { + return errorResponse(401, ErrorCode.UNAUTHORIZED, 'Missing or invalid authentication.', requestId); + } + + // 2. Path parameter + const taskId = event.pathParameters?.task_id; + if (!taskId) { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, 'Missing task_id path parameter.', requestId); + } + + // 3. Read task record + const taskResult = await ddb.send(new GetCommand({ + TableName: TABLE_NAME, + Key: { task_id: taskId }, + })); + + if (!taskResult.Item) { + return errorResponse(404, ErrorCode.TASK_NOT_FOUND, 'Task not found.', requestId); + } + + const task = taskResult.Item as TaskRecord; + + // Ownership check + if (task.user_id !== callerUserId) { + return errorResponse(404, ErrorCode.TASK_NOT_FOUND, 'Task not found.', requestId); + } + + // 4. Short-circuit: if not PENDING_UPLOADS, return current state (idempotent) + if (task.status !== TaskStatus.PENDING_UPLOADS) { + if (task.status === TaskStatus.SUBMITTED || task.status === TaskStatus.HYDRATING || + task.status === TaskStatus.RUNNING) { + return successResponse(200, toTaskDetail(task), requestId); + } + return errorResponse(409, ErrorCode.UPLOADS_NOT_PENDING, + `Task is in status '${task.status}' and cannot accept upload confirmations.`, requestId); + } + + // 5. Validate attachments exist + const attachments = task.attachments; + if (!attachments || attachments.length === 0) { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, + 'Task has no attachments to confirm.', requestId); + } + + const pendingAttachments = attachments.filter(a => a.screening.status === 'pending'); + if (pendingAttachments.length === 0) { + // All already screened — transition (handles retry case) + return await transitionToSubmitted(task, attachments, requestId); + } + + // 6. HeadObject per pending attachment — verify uploads exist + const s3Meta = new Map(); + for (const att of pendingAttachments) { + const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${task.user_id}/${taskId}/${att.attachment_id}/${att.filename}`; + const headResult = await headObjectWithRetry(s3Key); + if (!headResult.exists) { + return errorResponse(400, ErrorCode.ATTACHMENT_UPLOAD_MISSING, + `Upload for '${att.filename}' not found. Ensure the upload completed successfully before calling confirm-uploads.`, + requestId); + } + s3Meta.set(att.attachment_id, { + s3Key, + versionId: headResult.versionId, + sizeBytes: headResult.contentLength!, + }); + } + + // 7. Screen attachments in parallel with bounded concurrency + const screeningConfig = await buildScreeningConfig(); + if (!screeningConfig) { + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is not configured. Please contact your administrator.', requestId); + } + + const screenedAttachments: AttachmentRecord[] = []; + + // Internal deadline timer (design §ConfirmUploadsFunction): abort screening + // before the Lambda times out so we can return a graceful 503 + Retry-After + // instead of an opaque timeout error. On retry, already-screened attachments + // (status === 'passed' in DDB) are skipped, so retries make forward progress. + const deadlineMs = context.getRemainingTimeInMillis() - DEADLINE_MARGIN_MS; + + // Process in batches of SCREENING_CONCURRENCY + for (let i = 0; i < pendingAttachments.length; i += SCREENING_CONCURRENCY) { + // Deadline check before starting a new batch + if (context.getRemainingTimeInMillis() <= DEADLINE_MARGIN_MS) { + const screened = screenedAttachments.length; + const remaining = pendingAttachments.length - screened; + logger.warn('Confirm-uploads deadline reached — aborting remaining screening', { + task_id: taskId, + screened, + remaining, + deadline_ms: deadlineMs, + request_id: requestId, + metric_type: 'confirm_uploads_deadline_exceeded', + }); + + // Persist any already-screened results so retries skip them + if (screenedAttachments.length > 0) { + await persistPartialScreeningState(task, taskId, screenedAttachments, requestId); + } + + return { + statusCode: 503, + headers: { 'Retry-After': '30', 'Content-Type': 'application/json' }, + body: JSON.stringify({ + error: { + code: ErrorCode.SCREENING_DEADLINE_EXCEEDED, + message: + 'Attachment screening did not complete within the time limit. ' + + 'Reduce the number or size of attachments and try again, or retry after 30 seconds ' + + '(already-screened attachments will be skipped on retry).', + request_id: requestId, + }, + }), + }; + } + + const batch = pendingAttachments.slice(i, i + SCREENING_CONCURRENCY); + const results = await Promise.allSettled( + batch.map(att => screenSingleAttachment(att, task, screeningConfig, taskId, s3Meta.get(att.attachment_id)!)), + ); + + for (let j = 0; j < results.length; j++) { + const result = results[j]; + const att = batch[j]; + + if (result.status === 'rejected') { + const err = result.reason; + if (err instanceof AttachmentScreeningError) { + logger.warn('Attachment screening rejected content during confirm-uploads', { + attachment_id: att.attachment_id, + filename: att.filename, + error: err.message, + task_id: taskId, + request_id: requestId, + }); + // Fail the entire task + await failTaskOnScreening(task, taskId, att.filename, err.message, requestId); + await cleanupAllAttachments(task, taskId); + return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, + `Attachment '${att.filename}' was rejected: ${err.message}`, requestId); + } + + // Non-screening error — fail-closed + logger.error('Attachment screening failed during confirm-uploads (fail-closed)', { + attachment_id: att.attachment_id, + filename: att.filename, + error: err instanceof Error ? err.message : String(err), + task_id: taskId, + request_id: requestId, + metric_type: 'confirm_uploads_screening_failure', + }); + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is temporarily unavailable. Please try again later.', requestId); + } + + // Screening passed — record the result + screenedAttachments.push(result.value); + } + } + + // If any attachment was blocked via the result (not error), handle that + const blockedAtt = screenedAttachments.find(a => a.screening.status === 'blocked'); + if (blockedAtt) { + const categories = blockedAtt.screening.status === 'blocked' + ? blockedAtt.screening.categories.join(', ') + : 'content_policy_violation'; + await failTaskOnScreening(task, taskId, blockedAtt.filename, categories, requestId); + await cleanupAllAttachments(task, taskId); + return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, + `Attachment '${blockedAtt.filename}' was blocked by content policy (${categories}).`, requestId); + } + + // 8. All passed — merge screened records with any already-screened ones + const finalAttachments = attachments.map(existing => { + const screened = screenedAttachments.find(s => s.attachment_id === existing.attachment_id); + return screened ?? existing; + }); + + // 9. Transition to SUBMITTED + return await transitionToSubmitted(task, finalAttachments, requestId); + } catch (err) { + logger.error('Unhandled error in confirm-uploads', { + error: err instanceof Error ? err.message : String(err), + stack: err instanceof Error ? err.stack : undefined, + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, + 'An unexpected error occurred. Please try again.', requestId); + } +} + +// --------------------------------------------------------------------------- +// Screen a single attachment: download from S3, screen, re-upload cleaned version +// --------------------------------------------------------------------------- + +async function screenSingleAttachment( + att: AttachmentRecord, + task: TaskRecord, + screeningConfig: ScreeningConfig, + taskId: string, + meta: S3ObjectMeta, +): Promise { + const { s3Key, versionId, sizeBytes } = meta; + + // Download the object content for screening + const getResult = await s3Client.send(new GetObjectCommand({ + Bucket: ATTACHMENTS_BUCKET, + Key: s3Key, + ...(versionId && { VersionId: versionId }), + })); + + if (!getResult.Body) { + throw new AttachmentScreeningError( + `Upload for '${att.filename}' could not be read from storage. Please re-upload the file.`, + ); + } + const content = Buffer.from(await getResult.Body.transformToByteArray()); + + // Screen based on type + const isImage = att.type === 'image'; + const screenResult = isImage + ? await screenImage(content, att.content_type, att.filename, screeningConfig) + : await screenTextFile(content, att.content_type, att.filename, screeningConfig); + + if (screenResult.screening.status === 'blocked') { + return createAttachmentRecord({ + attachment_id: att.attachment_id, + type: att.type, + content_type: att.content_type, + filename: att.filename, + s3_key: s3Key, + s3_version_id: versionId ?? 'unversioned', + size_bytes: sizeBytes, + screening: { + status: 'blocked', + screened_at: new Date().toISOString(), + categories: screenResult.screening.categories, + }, + checksum_sha256: screenResult.checksum, + }); + } + + // Screening passed — re-upload cleaned content (EXIF-stripped for images) + const putResult = await s3Client.send(new PutObjectCommand({ + Bucket: ATTACHMENTS_BUCKET, + Key: s3Key, + Body: screenResult.content, + ContentType: att.content_type, + })); + + // Estimate token cost for images + let tokenEstimate: number | undefined; + if (isImage) { + try { + const sharp = (await import('sharp')).default; + const metadata = await sharp(screenResult.content).metadata(); + if (metadata.width && metadata.height) { + tokenEstimate = estimateImageTokens(metadata.width, metadata.height); + } + } catch (err) { + logger.warn('Failed to estimate image tokens in confirm-uploads (non-fatal)', { + error: err instanceof Error ? err.message : String(err), + attachment_id: att.attachment_id, + }); + } + } + + return createAttachmentRecord({ + attachment_id: att.attachment_id, + type: att.type, + content_type: att.content_type, + filename: att.filename, + s3_key: s3Key, + s3_version_id: putResult.VersionId ?? 'unversioned', + size_bytes: screenResult.content.length, + screening: { status: 'passed', screened_at: new Date().toISOString() }, + checksum_sha256: screenResult.checksum, + ...(tokenEstimate !== undefined && { token_estimate: tokenEstimate }), + }); +} + +// --------------------------------------------------------------------------- +// HeadObject with retry (S3 eventual consistency) +// --------------------------------------------------------------------------- + +async function headObjectWithRetry(s3Key: string): Promise<{ + exists: boolean; + versionId?: string; + contentLength?: number; +}> { + for (let attempt = 0; attempt <= HEAD_OBJECT_RETRIES; attempt++) { + try { + const result = await s3Client.send(new HeadObjectCommand({ + Bucket: ATTACHMENTS_BUCKET, + Key: s3Key, + })); + return { + exists: true, + versionId: result.VersionId, + contentLength: result.ContentLength, + }; + } catch (err: any) { + const statusCode = err?.$metadata?.httpStatusCode; + if (statusCode === 404 && attempt < HEAD_OBJECT_RETRIES) { + await new Promise(resolve => setTimeout(resolve, HEAD_OBJECT_RETRY_DELAY_MS)); + continue; + } + if (statusCode === 404) { + return { exists: false }; + } + throw err; + } + } + return { exists: false }; +} + +// --------------------------------------------------------------------------- +// Transition to SUBMITTED (conditional write + orchestrator invoke) +// --------------------------------------------------------------------------- + +async function transitionToSubmitted( + task: TaskRecord, + finalAttachments: AttachmentRecord[], + requestId: string, +): Promise { + const now = new Date().toISOString(); + const taskId = task.task_id; + + // Admission control — check concurrency before transitioning + const admitted = await checkConcurrency(task.user_id); + if (!admitted) { + return errorResponse(429, ErrorCode.RATE_LIMIT_EXCEEDED, + 'User concurrency limit reached. Wait for a running task to finish or cancel one, then retry.', requestId); + } + + // Conditional DynamoDB write: status PENDING_UPLOADS → SUBMITTED + try { + await ddb.send(new UpdateCommand({ + TableName: TABLE_NAME, + Key: { task_id: taskId }, + UpdateExpression: 'SET #s = :submitted, #sca = :status_created_at, attachments = :atts, updated_at = :now', + ConditionExpression: '#s = :pending_uploads', + ExpressionAttributeNames: { + '#s': 'status', + '#sca': 'status_created_at', + }, + ExpressionAttributeValues: { + ':submitted': TaskStatus.SUBMITTED, + ':pending_uploads': TaskStatus.PENDING_UPLOADS, + ':status_created_at': `${TaskStatus.SUBMITTED}#${now}`, + ':atts': finalAttachments, + ':now': now, + }, + })); + } catch (err: any) { + if (err.name === 'ConditionalCheckFailedException') { + // Another caller already transitioned (e.g. cleanup Lambda cancelled + // the task). Roll back the concurrency counter we just incremented. + await decrementConcurrency(task.user_id); + // Return current state (idempotent) + const current = await ddb.send(new GetCommand({ + TableName: TABLE_NAME, + Key: { task_id: taskId }, + })); + if (current.Item) { + return successResponse(200, toTaskDetail(current.Item as TaskRecord), requestId); + } + return errorResponse(404, ErrorCode.TASK_NOT_FOUND, 'Task not found.', requestId); + } + throw err; + } + + // Write uploads_confirmed event (best-effort) + try { + await ddb.send(new PutCommand({ + TableName: EVENTS_TABLE_NAME, + Item: { + task_id: taskId, + event_id: ulid(), + event_type: 'uploads_confirmed', + timestamp: now, + ttl: computeTtlEpoch(TASK_RETENTION_DAYS), + metadata: { + attachment_count: finalAttachments.length, + total_size_bytes: finalAttachments.reduce((sum, a) => sum + (a.size_bytes ?? 0), 0), + }, + }, + })); + } catch (eventErr) { + logger.error('Failed to write uploads_confirmed event', { + task_id: taskId, + error: String(eventErr), + request_id: requestId, + }); + } + + // Invoke orchestrator (fire-and-forget) + if (lambdaClient && process.env.ORCHESTRATOR_FUNCTION_ARN) { + try { + await lambdaClient.send(new InvokeCommand({ + FunctionName: process.env.ORCHESTRATOR_FUNCTION_ARN, + InvocationType: 'Event', + Payload: new TextEncoder().encode(JSON.stringify({ task_id: taskId })), + })); + logger.info('Orchestrator invoked after confirm-uploads', { + task_id: taskId, + request_id: requestId, + }); + } catch (orchErr) { + logger.error('Failed to invoke orchestrator after confirm-uploads', { + error: String(orchErr), + task_id: taskId, + request_id: requestId, + }); + } + } + + // Return updated task + const updatedTask: TaskRecord = { + ...task, + status: TaskStatus.SUBMITTED, + attachments: finalAttachments, + updated_at: now, + }; + return successResponse(200, toTaskDetail(updatedTask), requestId); +} + +// --------------------------------------------------------------------------- +// Fail task on screening failure (conditional write) +// --------------------------------------------------------------------------- + +async function failTaskOnScreening( + task: TaskRecord, + taskId: string, + filename: string, + reason: string, + requestId: string, +): Promise { + const now = new Date().toISOString(); + try { + await ddb.send(new UpdateCommand({ + TableName: TABLE_NAME, + Key: { task_id: taskId }, + UpdateExpression: 'SET #s = :failed, #sca = :status_created_at, error_message = :err, updated_at = :now', + ConditionExpression: '#s = :pending_uploads', + ExpressionAttributeNames: { + '#s': 'status', + '#sca': 'status_created_at', + }, + ExpressionAttributeValues: { + ':failed': TaskStatus.FAILED, + ':pending_uploads': TaskStatus.PENDING_UPLOADS, + ':status_created_at': `${TaskStatus.FAILED}#${now}`, + ':err': `Attachment '${filename}' blocked: ${reason}`, + ':now': now, + }, + })); + } catch (err: any) { + if (err.name === 'ConditionalCheckFailedException') { + // Another caller already transitioned — skip + logger.info('Task already transitioned during screening failure', { + task_id: taskId, + request_id: requestId, + }); + return; + } + throw err; + } + + // Write event (best-effort) + try { + await ddb.send(new PutCommand({ + TableName: EVENTS_TABLE_NAME, + Item: { + task_id: taskId, + event_id: ulid(), + event_type: 'attachment_blocked', + timestamp: now, + ttl: computeTtlEpoch(TASK_RETENTION_DAYS), + metadata: { filename, reason }, + }, + })); + } catch (eventErr) { + logger.error('Failed to write attachment_blocked event (best-effort)', { + task_id: taskId, + filename, + error: String(eventErr), + request_id: requestId, + }); + } +} + +// --------------------------------------------------------------------------- +// Cleanup all S3 attachments for a task +// --------------------------------------------------------------------------- + +async function cleanupAllAttachments(task: TaskRecord, taskId: string): Promise { + if (!task.attachments || task.attachments.length === 0) return; + + const keys = task.attachments.map(att => + `${ATTACHMENT_OBJECT_KEY_PREFIX}${task.user_id}/${taskId}/${att.attachment_id}/${att.filename}`, + ); + + try { + const result = await s3Client.send(new DeleteObjectsCommand({ + Bucket: ATTACHMENTS_BUCKET, + Delete: { Objects: keys.map(Key => ({ Key })) }, + })); + if (result.Errors && result.Errors.length > 0) { + logger.error('Partial cleanup failure in confirm-uploads', { + task_id: taskId, + failedKeys: result.Errors.map(e => e.Key), + }); + } + } catch (err) { + logger.error('S3 cleanup failed in confirm-uploads — 90-day lifecycle is safety net', { + task_id: taskId, + keys, + error: String(err), + }); + } +} + +// --------------------------------------------------------------------------- +// Persist partial screening state (for deadline-exceeded retries) +// --------------------------------------------------------------------------- + +/** + * Write already-screened attachment records back to DDB so retries skip them. + * Uses a conditional write to only update if the task is still in PENDING_UPLOADS. + * If this write fails (race with cleanup), the already-screened work is lost but + * the task was already being cancelled, so the client will see the correct state. + */ +async function persistPartialScreeningState( + task: TaskRecord, + taskId: string, + screenedAttachments: AttachmentRecord[], + requestId: string, +): Promise { + // Merge screened results with existing attachment records + const mergedAttachments = (task.attachments ?? []).map(existing => { + const screened = screenedAttachments.find(s => s.attachment_id === existing.attachment_id); + return screened ?? existing; + }); + + try { + await ddb.send(new UpdateCommand({ + TableName: TABLE_NAME, + Key: { task_id: taskId }, + UpdateExpression: 'SET attachments = :atts, updated_at = :now', + ConditionExpression: '#s = :pending_uploads', + ExpressionAttributeNames: { '#s': 'status' }, + ExpressionAttributeValues: { + ':atts': mergedAttachments, + ':pending_uploads': TaskStatus.PENDING_UPLOADS, + ':now': new Date().toISOString(), + }, + })); + logger.info('Persisted partial screening state for deadline-exceeded retry', { + task_id: taskId, + screened_count: screenedAttachments.length, + request_id: requestId, + }); + } catch (err: any) { + if (err.name === 'ConditionalCheckFailedException') { + logger.info('Task already transitioned during partial screening persist — skipping', { + task_id: taskId, + request_id: requestId, + }); + return; + } + logger.error('Failed to persist partial screening state (best-effort)', { + task_id: taskId, + error: err instanceof Error ? err.message : String(err), + request_id: requestId, + }); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +let _bedrockClient: import('@aws-sdk/client-bedrock-runtime').BedrockRuntimeClient | undefined; + +async function buildScreeningConfig(): Promise { + if (!process.env.GUARDRAIL_ID || !process.env.GUARDRAIL_VERSION) return undefined; + if (!_bedrockClient) { + const { BedrockRuntimeClient } = await import('@aws-sdk/client-bedrock-runtime'); + _bedrockClient = new BedrockRuntimeClient({}); + } + return { + guardrailId: process.env.GUARDRAIL_ID, + guardrailVersion: process.env.GUARDRAIL_VERSION, + bedrockClient: _bedrockClient, + }; +} + +async function checkConcurrency(userId: string): Promise { + try { + await ddb.send(new UpdateCommand({ + TableName: CONCURRENCY_TABLE_NAME, + Key: { user_id: userId }, + UpdateExpression: 'SET active_count = if_not_exists(active_count, :zero) + :one, updated_at = :now', + ConditionExpression: 'attribute_not_exists(active_count) OR active_count < :max', + ExpressionAttributeValues: { + ':zero': 0, + ':one': 1, + ':max': MAX_CONCURRENT, + ':now': new Date().toISOString(), + }, + })); + return true; + } catch (err: any) { + if (err.name === 'ConditionalCheckFailedException') { + return false; + } + throw err; + } +} + +async function decrementConcurrency(userId: string): Promise { + try { + await ddb.send(new UpdateCommand({ + TableName: CONCURRENCY_TABLE_NAME, + Key: { user_id: userId }, + UpdateExpression: 'SET active_count = active_count - :one, updated_at = :now', + ConditionExpression: 'attribute_exists(active_count) AND active_count > :zero', + ExpressionAttributeValues: { + ':one': 1, + ':zero': 0, + ':now': new Date().toISOString(), + }, + })); + } catch (err: any) { + if (err.name === 'ConditionalCheckFailedException') { + // Counter already at 0 or doesn't exist — nothing to roll back + return; + } + logger.error('Failed to decrement concurrency counter (leak possible)', { + user_id: userId, + error: err instanceof Error ? err.message : String(err), + }); + } +} + +const MAX_IMAGE_SIDE = 1568; +const MAX_IMAGE_TOKENS = 1568; +const TOKEN_SAFETY_MARGIN = 1.2; +const TILE_SIZE = 28; + +function estimateImageTokens(width: number, height: number): number { + let w = width; + let h = height; + const maxSide = Math.max(w, h); + if (maxSide > MAX_IMAGE_SIDE) { + const scale = MAX_IMAGE_SIDE / maxSide; + w = Math.round(w * scale); + h = Math.round(h * scale); + } + w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; + h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; + const rawTokens = Math.ceil((w * h) / 750); + return Math.min(Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN), MAX_IMAGE_TOKENS); +} diff --git a/cdk/src/handlers/linear-webhook-processor.ts b/cdk/src/handlers/linear-webhook-processor.ts index 3aecc4b0..14592ff2 100644 --- a/cdk/src/handlers/linear-webhook-processor.ts +++ b/cdk/src/handlers/linear-webhook-processor.ts @@ -23,6 +23,7 @@ import { DynamoDBDocumentClient, GetCommand } from '@aws-sdk/lib-dynamodb'; import { createTaskCore } from './shared/create-task-core'; import { reportIssueFailure } from './shared/linear-feedback'; import { logger } from './shared/logger'; +import type { Attachment } from './shared/types'; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); @@ -223,11 +224,16 @@ export async function handler(event: ProcessorEvent): Promise { channelMetadata.linear_team_id = issue.teamId; } + // Extract embedded image URLs from the issue description markdown. + // These become URL attachments that are fetched and screened during context hydration. + const attachments = extractImageUrlAttachments(issue.description); + const requestId = crypto.randomUUID(); const result = await createTaskCore( { repo, task_description: taskDescription, + ...(attachments.length > 0 && { attachments }), }, { userId: platformUserId, @@ -343,6 +349,35 @@ function buildTaskDescription(issue: LinearIssueEvent['data']): string { return parts.join('\n') || 'Linear issue'; } +/** + * Extract image URL attachments from Linear issue description markdown. + * + * Scans for standard markdown image references: `![alt](url)`. + * Only HTTPS URLs are included (security: no HTTP, no data: URIs). + * Capped at 10 images per issue to stay within attachment limits. + */ +function extractImageUrlAttachments(description: string | undefined): Attachment[] { + if (!description) return []; + + const imagePattern = /!\[[^\]]*\]\((https:\/\/[^)]+)\)/g; + const attachments: Attachment[] = []; + let match: RegExpExecArray | null; + + while ((match = imagePattern.exec(description)) !== null) { + if (attachments.length >= 10) break; + const url = match[1]; + attachments.push({ type: 'url', url }); + } + + if (attachments.length > 0) { + logger.info('Extracted image URL attachments from Linear issue description', { + count: attachments.length, + }); + } + + return attachments; +} + async function lookupPlatformUser(workspaceId: string, userId: string): Promise { const key = `${workspaceId}#${userId}`; const result = await ddb.send(new GetCommand({ diff --git a/cdk/src/handlers/shared/attachment-screening.ts b/cdk/src/handlers/shared/attachment-screening.ts new file mode 100644 index 00000000..6fda0ad2 --- /dev/null +++ b/cdk/src/handlers/shared/attachment-screening.ts @@ -0,0 +1,388 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { createHash } from 'crypto'; +import { ApplyGuardrailCommand, type BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import sharp from 'sharp'; +import { logger } from './logger'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const MAX_RETRIES = 3; +const BASE_DELAY_MS = 200; +const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503]); + +export const MAX_ATTACHMENT_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface ScreeningConfig { + readonly guardrailId: string; + readonly guardrailVersion: string; + readonly bedrockClient: BedrockRuntimeClient; +} + +export type ScreeningOutcome = + | { readonly status: 'passed' } + | { readonly status: 'blocked'; readonly categories: [string, ...string[]] }; + +export interface ScreenedAttachment { + readonly content: Buffer; + readonly contentType: string; + readonly checksum: string; + readonly screening: ScreeningOutcome; +} + +// --------------------------------------------------------------------------- +// Retry utility +// --------------------------------------------------------------------------- + +/** + * Retry with exponential backoff for transient Bedrock errors. + * Non-retryable errors (4xx except 429, validation errors) propagate immediately. + */ +async function retryWithBackoff( + fn: () => Promise, + opts: { maxRetries: number; baseDelayMs: number; context: string }, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt <= opts.maxRetries; attempt++) { + try { + return await fn(); + } catch (err: any) { + lastError = err; + const statusCode = err?.$metadata?.httpStatusCode ?? err?.statusCode; + const isRetryable = RETRYABLE_STATUS_CODES.has(statusCode); + if (!isRetryable || attempt === opts.maxRetries) { + if (isRetryable && attempt === opts.maxRetries) { + logger.error('All retries exhausted for Bedrock screening', { + context: opts.context, + total_attempts: attempt + 1, + status_code: statusCode, + error: err instanceof Error ? err.message : String(err), + }); + } + throw err; + } + const delay = opts.baseDelayMs * Math.pow(2, attempt); + logger.warn('Retrying after transient error', { + context: opts.context, + attempt: attempt + 1, + max_retries: opts.maxRetries, + status_code: statusCode, + delay_ms: delay, + error: err instanceof Error ? err.message : String(err), + }); + await sleep(delay); + } + } + throw lastError; +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +// --------------------------------------------------------------------------- +// Image screening +// --------------------------------------------------------------------------- + +/** + * Screen an image attachment through the Bedrock Guardrail. + * + * Flow: validate → convert GIF/WebP to PNG (Bedrock only accepts png|jpeg) → + * screen → strip EXIF / re-encode on pass. + * + * @returns ScreenedAttachment with cleaned content (EXIF-stripped, re-encoded) and checksum. + * @throws Error on sharp failure or guardrail unavailability (fail-closed). + */ +export async function screenImage( + content: Buffer, + contentType: string, + filename: string, + config: ScreeningConfig, +): Promise { + // Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) + let screeningBuffer: Buffer; + let screeningFormat: 'png' | 'jpeg'; + + if (contentType === 'image/jpeg') { + screeningBuffer = content; + screeningFormat = 'jpeg'; + } else if (contentType === 'image/gif' || contentType === 'image/webp') { + // GIF/WebP → PNG. For animated GIFs, extract first frame only. + try { + screeningBuffer = await sharp(content, { animated: false }).png().toBuffer(); + } catch (convErr) { + throw new AttachmentScreeningError( + `Image "${filename}" could not be converted from ${contentType} for screening. ` + + 'The file may be corrupt. Please re-export or use a PNG/JPEG format.', + { cause: convErr }, + ); + } + screeningFormat = 'png'; + + // Post-conversion size check: PNG expansion of compressed GIF/WebP can exceed limit. + if (screeningBuffer.length > MAX_ATTACHMENT_SIZE_BYTES) { + throw new AttachmentScreeningError( + `Image "${filename}" is ${contentType} and its PNG conversion for screening ` + + `exceeds the ${MAX_ATTACHMENT_SIZE_BYTES / (1024 * 1024)} MB limit ` + + `(${(screeningBuffer.length / (1024 * 1024)).toFixed(1)} MB after conversion). ` + + 'Please convert to JPEG or reduce image dimensions before uploading.', + ); + } + } else { + // PNG: use as-is + screeningBuffer = content; + screeningFormat = 'png'; + } + + // Screen through Bedrock Guardrail with retry + const result = await retryWithBackoff( + () => config.bedrockClient.send(new ApplyGuardrailCommand({ + guardrailIdentifier: config.guardrailId, + guardrailVersion: config.guardrailVersion, + source: 'INPUT', + content: [{ + image: { + format: screeningFormat, + source: { bytes: screeningBuffer }, + }, + }], + })), + { maxRetries: MAX_RETRIES, baseDelayMs: BASE_DELAY_MS, context: `image_screening:${filename}` }, + ); + + if (result.action === 'GUARDRAIL_INTERVENED') { + const categories = extractBlockedCategories(result.assessments); + return { + content: screeningBuffer, + contentType, + checksum: computeSha256(screeningBuffer), + screening: { status: 'blocked', categories }, + }; + } + + // Screening passed — strip EXIF and re-encode. + // Note: NOT calling .withMetadata() — sharp strips all metadata by default + // when withMetadata is omitted. Calling .withMetadata({}) would opt INTO + // metadata preservation, which is the opposite of what we want. + let cleanedContent: Buffer; + try { + cleanedContent = await sharp(content) + .rotate() // Apply EXIF orientation before stripping + .toBuffer(); + } catch (sharpErr) { + throw new AttachmentScreeningError( + `Image "${filename}" could not be processed for security sanitization. ` + + 'Please re-export the image in a standard format and try again.', + { cause: sharpErr }, + ); + } + + const checksum = computeSha256(cleanedContent); + return { + content: cleanedContent, + contentType, + checksum, + screening: { status: 'passed' }, + }; +} + +// --------------------------------------------------------------------------- +// Text/file screening +// --------------------------------------------------------------------------- + +/** + * Screen a text-based file attachment through the Bedrock Guardrail. + * Supports plain text, CSV, Markdown, JSON, and log files directly. + * PDFs have their text extracted first. + * + * @returns ScreenedAttachment with the original content (text files are not re-encoded). + * @throws Error on guardrail unavailability (fail-closed). + */ +export async function screenTextFile( + content: Buffer, + contentType: string, + filename: string, + config: ScreeningConfig, +): Promise { + let textToScreen: string; + + if (contentType === 'application/pdf') { + textToScreen = await extractPdfText(content, filename); + if (textToScreen.trim().length === 0) { + throw new AttachmentScreeningError( + `PDF "${filename}" contains no extractable text (it may be image-only or encrypted). ` + + 'Please use an OCR tool to add a text layer, or convert to an image attachment.', + ); + } + } else { + textToScreen = content.toString('utf-8'); + } + + // Screen through Bedrock Guardrail with retry + const result = await retryWithBackoff( + () => config.bedrockClient.send(new ApplyGuardrailCommand({ + guardrailIdentifier: config.guardrailId, + guardrailVersion: config.guardrailVersion, + source: 'INPUT', + content: [{ text: { text: textToScreen } }], + })), + { maxRetries: MAX_RETRIES, baseDelayMs: BASE_DELAY_MS, context: `text_screening:${filename}` }, + ); + + const checksum = computeSha256(content); + + if (result.action === 'GUARDRAIL_INTERVENED') { + const categories = extractBlockedCategories(result.assessments); + return { + content, + contentType, + checksum, + screening: { status: 'blocked', categories }, + }; + } + + return { + content, + contentType, + checksum, + screening: { status: 'passed' }, + }; +} + +// --------------------------------------------------------------------------- +// PDF text extraction +// --------------------------------------------------------------------------- + +const PDF_MAX_PAGES = 50; +const PDF_MAX_TEXT_BYTES = 1024 * 1024; // 1 MB extracted text cap +const PDF_EXTRACT_TIMEOUT_MS = 15_000; + +async function extractPdfText(content: Buffer, filename: string): Promise { + // Dynamic import — pdf-parse is only used for PDF attachments. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let pdfParseFn: (data: Buffer, options?: { max?: number }) => Promise<{ text: string }>; + try { + // pdf-parse uses a default export; handle both CJS and ESM module shapes. + const mod = await import(/* webpackIgnore: true */ 'pdf-parse'); + pdfParseFn = (mod as any).default ?? mod; + } catch (importErr) { + logger.error('pdf-parse module could not be imported — PDF screening unavailable', { + error: importErr instanceof Error ? importErr.message : String(importErr), + metric_type: 'pdf_parse_import_failure', + }); + throw new AttachmentScreeningError( + `PDF processing is unavailable. Cannot screen "${filename}".`, + { cause: importErr }, + ); + } + + let timeoutId: ReturnType; + try { + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => reject(new Error('PDF extraction timed out')), PDF_EXTRACT_TIMEOUT_MS); + }); + + const result = await Promise.race([ + pdfParseFn(content, { max: PDF_MAX_PAGES }), + timeoutPromise, + ]); + + let text: string = result.text ?? ''; + if (Buffer.byteLength(text, 'utf-8') > PDF_MAX_TEXT_BYTES) { + text = text.slice(0, PDF_MAX_TEXT_BYTES); + } + return text; + } catch (err) { + throw new AttachmentScreeningError( + `PDF "${filename}" could not be processed. It may be corrupt or use unsupported features. ` + + 'Try exporting to a simpler PDF format.', + { cause: err }, + ); + } finally { + clearTimeout(timeoutId!); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function computeSha256(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + +function extractBlockedCategories( + assessments: any[] | undefined, +): [string, ...string[]] { + const categories: string[] = []; + if (assessments) { + for (const assessment of assessments) { + // Extract topic/content/word/sensitive-info policy categories + for (const policyResult of Object.values(assessment) as any[]) { + if (Array.isArray(policyResult?.topics)) { + for (const t of policyResult.topics) { + if (t.name) categories.push(t.name); + } + } + if (Array.isArray(policyResult?.filters)) { + for (const f of policyResult.filters) { + if (f.type) categories.push(f.type); + } + } + if (Array.isArray(policyResult?.managedWordLists)) { + for (const w of policyResult.managedWordLists) { + if (w.match) categories.push(`word:${w.match}`); + } + } + if (Array.isArray(policyResult?.piiEntities)) { + for (const p of policyResult.piiEntities) { + if (p.type) categories.push(`pii:${p.type}`); + } + } + } + } + } + if (categories.length === 0) { + logger.warn('Could not extract specific categories from guardrail assessment — using generic fallback', { + has_assessments: !!assessments, + assessment_count: assessments?.length ?? 0, + assessment_keys: assessments?.[0] ? Object.keys(assessments[0]) : [], + }); + categories.push('content_policy_violation'); + } + return categories as [string, ...string[]]; +} + +// --------------------------------------------------------------------------- +// Error class +// --------------------------------------------------------------------------- + +export class AttachmentScreeningError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'AttachmentScreeningError'; + } +} diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index e88e11e1..01beaf6e 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -21,16 +21,18 @@ // Idempotent replay: same user + same Idempotency-Key → 200 + TaskDetail (no duplicate write, no orchestrator re-invoke). // Tests: cdk/test/handlers/shared/create-task-core.test.ts, cdk/test/handlers/create-task.test.ts -import { createHash } from 'crypto'; import { BedrockRuntimeClient, ApplyGuardrailCommand } from '@aws-sdk/client-bedrock-runtime'; import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; import { PutObjectCommand, DeleteObjectsCommand, S3Client } from '@aws-sdk/client-s3'; import { DynamoDBDocumentClient, PutCommand, QueryCommand, GetCommand } from '@aws-sdk/lib-dynamodb'; +import { createPresignedPost } from '@aws-sdk/s3-presigned-post'; import type { APIGatewayProxyResult } from 'aws-lambda'; import { ulid } from 'ulid'; import { isDegeneratePattern, parseApprovalScope } from './approval-scope'; +import { screenImage, screenTextFile, AttachmentScreeningError, type ScreeningConfig } from './attachment-screening'; import { generateBranchName } from './gateway'; +import { estimateImageTokensFromBuffer } from './image-tokens'; import { logger } from './logger'; import { lookupRepo } from './repo-config'; import { ErrorCode, errorResponse, successResponse } from './response'; @@ -41,17 +43,19 @@ import { APPROVAL_TIMEOUT_S_MAX, APPROVAL_TIMEOUT_S_MIN, type AttachmentRecord, + type AttachmentUploadInstruction, type ChannelSource, type CreateTaskRequest, createAttachmentRecord, INITIAL_APPROVALS_MAX_ENTRIES, type InlineAttachment, isPrTaskType, + type PresignedAttachment, type TaskRecord, type TaskType, toTaskDetail, } from './types'; -import { computeTtlEpoch, DEFAULT_MAX_TURNS, hasTaskSpec, isValidIdempotencyKey, isValidRepo, isValidTaskDescriptionLength, isValidTaskType, MAX_TASK_DESCRIPTION_LENGTH, validateAttachments, validateMaxBudgetUsd, validateMaxTurns, validatePrNumber } from './validation'; +import { computeTtlEpoch, DEFAULT_MAX_TURNS, hasTaskSpec, isValidIdempotencyKey, isValidRepo, isValidTaskDescriptionLength, isValidTaskType, MAX_ATTACHMENT_SIZE_BYTES, MAX_TASK_DESCRIPTION_LENGTH, validateAttachments, validateMaxBudgetUsd, validateMaxTurns, validatePrNumber } from './validation'; import { ATTACHMENT_OBJECT_KEY_PREFIX } from '../../constructs/attachments-bucket'; import { TaskStatus } from '../../constructs/task-status'; @@ -331,16 +335,34 @@ export async function createTaskCore( // Generate task ID early so attachment S3 keys use the correct task ID const taskId = ulid(); - // 2b. Process inline attachments: screen, upload to S3, build records + // 2b. Process inline attachments: screen (with retry + EXIF strip), upload to S3, build records. + // Presigned attachments are deferred to confirm-uploads; URL attachments are resolved during hydration. const attachmentRecords: AttachmentRecord[] = []; const uploadedS3Keys: string[] = []; if (validatedAttachments.length > 0 && s3Client && ATTACHMENTS_BUCKET) { + // Build screening config — fail-closed if guardrail is not configured + if (!bedrockClient || !process.env.GUARDRAIL_ID || !process.env.GUARDRAIL_VERSION) { + const hasInline = validatedAttachments.some(a => a.delivery === 'inline'); + if (hasInline) { + logger.error('Inline attachment submitted but guardrail is not configured (fail-closed)', { + request_id: requestId, + }); + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is not configured. Please contact your administrator.', requestId); + } + } + + const screeningConfig: ScreeningConfig | undefined = bedrockClient && process.env.GUARDRAIL_ID && process.env.GUARDRAIL_VERSION + ? { bedrockClient, guardrailId: process.env.GUARDRAIL_ID, guardrailVersion: process.env.GUARDRAIL_VERSION } + : undefined; + for (const att of validatedAttachments) { if (att.delivery !== 'inline') continue; const inlineAtt = att as InlineAttachment; // Validate base64 encoding before decode if (!isValidBase64(inlineAtt.data)) { + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); return errorResponse(400, ErrorCode.ATTACHMENT_INVALID_CONTENT, `Attachment '${inlineAtt.filename}' has invalid base64 encoding.`, requestId); } @@ -348,85 +370,86 @@ export async function createTaskCore( const decoded = Buffer.from(inlineAtt.data, 'base64'); const attachmentId = ulid(); - // Screen inline attachment content via Bedrock Guardrail (fail-closed) - if (!bedrockClient) { + // Screen content via Bedrock with retry, EXIF stripping, and format conversion + let screenResult; + try { + const isImage = inlineAtt.type === 'image'; + screenResult = isImage + ? await screenImage(decoded, inlineAtt.content_type, inlineAtt.filename, screeningConfig!) + : await screenTextFile(decoded, inlineAtt.content_type, inlineAtt.filename, screeningConfig!); + } catch (screenErr) { await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - logger.error('Inline attachment submitted but guardrail is not configured (fail-closed)', { - request_id: requestId, - attachment_filename: inlineAtt.filename, - }); - return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, - 'Attachment content screening is not configured. Please contact your administrator.', requestId); - } - { - try { - const isImage = inlineAtt.type === 'image'; - const guardrailContent = isImage - ? [{ image: { format: mimeToGuardrailFormat(inlineAtt.content_type), source: { bytes: decoded } } }] - : [{ text: { text: decoded.toString('utf-8') } }]; - - const screenResult = await bedrockClient.send(new ApplyGuardrailCommand({ - guardrailIdentifier: process.env.GUARDRAIL_ID!, - guardrailVersion: process.env.GUARDRAIL_VERSION!, - source: 'INPUT', - content: guardrailContent, - })); - - if (screenResult.action === 'GUARDRAIL_INTERVENED') { - // Clean up any already-uploaded attachments - await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, - `Attachment '${inlineAtt.filename}' was blocked by content policy.`, requestId); - } - } catch (screenErr) { - await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - logger.error('Attachment screening failed (fail-closed)', { - error: String(screenErr), + if (screenErr instanceof AttachmentScreeningError) { + logger.warn('Attachment screening rejected content', { attachment_filename: inlineAtt.filename, request_id: requestId, + error: screenErr.message, }); - return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, - 'Attachment content screening is temporarily unavailable. Please try again later.', requestId); + return errorResponse(400, ErrorCode.ATTACHMENT_INVALID_CONTENT, screenErr.message, requestId); } + logger.error('Attachment screening failed (fail-closed)', { + error: screenErr instanceof Error ? screenErr.message : String(screenErr), + attachment_filename: inlineAtt.filename, + request_id: requestId, + metric_type: 'attachment_screening_failure', + }); + return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, + 'Attachment content screening is temporarily unavailable. Please try again later.', requestId); + } + + if (screenResult.screening.status === 'blocked') { + await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); + const categories = screenResult.screening.categories.join(', '); + return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, + `Attachment '${inlineAtt.filename}' was blocked by content policy (${categories}).`, requestId); } - // Upload to S3 + // Upload cleaned content to S3 (images are EXIF-stripped and re-encoded) const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${context.userId}/${taskId}/${attachmentId}/${inlineAtt.filename}`; + let putResult; try { - const putResult = await s3Client.send(new PutObjectCommand({ + putResult = await s3Client.send(new PutObjectCommand({ Bucket: ATTACHMENTS_BUCKET, Key: s3Key, - Body: decoded, + Body: screenResult.content, ContentType: inlineAtt.content_type, })); - - uploadedS3Keys.push(s3Key); - const checksum = createHash('sha256').update(decoded).digest('hex'); - - attachmentRecords.push(createAttachmentRecord({ - attachment_id: attachmentId, - type: inlineAtt.type, - content_type: inlineAtt.content_type, - filename: inlineAtt.filename, - s3_key: s3Key, - s3_version_id: putResult.VersionId ?? 'unversioned', - size_bytes: decoded.length, - screening: { status: 'passed', screened_at: new Date().toISOString() }, - checksum_sha256: checksum, - })); } catch (s3Err) { await cleanupOrphanedAttachments(s3Client, uploadedS3Keys); - logger.error('S3 upload failed for inline attachment', { - error: String(s3Err), + logger.error('S3 upload failed for attachment', { + error: s3Err instanceof Error ? s3Err.message : String(s3Err), attachment_filename: inlineAtt.filename, + s3_key: s3Key, request_id: requestId, + metric_type: 'attachment_upload_failure', }); return errorResponse(500, ErrorCode.INTERNAL_ERROR, - `Failed to upload attachment '${inlineAtt.filename}'.`, requestId); + `Failed to store attachment '${inlineAtt.filename}'. Please try again.`, requestId); } + + uploadedS3Keys.push(s3Key); + + // Estimate image token cost (best-effort, non-blocking) + const tokenEstimate = inlineAtt.type === 'image' + ? await estimateImageTokensFromBuffer(screenResult.content) + : undefined; + + attachmentRecords.push(createAttachmentRecord({ + attachment_id: attachmentId, + type: inlineAtt.type, + content_type: inlineAtt.content_type, + filename: inlineAtt.filename, + s3_key: s3Key, + s3_version_id: putResult.VersionId ?? 'unversioned', + size_bytes: screenResult.content.length, + screening: { status: 'passed', screened_at: new Date().toISOString() }, + checksum_sha256: screenResult.checksum, + ...(tokenEstimate !== undefined && { token_estimate: tokenEstimate }), + })); } - // URL attachments get pending records (resolved during hydration) + // URL attachments: store as pending records — resolved during hydration + // (resolveUrlAttachments in orchestrator fetches, screens, and uploads to S3). for (const att of validatedAttachments) { if (att.delivery !== 'url_fetch') continue; attachmentRecords.push(createAttachmentRecord({ @@ -434,8 +457,25 @@ export async function createTaskCore( type: att.type, content_type: att.content_type, filename: att.filename, - screening: { status: 'pending' }, source_url: att.url, + screening: { status: 'pending' }, + })); + } + + // Presigned upload attachments get pending records (confirmed via confirm-uploads endpoint) + // Generate presigned POST policies so the client can upload directly to S3. + for (const att of validatedAttachments) { + if (att.delivery !== 'presigned') continue; + const presignedAtt = att as PresignedAttachment; + const attachmentId = ulid(); + const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${context.userId}/${taskId}/${attachmentId}/${presignedAtt.filename}`; + + attachmentRecords.push(createAttachmentRecord({ + attachment_id: attachmentId, + type: presignedAtt.type, + content_type: presignedAtt.content_type, + filename: presignedAtt.filename, + screening: { status: 'pending' }, })); } } @@ -500,11 +540,16 @@ export async function createTaskCore( ? 'pending:pr_resolution' : generateBranchName(taskId, body.task_description ?? body.repo); + // Determine initial status: PENDING_UPLOADS if any presigned attachments need uploading, + // otherwise SUBMITTED (inline/url/no attachments go straight to the pipeline). + const hasPresignedAttachments = validatedAttachments.some(a => a.delivery === 'presigned'); + const initialStatus = hasPresignedAttachments ? TaskStatus.PENDING_UPLOADS : TaskStatus.SUBMITTED; + // 5. Build task record const taskRecord: TaskRecord = { task_id: taskId, user_id: context.userId, - status: TaskStatus.SUBMITTED, + status: initialStatus, repo: body.repo, ...(body.issue_number !== undefined && { issue_number: body.issue_number }), task_type: taskType, @@ -518,7 +563,7 @@ export async function createTaskCore( channel_source: context.channelSource, channel_metadata: context.channelMetadata, ...(attachmentRecords.length > 0 && { attachments: attachmentRecords }), - status_created_at: `${TaskStatus.SUBMITTED}#${now}`, + status_created_at: `${initialStatus}#${now}`, created_at: now, updated_at: now, // Cedar HITL extensions (§10.2). Only written when the submit @@ -584,8 +629,10 @@ export async function createTaskCore( approval_gate_cap_source: blueprintCap !== undefined ? 'blueprint' : 'platform_default', }); - // 8. Async-invoke the orchestrator (fire-and-forget) - if (lambdaClient && process.env.ORCHESTRATOR_FUNCTION_ARN) { + // 8. Async-invoke the orchestrator (fire-and-forget). + // Skip for PENDING_UPLOADS — the orchestrator is invoked from confirm-uploads + // once all attachments are uploaded and screened. + if (initialStatus === TaskStatus.SUBMITTED && lambdaClient && process.env.ORCHESTRATOR_FUNCTION_ARN) { try { await lambdaClient.send(new InvokeCommand({ FunctionName: process.env.ORCHESTRATOR_FUNCTION_ARN, @@ -607,18 +654,20 @@ export async function createTaskCore( } // 9. Return created task - return successResponse(201, toTaskDetail(taskRecord), requestId); -} + if (hasPresignedAttachments && s3Client && ATTACHMENTS_BUCKET) { + // Generate presigned POST policies for presigned attachments + const uploadInstructions = await generateUploadInstructions( + taskRecord, validatedAttachments, context.userId, taskId, s3Client, + ); + const taskExpiresAt = new Date(Date.now() + 30 * 60 * 1000).toISOString(); // 30 min auto-cancel window + return successResponse(202, { + ...toTaskDetail(taskRecord), + upload_instructions: uploadInstructions, + task_expires_at: taskExpiresAt, + }, requestId); + } -/** - * Map MIME type to Bedrock GuardrailImageFormat. - * The SDK currently supports 'png' | 'jpeg' — GIF and WebP are mapped - * to 'png' (lossless container) since the guardrail inspects visual - * content, not codec fidelity. - */ -function mimeToGuardrailFormat(contentType: string): 'png' | 'jpeg' { - if (contentType === 'image/jpeg') return 'jpeg'; - return 'png'; + return successResponse(201, toTaskDetail(taskRecord), requestId); } const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/; @@ -630,6 +679,57 @@ function isValidBase64(data: string): boolean { return BASE64_PATTERN.test(data); } +/** Presigned POST policy expiry: 10 minutes. */ +const PRESIGNED_POST_EXPIRY_SECONDS = 600; + +/** + * Generate presigned POST upload instructions for presigned-delivery attachments. + */ +async function generateUploadInstructions( + taskRecord: TaskRecord, + validatedAttachments: readonly import('./types').ValidatedAttachment[], + userId: string, + taskId: string, + client: S3Client, +): Promise { + const instructions: AttachmentUploadInstruction[] = []; + const presignedRecords = taskRecord.attachments?.filter(a => a.screening.status === 'pending' && !a.source_url) ?? []; + + // Match validated presigned attachments with their records (same order as created) + const presignedValidated = validatedAttachments.filter(a => a.delivery === 'presigned') as PresignedAttachment[]; + + for (let i = 0; i < presignedValidated.length; i++) { + const att = presignedValidated[i]; + const record = presignedRecords[i]; + if (!record) continue; + + const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${userId}/${taskId}/${record.attachment_id}/${att.filename}`; + const { url, fields } = await createPresignedPost(client, { + Bucket: ATTACHMENTS_BUCKET!, + Key: s3Key, + Conditions: [ + ['content-length-range', 1, MAX_ATTACHMENT_SIZE_BYTES], + ['eq', '$Content-Type', att.content_type], + ], + Fields: { + 'Content-Type': att.content_type, + }, + Expires: PRESIGNED_POST_EXPIRY_SECONDS, + }); + + const expiresAt = new Date(Date.now() + PRESIGNED_POST_EXPIRY_SECONDS * 1000).toISOString(); + instructions.push({ + attachment_id: record.attachment_id, + filename: att.filename, + upload_url: url, + upload_fields: fields, + upload_expires_at: expiresAt, + }); + } + + return instructions; +} + /** * Clean up S3 objects from a partially-failed inline upload. * Best-effort — the 90-day lifecycle is the safety net if cleanup fails. diff --git a/cdk/src/handlers/shared/image-tokens.ts b/cdk/src/handlers/shared/image-tokens.ts new file mode 100644 index 00000000..5b2d3e16 --- /dev/null +++ b/cdk/src/handlers/shared/image-tokens.ts @@ -0,0 +1,73 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// Image token estimation matching Anthropic's documented resize rules. +// Shared by create-task-core.ts and confirm-uploads.ts. + +import sharp from 'sharp'; +import { logger } from './logger'; + +const MAX_IMAGE_SIDE = 1568; +const MAX_IMAGE_TOKENS = 1568; +const TOKEN_SAFETY_MARGIN = 1.2; +const TILE_SIZE = 28; + +/** + * Estimate the token cost of an image given its pixel dimensions. + * Applies Anthropic's resize-and-tile algorithm with a safety margin. + */ +export function estimateImageTokens(width: number, height: number): number { + let w = width; + let h = height; + + // Scale to fit MAX_IMAGE_SIDE on longest side + const maxSide = Math.max(w, h); + if (maxSide > MAX_IMAGE_SIDE) { + const scale = MAX_IMAGE_SIDE / maxSide; + w = Math.round(w * scale); + h = Math.round(h * scale); + } + + // Pad to next multiple of tile size + w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; + h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; + + // Token calculation with safety margin, then capped to hard ceiling + const rawTokens = Math.ceil((w * h) / 750); + return Math.min(Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN), MAX_IMAGE_TOKENS); +} + +/** + * Estimate image tokens from a buffer by reading dimensions via sharp. + * Returns undefined if dimensions cannot be determined (non-fatal). + */ +export async function estimateImageTokensFromBuffer(content: Buffer): Promise { + try { + const metadata = await sharp(content).metadata(); + if (metadata.width && metadata.height) { + return estimateImageTokens(metadata.width, metadata.height); + } + } catch (err) { + logger.warn('Failed to estimate image tokens (non-fatal)', { + error: err instanceof Error ? err.message : String(err), + content_length: content.length, + }); + } + return undefined; +} diff --git a/cdk/src/handlers/shared/orchestrator.ts b/cdk/src/handlers/shared/orchestrator.ts index a3d99990..8fede25f 100644 --- a/cdk/src/handlers/shared/orchestrator.ts +++ b/cdk/src/handlers/shared/orchestrator.ts @@ -18,15 +18,17 @@ */ import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { S3Client } from '@aws-sdk/client-s3'; import { DynamoDBDocumentClient, GetCommand, PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; import { ulid } from 'ulid'; -import { hydrateContext } from './context-hydration'; +import { AttachmentBudgetExceededError, hydrateContext, resolveGitHubToken } from './context-hydration'; import { logger } from './logger'; import { writeMinimalEpisode } from './memory'; import { coerceNumericOrNull } from './numeric'; import { computePromptVersion } from './prompt-version'; import { loadRepoConfig, type BlueprintConfig, type ComputeType } from './repo-config'; -import { APPROVAL_GATE_CAP_MAX, APPROVAL_GATE_CAP_MIN, type TaskRecord } from './types'; +import { resolveUrlAttachments } from './resolve-url-attachments'; +import { APPROVAL_GATE_CAP_MAX, APPROVAL_GATE_CAP_MIN, type AgentAttachmentPayload, type AttachmentRecord, type TaskRecord } from './types'; import { computeTtlEpoch, DEFAULT_MAX_TURNS } from './validation'; import { TaskStatus, TERMINAL_STATUSES, VALID_TRANSITIONS, type TaskStatusType } from '../../constructs/task-status'; @@ -39,6 +41,7 @@ const RUNTIME_ARN = process.env.RUNTIME_ARN!; const MAX_CONCURRENT = Number(process.env.MAX_CONCURRENT_TASKS_PER_USER ?? '3'); const TASK_RETENTION_DAYS = Number(process.env.TASK_RETENTION_DAYS ?? '90'); const MEMORY_ID = process.env.MEMORY_ID; +const ATTACHMENTS_BUCKET_NAME = process.env.ATTACHMENTS_BUCKET_NAME; /** * State tracked across waitForCondition poll cycles. @@ -245,6 +248,55 @@ export async function loadBlueprintConfig(task: TaskRecord): Promise tokenBudget) { + throw new AttachmentBudgetExceededError( + `Image attachments require ~${totalImageTokens} tokens, exceeding the ${tokenBudget}-token budget. ` + + 'Reduce the number or resolution of image attachments.', + ); + } + + return payloads; +} + /** * Cedar HITL Chunk 7b: structural guard on the TaskRecord's persisted * ``approval_gate_cap`` before we thread it into the agent payload. The @@ -361,6 +413,61 @@ export async function hydrateAndTransition(task: TaskRecord, blueprintConfig?: B }); } + // Resolve URL attachments: fetch (SSRF-safe), screen, upload to S3. + // This step handles type: 'url' attachments that are still pending. + // Throws AttachmentResolutionError on failure (propagates to fail the task). + let resolvedAttachments = task.attachments ?? []; + if (resolvedAttachments.some(a => a.type === 'url' && a.screening.status === 'pending') && ATTACHMENTS_BUCKET_NAME) { + const { BedrockRuntimeClient } = await import('@aws-sdk/client-bedrock-runtime'); + const screeningConfig = process.env.GUARDRAIL_ID && process.env.GUARDRAIL_VERSION + ? { + guardrailId: process.env.GUARDRAIL_ID, + guardrailVersion: process.env.GUARDRAIL_VERSION, + bedrockClient: new BedrockRuntimeClient({}), + } + : undefined; + + if (!screeningConfig) { + throw new AttachmentBudgetExceededError( + 'URL attachments require content screening (Bedrock Guardrail) but screening is not configured. ' + + 'Set GUARDRAIL_ID and GUARDRAIL_VERSION environment variables.', + ); + } + + // Resolve the GitHub token for URL fetches that target GitHub domains + const githubTokenSecretArn = process.env.GITHUB_TOKEN_SECRET_ARN; + let githubToken: string | undefined; + if (githubTokenSecretArn) { + try { + githubToken = await resolveGitHubToken(githubTokenSecretArn); + } catch (err) { + logger.warn('Failed to resolve GitHub token for URL attachment fetch', { + task_id: task.task_id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + resolvedAttachments = await resolveUrlAttachments( + resolvedAttachments, + task.task_id, + task.user_id, + { + s3Client: new S3Client({}), + bucketName: ATTACHMENTS_BUCKET_NAME, + screeningConfig, + githubToken, + }, + ); + } + + // Resolve attachment payloads for the agent (only passed-screening attachments). + // Token budget check is applied here — throws AttachmentBudgetExceededError if + // image tokens exceed the configured prompt token budget. + const attachmentPayloads = resolvedAttachments.length > 0 && ATTACHMENTS_BUCKET_NAME + ? resolveAttachmentPayloads(resolvedAttachments, Number(process.env.USER_PROMPT_TOKEN_BUDGET ?? '100000')) + : []; + const payload: Record = { repo_url: task.repo, task_id: task.task_id, @@ -424,6 +531,7 @@ export async function hydrateAndTransition(task: TaskRecord, blueprintConfig?: B hydrated_context: hydratedContext, channel_source: task.channel_source, ...(task.channel_metadata && Object.keys(task.channel_metadata).length > 0 && { channel_metadata: task.channel_metadata }), + ...(attachmentPayloads.length > 0 && { attachments: attachmentPayloads }), }; if (hydratedContext.fallback_error) { diff --git a/cdk/src/handlers/shared/resolve-url-attachments.ts b/cdk/src/handlers/shared/resolve-url-attachments.ts new file mode 100644 index 00000000..93852fd8 --- /dev/null +++ b/cdk/src/handlers/shared/resolve-url-attachments.ts @@ -0,0 +1,447 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * URL attachment resolution: SSRF-safe fetch, screen, and upload to S3. + * + * During context hydration, URL attachments (type: 'url', screening: pending) + * are fetched from their source URLs with full SSRF protection: + * 1. DNS resolution pinning (resolve, validate IP, connect to resolved IP) + * 2. Private IP range blocking + * 3. Redirect validation (re-check IPs after each redirect, max 2) + * 4. Timeout and size limits + * + * Fetched content is screened through the same Bedrock Guardrail pipeline as + * inline/presigned attachments, then uploaded to S3. + * + * Tests: cdk/test/handlers/shared/resolve-url-attachments.test.ts + */ + +import { createHash } from 'crypto'; +import { promises as dns } from 'dns'; +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { screenImage, screenTextFile, AttachmentScreeningError, type ScreeningConfig } from './attachment-screening'; +import { AttachmentResolutionError } from './context-hydration'; +import { estimateImageTokensFromBuffer } from './image-tokens'; +import { logger } from './logger'; +import { createAttachmentRecord, type AttachmentRecord } from './types'; +import { ATTACHMENT_OBJECT_KEY_PREFIX } from '../../constructs/attachments-bucket'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const URL_FETCH_TIMEOUT_MS = 10_000; +const MAX_FETCH_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB +const MAX_REDIRECTS = 2; + +/** RFC 1918 + link-local + loopback + IPv6 equivalents. */ +const PRIVATE_IP_RANGES = [ + // IPv4 + { prefix: '10.', mask: null }, + { + prefix: '172.', + mask: (ip: string) => { + const second = parseInt(ip.split('.')[1], 10); + return second >= 16 && second <= 31; + }, + }, + { prefix: '192.168.', mask: null }, + { prefix: '169.254.', mask: null }, + { prefix: '127.', mask: null }, + { prefix: '0.', mask: null }, + // IPv6 + { prefix: '::1', mask: null }, + { prefix: 'fc', mask: null }, + { prefix: 'fd', mask: null }, + { prefix: 'fe80:', mask: null }, +] as const; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface ResolveUrlAttachmentsOptions { + readonly s3Client: S3Client; + readonly bucketName: string; + readonly screeningConfig: ScreeningConfig; + readonly githubToken?: string; + readonly githubInstallationDomain?: string; +} + +// --------------------------------------------------------------------------- +// SSRF protection +// --------------------------------------------------------------------------- + +/** + * Check if an IP address belongs to a private/internal range. + * Returns a reason string if private, undefined if public. + */ +function isPrivateIp(ip: string): string | undefined { + const normalized = ip.toLowerCase(); + + for (const range of PRIVATE_IP_RANGES) { + if (typeof range.mask === 'function') { + if (normalized.startsWith(range.prefix) && range.mask(normalized)) { + return `IP ${ip} is in private range (${range.prefix}x)`; + } + } else if (normalized.startsWith(range.prefix) || normalized === range.prefix) { + return `IP ${ip} is in private/reserved range`; + } + } + + return undefined; +} + +/** + * Resolve DNS and validate the resolved IP is not in a private range. + * Returns the resolved IP address for connection pinning. + * + * DNS resolution pinning prevents the DNS rebinding attack: + * 1. Attacker's DNS returns a public IP on first lookup (passes validation) + * 2. Attacker's DNS returns a private IP on second lookup (reaches internal services) + * 3. By resolving once and pinning the connection to that IP, we eliminate the TOCTOU window + */ +async function resolveAndValidate(hostname: string): Promise { + let addresses: string[]; + + try { + // Try IPv4 first (more common for HTTP endpoints) + addresses = await dns.resolve4(hostname); + } catch { + try { + addresses = await dns.resolve6(hostname); + } catch (err) { + throw new AttachmentResolutionError( + `DNS resolution failed for '${hostname}'. Check that the URL is correct and the server is reachable.`, + { cause: err }, + ); + } + } + + if (addresses.length === 0) { + throw new AttachmentResolutionError( + `DNS resolution returned no addresses for '${hostname}'.`, + ); + } + + // Validate all resolved IPs — reject if any is private + for (const ip of addresses) { + const privateReason = isPrivateIp(ip); + if (privateReason) { + throw new AttachmentResolutionError( + `URL attachment blocked: ${privateReason}. ` + + 'URL attachments cannot target private or internal network addresses.', + ); + } + } + + // Return the first valid IP for connection pinning + return addresses[0]; +} + +/** + * Build a pinned URL that connects to the resolved IP while preserving + * the original path, query, and port. The Host header carries the real + * hostname for TLS SNI / virtual-host routing. + */ +function buildPinnedUrl(originalUrl: URL, resolvedIp: string): URL { + const pinned = new URL(originalUrl.toString()); + // For IPv6 addresses, wrap in brackets for URL hostname + pinned.hostname = resolvedIp.includes(':') ? `[${resolvedIp}]` : resolvedIp; + return pinned; +} + +/** + * Fetch a URL with SSRF protections: DNS pinning, private IP rejection, + * redirect validation, timeout, and size limit. + * + * DNS pinning: we resolve the hostname, validate the IP is public, then + * rewrite the fetch URL to connect directly to that IP (with the original + * Host header for TLS SNI). This closes the DNS rebinding TOCTOU window. + */ +async function ssrfSafeFetch( + url: string, + options?: { githubToken?: string; githubInstallationDomain?: string }, +): Promise<{ content: Buffer; finalContentType: string }> { + const parsedUrl = new URL(url); + + if (parsedUrl.protocol !== 'https:') { + throw new AttachmentResolutionError( + `URL attachment must use HTTPS. Got: ${parsedUrl.protocol}`, + ); + } + + // Resolve DNS and validate IP before connecting + const resolvedIp = await resolveAndValidate(parsedUrl.hostname); + + // Build the pinned URL that connects to the validated IP + let pinnedUrl = buildPinnedUrl(parsedUrl, resolvedIp); + + // Determine if we should send auth headers (only to GitHub) + const headers: Record = { + 'Host': parsedUrl.hostname, + 'User-Agent': 'ABCA-Attachment-Fetcher/1.0', + }; + + if (options?.githubToken && isGitHubUrl(parsedUrl.hostname, options.githubInstallationDomain)) { + headers.Authorization = `Bearer ${options.githubToken}`; + } + + let currentUrl = url; + let redirectCount = 0; + let response: Response; + + // Fetch with redirect following (re-validate each redirect target) + while (true) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), URL_FETCH_TIMEOUT_MS); + + try { + // Use pinnedUrl (resolved IP) for the actual connection, with Host header + // preserving the real hostname for TLS SNI and virtual-host routing. + response = await fetch(pinnedUrl.toString(), { + headers, + signal: controller.signal, + redirect: 'manual', // Handle redirects manually for IP re-validation + }); + } catch (err: any) { + if (err.name === 'AbortError') { + throw new AttachmentResolutionError( + `URL attachment fetch timed out after ${URL_FETCH_TIMEOUT_MS / 1000}s: ${url}`, + ); + } + throw new AttachmentResolutionError( + `URL attachment fetch failed: ${err.message ?? String(err)}`, + { cause: err }, + ); + } finally { + clearTimeout(timeout); + } + + // Handle redirects + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + if (!location) { + throw new AttachmentResolutionError( + `URL attachment redirect (${response.status}) had no Location header.`, + ); + } + + redirectCount++; + if (redirectCount > MAX_REDIRECTS) { + throw new AttachmentResolutionError( + `URL attachment exceeded maximum redirects (${MAX_REDIRECTS}): ${url}`, + ); + } + + // Re-validate the redirect target + const redirectUrl = new URL(location, currentUrl); + if (redirectUrl.protocol !== 'https:') { + throw new AttachmentResolutionError( + `URL attachment redirect to non-HTTPS URL blocked: ${redirectUrl.protocol}`, + ); + } + const redirectIp = await resolveAndValidate(redirectUrl.hostname); + currentUrl = redirectUrl.toString(); + + // Pin the redirect target to its resolved IP + pinnedUrl = buildPinnedUrl(redirectUrl, redirectIp); + + // Don't send auth headers to non-GitHub redirect targets + if (!isGitHubUrl(redirectUrl.hostname, options?.githubInstallationDomain)) { + delete headers.Authorization; + } + headers.Host = redirectUrl.hostname; + continue; + } + + break; + } + + if (!response!.ok) { + throw new AttachmentResolutionError( + `URL attachment fetch failed with status ${response!.status}: ${url}`, + ); + } + + // Stream response with size limit enforcement + const contentType = response!.headers.get('content-type') ?? 'application/octet-stream'; + const chunks: Buffer[] = []; + let totalBytes = 0; + + const reader = response!.body?.getReader(); + if (!reader) { + throw new AttachmentResolutionError( + `URL attachment response has no body: ${url}`, + ); + } + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + totalBytes += value.length; + if (totalBytes > MAX_FETCH_SIZE_BYTES) { + await reader.cancel(); + throw new AttachmentResolutionError( + `URL attachment exceeds ${MAX_FETCH_SIZE_BYTES / (1024 * 1024)} MB size limit: ${url}`, + ); + } + chunks.push(Buffer.from(value)); + } + + return { + content: Buffer.concat(chunks), + finalContentType: contentType.split(';')[0].trim(), + }; +} + +function isGitHubUrl(hostname: string, installationDomain?: string): boolean { + const githubHosts = ['github.com', 'raw.githubusercontent.com', 'api.github.com']; + if (installationDomain) githubHosts.push(installationDomain); + const lower = hostname.toLowerCase(); + return githubHosts.some(h => lower === h || lower.endsWith(`.${h}`)); +} + +// --------------------------------------------------------------------------- +// Main resolution function +// --------------------------------------------------------------------------- + +/** + * Resolve URL attachments: fetch, screen, upload to S3. + * + * Takes the task's attachment records (from DynamoDB), resolves any with + * screening.status === 'pending' and type === 'url', and returns the full + * updated attachment list. + * + * Throws AttachmentResolutionError if any URL attachment cannot be resolved. + * The caller (orchestrator) should let this propagate to fail the task. + */ +export async function resolveUrlAttachments( + attachments: AttachmentRecord[], + taskId: string, + userId: string, + options: ResolveUrlAttachmentsOptions, +): Promise { + const pendingUrls = attachments.filter( + att => att.type === 'url' && att.screening.status === 'pending', + ); + + if (pendingUrls.length === 0) { + return attachments; + } + + logger.info('Resolving URL attachments', { + task_id: taskId, + count: pendingUrls.length, + }); + + const resolved = new Map(); + + for (const att of pendingUrls) { + if (!att.source_url) { + throw new AttachmentResolutionError( + `URL attachment '${att.filename}' has no source_url.`, + ); + } + + // Fetch with SSRF protection + const { content, finalContentType } = await ssrfSafeFetch(att.source_url, { + githubToken: options.githubToken, + }); + + logger.info('URL attachment fetched', { + attachment_id: att.attachment_id, + filename: att.filename, + size_bytes: content.length, + content_type: finalContentType, + }); + + // Determine if this is an image or file based on content type + const isImage = finalContentType.startsWith('image/'); + const resolvedContentType = att.content_type || finalContentType; + + // Screen the fetched content + let screenResult; + try { + screenResult = isImage + ? await screenImage(content, resolvedContentType, att.filename, options.screeningConfig) + : await screenTextFile(content, resolvedContentType, att.filename, options.screeningConfig); + } catch (err) { + if (err instanceof AttachmentScreeningError) { + throw new AttachmentResolutionError( + `URL attachment '${att.filename}' was blocked by content screening: ${err.message}`, + { cause: err }, + ); + } + throw new AttachmentResolutionError( + `URL attachment '${att.filename}' could not be screened: ${err instanceof Error ? err.message : String(err)}`, + { cause: err }, + ); + } + + if (screenResult.screening.status === 'blocked') { + throw new AttachmentResolutionError( + `URL attachment '${att.filename}' was blocked by content policy: ${screenResult.screening.categories.join(', ')}`, + ); + } + + // Upload screened content to S3 + const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${userId}/${taskId}/${att.attachment_id}/${att.filename}`; + const putResult = await options.s3Client.send(new PutObjectCommand({ + Bucket: options.bucketName, + Key: s3Key, + Body: screenResult.content, + ContentType: resolvedContentType, + })); + + // Compute checksum + const checksum = createHash('sha256').update(screenResult.content).digest('hex'); + + // Estimate token cost for images + let tokenEstimate: number | undefined; + if (isImage) { + tokenEstimate = await estimateImageTokensFromBuffer(screenResult.content); + } + + resolved.set(att.attachment_id, createAttachmentRecord({ + attachment_id: att.attachment_id, + type: att.type, + content_type: resolvedContentType, + filename: att.filename, + s3_key: s3Key, + s3_version_id: putResult.VersionId ?? 'unversioned', + size_bytes: screenResult.content.length, + screening: { status: 'passed', screened_at: new Date().toISOString() }, + source_url: att.source_url, + checksum_sha256: checksum, + ...(tokenEstimate !== undefined && { token_estimate: tokenEstimate }), + })); + + logger.info('URL attachment resolved and stored', { + attachment_id: att.attachment_id, + filename: att.filename, + s3_key: s3Key, + }); + } + + // Merge resolved records back into the full attachment list + return attachments.map(att => resolved.get(att.attachment_id) ?? att); +} diff --git a/cdk/src/handlers/shared/response.ts b/cdk/src/handlers/shared/response.ts index 394c905e..f4d37087 100644 --- a/cdk/src/handlers/shared/response.ts +++ b/cdk/src/handlers/shared/response.ts @@ -44,6 +44,10 @@ export const ErrorCode = { ATTACHMENT_INVALID_CONTENT: 'ATTACHMENT_INVALID_CONTENT', ATTACHMENT_INVALID_FILENAME: 'ATTACHMENT_INVALID_FILENAME', ATTACHMENT_SCREENING_UNAVAILABLE: 'ATTACHMENT_SCREENING_UNAVAILABLE', + ATTACHMENT_SIZE_MISMATCH: 'ATTACHMENT_SIZE_MISMATCH', + ATTACHMENT_UPLOAD_MISSING: 'ATTACHMENT_UPLOAD_MISSING', + UPLOADS_NOT_PENDING: 'UPLOADS_NOT_PENDING', + SCREENING_DEADLINE_EXCEEDED: 'SCREENING_DEADLINE_EXCEEDED', // Cedar HITL (§7.1 / §7.2 / §7.3). REQUEST_NOT_FOUND collapses "row // missing" and "wrong caller" into a single 404 response so the API // surface does not function as an existence oracle (§7.1 finding #6). diff --git a/cdk/src/handlers/shared/types.ts b/cdk/src/handlers/shared/types.ts index c4b615fb..36d5121f 100644 --- a/cdk/src/handlers/shared/types.ts +++ b/cdk/src/handlers/shared/types.ts @@ -476,6 +476,12 @@ export interface AttachmentUploadInstruction { readonly upload_expires_at: string; } +/** Response from POST /v1/tasks when presigned uploads are required. */ +export interface CreateTaskResponse extends TaskDetail { + readonly upload_instructions?: readonly AttachmentUploadInstruction[]; + readonly task_expires_at?: string; +} + // --------------------------------------------------------------------------- // Agent attachment payload (orchestrator → agent runtime) // --------------------------------------------------------------------------- diff --git a/cdk/src/handlers/slack-command-processor.ts b/cdk/src/handlers/slack-command-processor.ts index 9163ee29..682c3718 100644 --- a/cdk/src/handlers/slack-command-processor.ts +++ b/cdk/src/handlers/slack-command-processor.ts @@ -24,6 +24,7 @@ import { createTaskCore } from './shared/create-task-core'; import { logger } from './shared/logger'; import { slackFetch } from './shared/slack-api'; import { getSlackSecret, SLACK_SECRET_PREFIX } from './shared/slack-verify'; +import type { Attachment } from './shared/types'; import type { SlackCommandPayload } from './slack-commands'; /** @@ -42,10 +43,20 @@ export interface SlashCommandEvent extends BasePayload, SlackCommandPayload { readonly source: 'slash'; } +/** Metadata for a file attached to a Slack message. */ +export interface SlackFileRef { + readonly id: string; + readonly name: string; + readonly mimetype: string; + readonly size: number; + readonly url_private_download: string; +} + /** @mention invocation — no response_url; reply via chat.postMessage in-thread. */ export interface MentionEvent extends BasePayload { readonly source: 'mention'; readonly mention_thread_ts?: string; + readonly files?: readonly SlackFileRef[]; } /** Discriminated union of the inbound events the processor accepts. */ @@ -209,12 +220,24 @@ async function handleSubmit(event: MentionEvent, args: string[], reply: ReplyFn) channelMetadata.slack_thread_ts = event.mention_thread_ts; } + // Extract file attachments from the Slack event (if present). + // Files are downloaded from Slack CDN and passed as inline base64 attachments. + const attachments = await extractSlackFileAttachments(event, reply); + if (attachments === null) { + // extractSlackFileAttachments already replied with the error + if (event.mention_thread_ts) { + await swapReaction(event.team_id, event.channel_id, event.mention_thread_ts, 'eyes', 'x'); + } + return; + } + // Create the task through the shared core. const result = await createTaskCore( { repo, issue_number: issueNumber, task_description: description, + ...(attachments.length > 0 && { attachments }), }, { userId: platformUserId, @@ -345,6 +368,106 @@ async function checkChannelAccess(teamId: string, channelId: string): Promise<{ } } +// ─── Slack File Extraction ─────────────────────────────────────────────────── + +/** Max size for a Slack file attachment (10 MB per the design doc). */ +const SLACK_FILE_MAX_SIZE_BYTES = 10 * 1024 * 1024; +/** Max number of file attachments per Slack message. */ +const SLACK_FILE_MAX_COUNT = 10; + +/** MIME types supported for attachments (must match validation.ts). */ +const SUPPORTED_IMAGE_MIMES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']); +const SUPPORTED_FILE_MIMES = new Set([ + 'text/plain', 'text/csv', 'text/markdown', 'application/json', + 'application/pdf', 'text/x-log', +]); + +/** + * Download Slack file attachments and convert them to inline Attachment objects. + * Returns null if validation fails (reply already sent). Returns an empty array + * if no files are attached. + * + * Implements atomic failure semantics: if ANY file fails validation or download, + * the entire submission is rejected with a descriptive error listing all failures. + */ +async function extractSlackFileAttachments( + event: MentionEvent, + reply: ReplyFn, +): Promise { + const files = event.files; + if (!files || files.length === 0) return []; + + if (files.length > SLACK_FILE_MAX_COUNT) { + await reply(`:x: Task not created. Too many attachments (${files.length}, max ${SLACK_FILE_MAX_COUNT}).`); + return null; + } + + const errors: string[] = []; + const attachments: Attachment[] = []; + + const botToken = await getBotToken(event.team_id); + if (!botToken) { + await reply(':x: Task not created. Cannot download attachments (bot token not found).'); + return null; + } + + for (const file of files) { + // Validate size + if (file.size > SLACK_FILE_MAX_SIZE_BYTES) { + const sizeMb = (file.size / (1024 * 1024)).toFixed(1); + errors.push(`\`${file.name}\` (too large, ${sizeMb} MB > 10 MB limit)`); + continue; + } + + // Validate MIME type + const mime = file.mimetype.toLowerCase(); + const isImage = SUPPORTED_IMAGE_MIMES.has(mime); + const isFile = SUPPORTED_FILE_MIMES.has(mime); + if (!isImage && !isFile) { + errors.push(`\`${file.name}\` has unsupported type \`${mime}\``); + continue; + } + + // Download the file from Slack CDN using the bot token + try { + const response = await fetch(file.url_private_download, { + headers: { Authorization: `Bearer ${botToken}` }, + }); + + if (!response.ok) { + errors.push(`\`${file.name}\` (download failed: HTTP ${response.status})`); + continue; + } + + const buffer = Buffer.from(await response.arrayBuffer()); + + attachments.push({ + type: isImage ? 'image' : 'file', + content_type: mime, + filename: file.name, + data: buffer.toString('base64'), + }); + } catch (err) { + logger.error('Failed to download Slack file', { + filename: file.name, + error: err instanceof Error ? err.message : String(err), + }); + errors.push(`\`${file.name}\` (download failed)`); + } + } + + // Atomic failure: if any file failed, reject the entire submission + if (errors.length > 0) { + const errorList = errors.length === 1 + ? errors[0] + : `${errors.length} attachment errors: ${errors.join(', ')}`; + await reply(`:x: Task not created. ${errorList}. Fix or remove these files and try again.`); + return null; + } + + return attachments; +} + // ─── Helpers ────────────────────────────────────────────────────────────────── async function lookupPlatformUser(teamId: string, userId: string): Promise { diff --git a/cdk/src/handlers/slack-events.ts b/cdk/src/handlers/slack-events.ts index 206ba32a..954f53e7 100644 --- a/cdk/src/handlers/slack-events.ts +++ b/cdk/src/handlers/slack-events.ts @@ -25,7 +25,7 @@ import type { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda'; import { logger } from './shared/logger'; import { slackFetch } from './shared/slack-api'; import { getSlackSecret, SLACK_SECRET_PREFIX, verifySlackRequest } from './shared/slack-verify'; -import type { MentionEvent } from './slack-command-processor'; +import type { MentionEvent, SlackFileRef } from './slack-command-processor'; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); const sm = new SecretsManagerClient({}); @@ -201,6 +201,18 @@ async function handleAppMention( const description = text.replace(repo, '').replace(/\s+/g, ' ').trim(); const commandText = `submit ${repo} ${description}`; + // Extract file references from the Slack event (if any attached) + const rawFiles = Array.isArray(event.files) ? event.files as Array> : []; + const files: SlackFileRef[] = rawFiles + .filter(f => typeof f.url_private_download === 'string' && typeof f.name === 'string') + .map(f => ({ + id: String(f.id ?? ''), + name: String(f.name), + mimetype: String(f.mimetype ?? 'application/octet-stream'), + size: typeof f.size === 'number' ? f.size : 0, + url_private_download: String(f.url_private_download), + })); + const mentionPayload: MentionEvent = { text: commandText, user_id: userId, @@ -208,6 +220,7 @@ async function handleAppMention( channel_id: channelId, source: 'mention', mention_thread_ts: threadTs ?? messageTs, + ...(files.length > 0 && { files }), }; // React with :eyes: immediately so the user knows the bot saw their message. diff --git a/cdk/src/handlers/slack-interactions.ts b/cdk/src/handlers/slack-interactions.ts index 59b68d1f..ac25ccff 100644 --- a/cdk/src/handlers/slack-interactions.ts +++ b/cdk/src/handlers/slack-interactions.ts @@ -132,21 +132,23 @@ async function handleCancelAction(payload: SlackInteractionPayload, actionId: st } // Attempt to cancel. - const ACTIVE_STATUSES = ['SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING']; + const CANCELLABLE_STATUSES = ['PENDING_UPLOADS', 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING']; try { await ddb.send(new UpdateCommand({ TableName: TASK_TABLE, Key: { task_id: taskId }, UpdateExpression: 'SET #s = :cancelled, updated_at = :now', - ConditionExpression: '#s IN (:s1, :s2, :s3, :s4)', + ConditionExpression: '#s IN (:s1, :s2, :s3, :s4, :s5, :s6)', ExpressionAttributeNames: { '#s': 'status' }, ExpressionAttributeValues: { ':cancelled': 'CANCELLED', ':now': new Date().toISOString(), - ':s1': ACTIVE_STATUSES[0], - ':s2': ACTIVE_STATUSES[1], - ':s3': ACTIVE_STATUSES[2], - ':s4': ACTIVE_STATUSES[3], + ':s1': CANCELLABLE_STATUSES[0], + ':s2': CANCELLABLE_STATUSES[1], + ':s3': CANCELLABLE_STATUSES[2], + ':s4': CANCELLABLE_STATUSES[3], + ':s5': CANCELLABLE_STATUSES[4], + ':s6': CANCELLABLE_STATUSES[5], }, })); diff --git a/cdk/src/stacks/agent.ts b/cdk/src/stacks/agent.ts index 4dca99ad..8cbbae24 100644 --- a/cdk/src/stacks/agent.ts +++ b/cdk/src/stacks/agent.ts @@ -42,6 +42,7 @@ import { DnsFirewall } from '../constructs/dns-firewall'; // import { EcsAgentCluster } from '../constructs/ecs-agent-cluster'; import { FanOutConsumer } from '../constructs/fanout-consumer'; import { LinearIntegration } from '../constructs/linear-integration'; +import { PendingUploadCleanup } from '../constructs/pending-upload-cleanup'; import { RepoTable } from '../constructs/repo-table'; import { SlackIntegration } from '../constructs/slack-integration'; import { StrandedTaskReconciler } from '../constructs/stranded-task-reconciler'; @@ -270,6 +271,7 @@ export class AgentStack extends Stack { agentCoreStopSessionRuntimeArn: lazyRuntimeArn, traceArtifactsBucket: traceArtifactsBucket.bucket, attachmentsBucket: attachmentsBucket.bucket, + userConcurrencyTable: userConcurrencyTable.table, }); // --- AgentCore Runtime (IAM-authed orchestrator path) --- @@ -600,6 +602,16 @@ export class AgentStack extends Stack { userConcurrencyTable: userConcurrencyTable.table, }); + // --- Pending-upload cleanup rule --- + // Auto-cancels PENDING_UPLOADS tasks that were never confirmed within + // 30 minutes (client crash, abandoned session, network failure). + // Cleans up orphaned S3 objects under the task's attachment prefix. + new PendingUploadCleanup(this, 'PendingUploadCleanup', { + taskTable: taskTable.table, + taskEventsTable: taskEventsTable.table, + attachmentsBucket: attachmentsBucket.bucket, + }); + // --- Fan-out plane consumer --- // Consumes TaskEventsTable DynamoDB Streams and dispatches events to // Slack / GitHub / email per per-channel default filters. GitHub diff --git a/cdk/src/types/pdf-parse.d.ts b/cdk/src/types/pdf-parse.d.ts new file mode 100644 index 00000000..82ec15de --- /dev/null +++ b/cdk/src/types/pdf-parse.d.ts @@ -0,0 +1,9 @@ +declare module 'pdf-parse' { + interface PdfParseResult { + text: string; + numpages: number; + info: Record; + } + function pdfParse(data: Buffer, options?: { max?: number }): Promise; + export = pdfParse; +} diff --git a/cdk/test/constructs/task-status.test.ts b/cdk/test/constructs/task-status.test.ts index 2a183fe9..cf01ee8c 100644 --- a/cdk/test/constructs/task-status.test.ts +++ b/cdk/test/constructs/task-status.test.ts @@ -17,19 +17,19 @@ * SOFTWARE. */ -import { ACTIVE_STATUSES, TaskStatus, TaskStatusType, TERMINAL_STATUSES, VALID_TRANSITIONS } from '../../src/constructs/task-status'; +import { ACTIVE_STATUSES, PRE_ACTIVE_STATUSES, TaskStatus, TaskStatusType, TERMINAL_STATUSES, VALID_TRANSITIONS } from '../../src/constructs/task-status'; const ALL_STATUSES: TaskStatusType[] = Object.values(TaskStatus); describe('TaskStatus', () => { - test('defines exactly 9 states', () => { - // 8 original + AWAITING_APPROVAL (Cedar HITL gates, §10.3). - expect(ALL_STATUSES).toHaveLength(9); + test('defines exactly 10 states', () => { + // 8 original + AWAITING_APPROVAL (Cedar HITL gates, §10.3) + PENDING_UPLOADS (attachments). + expect(ALL_STATUSES).toHaveLength(10); }); test('contains all expected states', () => { expect(ALL_STATUSES).toEqual(expect.arrayContaining([ - 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', + 'PENDING_UPLOADS', 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', 'COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT', ])); }); @@ -38,6 +38,11 @@ describe('TaskStatus', () => { expect(TaskStatus.AWAITING_APPROVAL).toBe('AWAITING_APPROVAL'); expect(ALL_STATUSES).toContain('AWAITING_APPROVAL'); }); + + test('PENDING_UPLOADS is included as a distinct state', () => { + expect(TaskStatus.PENDING_UPLOADS).toBe('PENDING_UPLOADS'); + expect(ALL_STATUSES).toContain('PENDING_UPLOADS'); + }); }); describe('TERMINAL_STATUSES', () => { @@ -73,14 +78,36 @@ describe('ACTIVE_STATUSES', () => { }); }); -describe('TERMINAL_STATUSES and ACTIVE_STATUSES', () => { - test('are disjoint (no overlap)', () => { - const overlap = TERMINAL_STATUSES.filter(s => ACTIVE_STATUSES.includes(s)); - expect(overlap).toHaveLength(0); +describe('PRE_ACTIVE_STATUSES', () => { + test('contains exactly 1 pre-active state', () => { + expect(PRE_ACTIVE_STATUSES).toHaveLength(1); + }); + + test('contains PENDING_UPLOADS', () => { + expect(PRE_ACTIVE_STATUSES).toContain(TaskStatus.PENDING_UPLOADS); + }); + + test('PENDING_UPLOADS is NOT in ACTIVE_STATUSES (no concurrency slot consumed)', () => { + expect(ACTIVE_STATUSES).not.toContain(TaskStatus.PENDING_UPLOADS); + }); + + test('PENDING_UPLOADS is NOT terminal', () => { + expect(TERMINAL_STATUSES).not.toContain(TaskStatus.PENDING_UPLOADS); + }); +}); + +describe('TERMINAL_STATUSES, ACTIVE_STATUSES, and PRE_ACTIVE_STATUSES', () => { + test('are pairwise disjoint (no overlap)', () => { + const overlapTA = TERMINAL_STATUSES.filter(s => ACTIVE_STATUSES.includes(s)); + const overlapTP = TERMINAL_STATUSES.filter(s => PRE_ACTIVE_STATUSES.includes(s)); + const overlapAP = ACTIVE_STATUSES.filter(s => PRE_ACTIVE_STATUSES.includes(s)); + expect(overlapTA).toHaveLength(0); + expect(overlapTP).toHaveLength(0); + expect(overlapAP).toHaveLength(0); }); test('together cover all states', () => { - const combined = [...TERMINAL_STATUSES, ...ACTIVE_STATUSES]; + const combined = [...TERMINAL_STATUSES, ...ACTIVE_STATUSES, ...PRE_ACTIVE_STATUSES]; expect(combined).toHaveLength(ALL_STATUSES.length); expect(combined).toEqual(expect.arrayContaining(ALL_STATUSES)); }); @@ -150,4 +177,28 @@ describe('VALID_TRANSITIONS', () => { // gate can still fire; §10.3 explicitly lists it. expect(VALID_TRANSITIONS[TaskStatus.HYDRATING]).toContain(TaskStatus.AWAITING_APPROVAL); }); + + test('PENDING_UPLOADS can transition to SUBMITTED (confirm-uploads success)', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).toContain(TaskStatus.SUBMITTED); + }); + + test('PENDING_UPLOADS can transition to FAILED (screening blocked)', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).toContain(TaskStatus.FAILED); + }); + + test('PENDING_UPLOADS can transition to CANCELLED (user cancel or auto-cancel)', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).toContain(TaskStatus.CANCELLED); + }); + + test('PENDING_UPLOADS cannot skip to RUNNING or later states', () => { + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).not.toContain(TaskStatus.RUNNING); + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).not.toContain(TaskStatus.HYDRATING); + expect(VALID_TRANSITIONS[TaskStatus.PENDING_UPLOADS]).not.toContain(TaskStatus.FINALIZING); + }); + + test('pre-active states have at least one outgoing transition', () => { + for (const status of PRE_ACTIVE_STATUSES) { + expect(VALID_TRANSITIONS[status].length).toBeGreaterThan(0); + } + }); }); diff --git a/cdk/test/handlers/confirm-uploads.test.ts b/cdk/test/handlers/confirm-uploads.test.ts new file mode 100644 index 00000000..b38b6f33 --- /dev/null +++ b/cdk/test/handlers/confirm-uploads.test.ts @@ -0,0 +1,199 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import type { APIGatewayProxyEvent, Context } from 'aws-lambda'; + +// --- Mocks --- +const ddbSend = jest.fn(); +jest.mock('@aws-sdk/client-dynamodb', () => ({ DynamoDBClient: jest.fn(() => ({})) })); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + DynamoDBDocumentClient: { from: jest.fn(() => ({ send: ddbSend })) }, + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), + PutCommand: jest.fn((input: unknown) => ({ _type: 'Put', input })), + UpdateCommand: jest.fn((input: unknown) => ({ _type: 'Update', input })), +})); + +const s3Send = jest.fn(); +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(() => ({ send: (...args: unknown[]) => s3Send(...args) })), + DeleteObjectsCommand: jest.fn((input: unknown) => ({ _type: 'S3Delete', input })), + GetObjectCommand: jest.fn((input: unknown) => ({ _type: 'S3Get', input })), + HeadObjectCommand: jest.fn((input: unknown) => ({ _type: 'S3Head', input })), + PutObjectCommand: jest.fn((input: unknown) => ({ _type: 'S3Put', input })), +})); + +const lambdaSend = jest.fn(); +jest.mock('@aws-sdk/client-lambda', () => ({ + LambdaClient: jest.fn(() => ({ send: lambdaSend })), + InvokeCommand: jest.fn((input: unknown) => ({ _type: 'Invoke', input })), +})); + +jest.mock('@aws-sdk/client-bedrock-runtime', () => ({ + BedrockRuntimeClient: jest.fn(() => ({})), +})); + +jest.mock('../../src/handlers/shared/attachment-screening', () => ({ + screenImage: jest.fn(), + screenTextFile: jest.fn(), + AttachmentScreeningError: class extends Error { constructor(msg: string) { super(msg); this.name = 'AttachmentScreeningError'; } }, +})); + +jest.mock('../../src/handlers/shared/image-tokens', () => ({ + estimateImageTokensFromBuffer: jest.fn(() => 100), +})); + +jest.mock('ulid', () => ({ ulid: jest.fn(() => 'ULID-1') })); + +process.env.TASK_TABLE_NAME = 'Tasks'; +process.env.TASK_EVENTS_TABLE_NAME = 'Events'; +process.env.ATTACHMENTS_BUCKET_NAME = 'attachments-bucket'; +process.env.USER_CONCURRENCY_TABLE_NAME = 'Concurrency'; +process.env.MAX_CONCURRENT_TASKS_PER_USER = '3'; +process.env.ORCHESTRATOR_FUNCTION_ARN = 'arn:aws:lambda:us-east-1:123:function:orchestrator'; +process.env.GUARDRAIL_ID = 'guardrail-1'; +process.env.GUARDRAIL_VERSION = '1'; + +import { handler } from '../../src/handlers/confirm-uploads'; + +function makeEvent(taskId: string, userId = 'user-1'): APIGatewayProxyEvent { + return { + pathParameters: { task_id: taskId }, + requestContext: { + authorizer: { claims: { sub: userId } }, + }, + headers: {}, + body: null, + } as unknown as APIGatewayProxyEvent; +} + +function makeContext(remainingMs: number): Context { + return { + getRemainingTimeInMillis: jest.fn(() => remainingMs), + functionName: 'confirm-uploads', + functionVersion: '1', + invokedFunctionArn: 'arn:aws:lambda:us-east-1:123:function:confirm-uploads', + memoryLimitInMB: '2048', + awsRequestId: 'req-1', + logGroupName: '/aws/lambda/confirm-uploads', + logStreamName: '2025/01/01/[1]abc', + callbackWaitsForEmptyEventLoop: true, + done: jest.fn(), + fail: jest.fn(), + succeed: jest.fn(), + }; +} + +const PENDING_TASK = { + task_id: 'task-1', + user_id: 'user-1', + status: 'PENDING_UPLOADS', + repo: 'org/repo', + branch_name: 'bgagent/task-1/fix', + channel_source: 'api', + status_created_at: 'PENDING_UPLOADS#2025-03-15T10:30:00Z', + created_at: '2025-03-15T10:30:00Z', + updated_at: '2025-03-15T10:30:00Z', + attachments: [ + { + attachment_id: 'att-1', + type: 'image', + content_type: 'image/png', + filename: 'screenshot.png', + s3_key: 'attachments/user-1/task-1/att-1/screenshot.png', + size_bytes: 1024, + screening: { status: 'pending' }, + }, + { + attachment_id: 'att-2', + type: 'file', + content_type: 'text/plain', + filename: 'notes.txt', + s3_key: 'attachments/user-1/task-1/att-2/notes.txt', + size_bytes: 512, + screening: { status: 'pending' }, + }, + ], +}; + +describe('confirm-uploads handler', () => { + beforeEach(() => { + ddbSend.mockReset(); + s3Send.mockReset(); + lambdaSend.mockReset(); + }); + + test('returns 401 when no auth', async () => { + const event = { pathParameters: { task_id: 'task-1' }, requestContext: { authorizer: {} }, headers: {}, body: null } as unknown as APIGatewayProxyEvent; + const result = await handler(event, makeContext(180_000)); + expect(result.statusCode).toBe(401); + }); + + test('returns 404 when task not found', async () => { + ddbSend.mockResolvedValueOnce({ Item: undefined }); + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(404); + }); + + test('returns 404 when caller does not own the task', async () => { + ddbSend.mockResolvedValueOnce({ Item: { ...PENDING_TASK, user_id: 'other-user' } }); + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(404); + }); + + test('returns 200 idempotently when task is already SUBMITTED', async () => { + ddbSend.mockResolvedValueOnce({ Item: { ...PENDING_TASK, status: 'SUBMITTED' } }); + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(200); + }); + + test('returns 409 when task is in a terminal status', async () => { + ddbSend.mockResolvedValueOnce({ Item: { ...PENDING_TASK, status: 'CANCELLED' } }); + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(409); + }); + + test('returns 503 with Retry-After when deadline is exceeded before screening starts', async () => { + ddbSend.mockResolvedValueOnce({ Item: PENDING_TASK }); + // HeadObject succeeds for both attachments + s3Send + .mockResolvedValueOnce({ VersionId: 'v1', ContentLength: 1024 }) + .mockResolvedValueOnce({ VersionId: 'v2', ContentLength: 512 }); + + // Context reports only 10s remaining (below the 15s deadline margin) + const context = makeContext(10_000); + const result = await handler(makeEvent('task-1'), context); + + expect(result.statusCode).toBe(503); + expect(result.headers?.['Retry-After']).toBe('30'); + const body = JSON.parse(result.body); + expect(body.error.code).toBe('SCREENING_DEADLINE_EXCEEDED'); + expect(body.error.message).toContain('did not complete within the time limit'); + }); + + test('returns 400 when uploads are missing (HeadObject 404)', async () => { + ddbSend.mockResolvedValueOnce({ Item: PENDING_TASK }); + // HeadObject returns 404 for first attachment after all retries + s3Send.mockRejectedValue({ $metadata: { httpStatusCode: 404 } }); + + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error.code).toBe('ATTACHMENT_UPLOAD_MISSING'); + }); +}); diff --git a/cdk/test/handlers/linear-webhook-processor.test.ts b/cdk/test/handlers/linear-webhook-processor.test.ts index cbc48ff2..5ad948e1 100644 --- a/cdk/test/handlers/linear-webhook-processor.test.ts +++ b/cdk/test/handlers/linear-webhook-processor.test.ts @@ -362,4 +362,78 @@ describe('linear-webhook-processor handler', () => { expect(reportIssueFailureMock).toHaveBeenCalledTimes(1); }); }); + + // ─── Image URL extraction from issue description ───────────────────────────── + + describe('image URL attachment extraction', () => { + beforeEach(() => { + ddbSend + .mockResolvedValueOnce({ Item: { repo: 'org/repo', status: 'active' } }) + .mockResolvedValueOnce({ Item: { platform_user_id: 'cognito-user-1', status: 'active' } }); + createTaskCoreMock.mockResolvedValueOnce({ statusCode: 201, body: JSON.stringify({ data: { task_id: 'T1' } }) }); + }); + + test('extracts markdown image URLs from issue description', async () => { + const payload = issue(); + const data = payload.data as Record; + data.description = 'See this bug:\n\n![screenshot](https://linear.app/uploads/img1.png)\n\nAnd also ![diagram](https://linear.app/uploads/arch.png)'; + + await handler(eventWith(payload)); + + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [reqBody] = createTaskCoreMock.mock.calls[0]; + expect(reqBody.attachments).toHaveLength(2); + expect(reqBody.attachments[0]).toEqual({ type: 'url', url: 'https://linear.app/uploads/img1.png' }); + expect(reqBody.attachments[1]).toEqual({ type: 'url', url: 'https://linear.app/uploads/arch.png' }); + }); + + test('does not extract HTTP (non-HTTPS) URLs', async () => { + const payload = issue(); + const data = payload.data as Record; + data.description = '![unsafe](http://evil.com/img.png)'; + + await handler(eventWith(payload)); + + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [reqBody] = createTaskCoreMock.mock.calls[0]; + expect(reqBody.attachments).toBeUndefined(); + }); + + test('caps image extraction at 10 URLs', async () => { + const payload = issue(); + const data = payload.data as Record; + const lines = Array.from({ length: 15 }, (_, i) => `![img${i}](https://cdn.linear.app/img${i}.png)`); + data.description = lines.join('\n'); + + await handler(eventWith(payload)); + + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [reqBody] = createTaskCoreMock.mock.calls[0]; + expect(reqBody.attachments).toHaveLength(10); + }); + + test('no attachments when description has no images', async () => { + const payload = issue(); + const data = payload.data as Record; + data.description = 'Just text, no images here.'; + + await handler(eventWith(payload)); + + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [reqBody] = createTaskCoreMock.mock.calls[0]; + expect(reqBody.attachments).toBeUndefined(); + }); + + test('no attachments when description is undefined', async () => { + const payload = issue(); + const data = payload.data as Record; + delete data.description; + + await handler(eventWith(payload)); + + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [reqBody] = createTaskCoreMock.mock.calls[0]; + expect(reqBody.attachments).toBeUndefined(); + }); + }); }); diff --git a/cdk/test/handlers/slack-command-processor.test.ts b/cdk/test/handlers/slack-command-processor.test.ts index bed9a65e..f5d299da 100644 --- a/cdk/test/handlers/slack-command-processor.test.ts +++ b/cdk/test/handlers/slack-command-processor.test.ts @@ -227,4 +227,114 @@ describe('slack-command-processor handler', () => { ); expect(posted).toBeTruthy(); }); + + // ─── Slack file attachment extraction ──────────────────────────────────────── + + describe('file attachments', () => { + beforeEach(() => { + // Standard setup: linked user, public channel + ddbSend.mockResolvedValueOnce({ Item: { status: 'active', platform_user_id: 'cognito-1' } }); + ddbSend.mockResolvedValue({ Item: { status: 'active' } }); + fetchMock.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ ok: true, channel: { is_private: false, is_member: true } }), + }); + }); + + test('downloads Slack files and passes as inline attachments', async () => { + // fetch for file download + fetchMock.mockResolvedValueOnce({ + ok: true, + arrayBuffer: () => Promise.resolve(Buffer.from('file content')), + }); + createTaskCoreMock.mockResolvedValueOnce({ + statusCode: 201, + body: JSON.stringify({ data: { task_id: 'T1' } }), + }); + + await handler(mention({ + text: 'submit org/repo fix the bug', + files: [{ + id: 'F1', + name: 'screenshot.png', + mimetype: 'image/png', + size: 1024, + url_private_download: 'https://files.slack.com/files/F1/screenshot.png', + }], + })); + + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [reqBody] = createTaskCoreMock.mock.calls[0]; + expect(reqBody.attachments).toHaveLength(1); + expect(reqBody.attachments[0].type).toBe('image'); + expect(reqBody.attachments[0].content_type).toBe('image/png'); + expect(reqBody.attachments[0].filename).toBe('screenshot.png'); + expect(reqBody.attachments[0].data).toBe(Buffer.from('file content').toString('base64')); + }); + + test('rejects files with unsupported MIME types', async () => { + await handler(mention({ + text: 'submit org/repo fix', + files: [{ + id: 'F1', + name: 'virus.exe', + mimetype: 'application/x-executable', + size: 1024, + url_private_download: 'https://files.slack.com/files/F1/virus.exe', + }], + })); + + expect(createTaskCoreMock).not.toHaveBeenCalled(); + const reply = fetchMock.mock.calls.find( + ([url, opts]) => String(url).includes('chat.postMessage') && String((opts as { body: string }).body).includes('unsupported type'), + ); + expect(reply).toBeTruthy(); + }); + + test('rejects files exceeding 10 MB size limit', async () => { + await handler(mention({ + text: 'submit org/repo fix', + files: [{ + id: 'F1', + name: 'huge.png', + mimetype: 'image/png', + size: 11 * 1024 * 1024, // 11 MB + url_private_download: 'https://files.slack.com/files/F1/huge.png', + }], + })); + + expect(createTaskCoreMock).not.toHaveBeenCalled(); + const reply = fetchMock.mock.calls.find( + ([url, opts]) => String(url).includes('chat.postMessage') && String((opts as { body: string }).body).includes('too large'), + ); + expect(reply).toBeTruthy(); + }); + + test('reports all file errors atomically', async () => { + await handler(mention({ + text: 'submit org/repo fix', + files: [ + { id: 'F1', name: 'big.png', mimetype: 'image/png', size: 11 * 1024 * 1024, url_private_download: 'https://files.slack.com/files/F1/big.png' }, + { id: 'F2', name: 'bad.exe', mimetype: 'application/x-executable', size: 100, url_private_download: 'https://files.slack.com/files/F2/bad.exe' }, + ], + })); + + expect(createTaskCoreMock).not.toHaveBeenCalled(); + const reply = fetchMock.mock.calls.find( + ([url, opts]) => String(url).includes('chat.postMessage') && String((opts as { body: string }).body).includes('2 attachment errors'), + ); + expect(reply).toBeTruthy(); + }); + + test('proceeds without attachments when no files are present', async () => { + createTaskCoreMock.mockResolvedValueOnce({ + statusCode: 201, + body: JSON.stringify({ data: { task_id: 'T1' } }), + }); + await handler(mention({ text: 'submit org/repo fix' })); + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [reqBody] = createTaskCoreMock.mock.calls[0]; + expect(reqBody.attachments).toBeUndefined(); + }); + }); }); diff --git a/cli/src/api-client.ts b/cli/src/api-client.ts index a3f58cb4..89d37a73 100644 --- a/cli/src/api-client.ts +++ b/cli/src/api-client.ts @@ -27,6 +27,7 @@ import { ApprovalScope, CancelTaskResponse, CreateTaskRequest, + CreateTaskResponse, CreateWebhookRequest, CreateWebhookResponse, DenyRequest, @@ -137,12 +138,18 @@ export class ApiClient { } /** POST /tasks — create a new task. */ - async createTask(req: CreateTaskRequest, idempotencyKey?: string): Promise { + async createTask(req: CreateTaskRequest, idempotencyKey?: string): Promise { const headers: Record = {}; if (idempotencyKey) { headers['Idempotency-Key'] = idempotencyKey; } - const res = await this.request>('POST', '/tasks', req, headers); + const res = await this.request>('POST', '/tasks', req, headers); + return res.data; + } + + /** POST /tasks/{task_id}/confirm-uploads — confirm presigned uploads. */ + async confirmUploads(taskId: string): Promise { + const res = await this.request>('POST', `/tasks/${taskId}/confirm-uploads`); return res.data; } diff --git a/cli/src/commands/submit.ts b/cli/src/commands/submit.ts index 2346b178..23030a72 100644 --- a/cli/src/commands/submit.ts +++ b/cli/src/commands/submit.ts @@ -17,6 +17,8 @@ * SOFTWARE. */ +import * as fs from 'fs'; +import * as path from 'path'; import { Command } from 'commander'; import { ApiClient } from '../api-client'; import { CliError } from '../errors'; @@ -25,6 +27,9 @@ import { APPROVAL_TIMEOUT_S_MAX, APPROVAL_TIMEOUT_S_MIN, ApprovalScope, + Attachment, + AttachmentType, + AttachmentUploadInstruction, CreateTaskRequest, INITIAL_APPROVALS_MAX_ENTRIES, INITIAL_APPROVALS_MAX_ENTRY_LENGTH, @@ -63,6 +68,12 @@ export function makeSubmitCommand(): Command { .option('--review-pr ', 'PR number to review (sets task_type to pr_review)', parseInt) .option('--idempotency-key ', 'Idempotency key for deduplication') .option('--trace', 'Capture 4 KB debug previews (design §10.1). Opt-in per task; not routine observability.') + .option( + '--attachment ', + 'Attach a local file or URL (repeatable). Local files ≤ 500 KB are sent inline; URLs are fetched by the agent.', + collect, + [] as readonly string[], + ) .option( '--approval-timeout ', `Cedar HITL per-task default approval timeout (${APPROVAL_TIMEOUT_S_MIN}-${APPROVAL_TIMEOUT_S_MAX}s). ` @@ -146,6 +157,18 @@ export function makeSubmitCommand(): Command { initialApprovals = preApproveRaw as readonly ApprovalScope[]; } + // Resolve --attachment arguments into API attachment objects + const attachmentArgs = (opts.attachment ?? []) as readonly string[]; + const attachments: Attachment[] = []; + if (attachmentArgs.length > 0) { + if (attachmentArgs.length > 10) { + throw new CliError('Maximum 10 attachments per task.'); + } + for (const arg of attachmentArgs) { + attachments.push(resolveAttachmentArg(arg)); + } + } + const client = new ApiClient(); const body: CreateTaskRequest = { repo: opts.repo, @@ -159,9 +182,34 @@ export function makeSubmitCommand(): Command { ...(opts.trace && { trace: true }), ...(opts.approvalTimeout !== undefined && { approval_timeout_s: opts.approvalTimeout }), ...(initialApprovals !== undefined && { initial_approvals: initialApprovals }), + ...(attachments.length > 0 && { attachments }), }; - const task = await client.createTask(body, opts.idempotencyKey); + const createResponse = await client.createTask(body, opts.idempotencyKey); + + // If presigned uploads are needed, upload files and confirm + let task = createResponse; + if (createResponse.upload_instructions && createResponse.upload_instructions.length > 0) { + process.stderr.write(`Uploading ${createResponse.upload_instructions.length} attachment(s)...\n`); + for (const instruction of createResponse.upload_instructions) { + const localAtt = attachments.find(a => a.filename === instruction.filename); + if (!localAtt || !localAtt.filename) { + throw new CliError(`No local file found for upload instruction: ${instruction.filename}`); + } + const filePath = attachmentArgs.find(arg => + !arg.startsWith('http') && path.basename(path.resolve(arg)) === instruction.filename, + ); + if (!filePath) { + throw new CliError(`Cannot locate local file for presigned upload: ${instruction.filename}`); + } + await uploadViaPresignedPost(path.resolve(filePath), instruction); + process.stderr.write(` Uploaded: ${instruction.filename}\n`); + } + + // Confirm uploads to trigger screening and transition to SUBMITTED + process.stderr.write('Confirming uploads...\n'); + task = await client.confirmUploads(createResponse.task_id); + } if (opts.wait) { process.stderr.write('\n'); @@ -174,3 +222,135 @@ export function makeSubmitCommand(): Command { } }); } + +// --------------------------------------------------------------------------- +// Attachment resolution helpers +// --------------------------------------------------------------------------- + +const MAX_INLINE_SIZE_BYTES = 500 * 1024; // 500 KB + +/** MIME type lookup by file extension. */ +const MIME_BY_EXT: Record = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.txt': 'text/plain', + '.log': 'text/x-log', + '.csv': 'text/csv', + '.md': 'text/markdown', + '.json': 'application/json', + '.pdf': 'application/pdf', +}; + +const IMAGE_MIMES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']); + +/** + * Resolve a CLI --attachment argument to an Attachment object. + * Handles URLs (https://...) and local file paths. + */ +function resolveAttachmentArg(arg: string): Attachment { + // URL detection: starts with https:// + // Omit content_type for URLs — the server-side resolver determines the + // actual content type from the HTTP response's Content-Type header. + if (arg.startsWith('https://')) { + return { type: 'url', url: arg }; + } + + if (arg.startsWith('http://')) { + throw new CliError(`URL attachments must use HTTPS: ${arg}`); + } + + // Local file + const resolvedPath = path.resolve(arg); + if (!fs.existsSync(resolvedPath)) { + throw new CliError(`Attachment file not found: ${arg}`); + } + + const stat = fs.statSync(resolvedPath); + if (!stat.isFile()) { + throw new CliError(`Attachment path is not a file: ${arg}`); + } + + const ext = path.extname(resolvedPath).toLowerCase(); + const contentType = MIME_BY_EXT[ext]; + if (!contentType) { + throw new CliError( + `Unsupported file type '${ext}' for attachment: ${arg}. ` + + `Supported: ${Object.keys(MIME_BY_EXT).join(', ')}`, + ); + } + + const type: AttachmentType = IMAGE_MIMES.has(contentType) ? 'image' : 'file'; + + if (stat.size > MAX_INLINE_SIZE_BYTES) { + // Large file: use presigned upload path (metadata only, no data) + return { + type, + content_type: contentType, + filename: path.basename(resolvedPath), + expected_size_bytes: stat.size, + }; + } + + const data = fs.readFileSync(resolvedPath); + + return { + type, + content_type: contentType, + filename: path.basename(resolvedPath), + data: data.toString('base64'), + }; +} + +// --------------------------------------------------------------------------- +// Presigned POST upload helper +// --------------------------------------------------------------------------- + +/** + * Upload a local file to S3 via a presigned POST (multipart/form-data). + * The presigned fields (policy, signature, etc.) are included as form fields + * before the file content, as required by S3's POST Object API. + */ +async function uploadViaPresignedPost( + filePath: string, + instruction: AttachmentUploadInstruction, +): Promise { + const fileData = fs.readFileSync(filePath); + const boundary = `----AttachmentBoundary${Date.now()}${Math.random().toString(36).slice(2)}`; + + // Build multipart/form-data body: presigned fields first, then the file. + const parts: Buffer[] = []; + + for (const [key, value] of Object.entries(instruction.upload_fields)) { + parts.push(Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`, + )); + } + + // The file field must be last per S3 POST Object requirements. + const ext = path.extname(filePath).toLowerCase(); + const contentType = MIME_BY_EXT[ext] ?? 'application/octet-stream'; + parts.push(Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${path.basename(filePath)}"\r\n` + + `Content-Type: ${contentType}\r\n\r\n`, + )); + parts.push(fileData); + parts.push(Buffer.from(`\r\n--${boundary}--\r\n`)); + + const body = Buffer.concat(parts); + + const res = await fetch(instruction.upload_url, { + method: 'POST', + headers: { 'Content-Type': `multipart/form-data; boundary=${boundary}` }, + body, + }); + + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new CliError( + `Presigned upload failed for ${instruction.filename}: HTTP ${res.status}${text ? ` — ${text.slice(0, 200)}` : ''}`, + ); + } +} diff --git a/cli/src/types.ts b/cli/src/types.ts index 5450711b..4dcb57bb 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -30,6 +30,7 @@ export type AttachmentType = 'image' | 'file' | 'url'; * surface stays portable. */ export type TaskStatusType = + | 'PENDING_UPLOADS' | 'SUBMITTED' | 'HYDRATING' | 'RUNNING' @@ -212,6 +213,12 @@ export interface AttachmentUploadInstruction { readonly upload_expires_at: string; } +/** Response from POST /v1/tasks when presigned uploads are required. */ +export interface CreateTaskResponse extends TaskDetail { + readonly upload_instructions?: readonly AttachmentUploadInstruction[]; + readonly task_expires_at?: string; +} + /** Create task request body for POST /v1/tasks. */ export interface CreateTaskRequest { readonly repo: string; diff --git a/docs/design/ATTACHMENTS.md b/docs/design/ATTACHMENTS.md index cf744f8c..6ee5dcd2 100644 --- a/docs/design/ATTACHMENTS.md +++ b/docs/design/ATTACHMENTS.md @@ -1293,7 +1293,7 @@ export const VALID_TRANSITIONS: Record = { ```typescript export const ACTIVE_STATUSES: TaskStatusType[] = [ // PENDING_UPLOADS is NOT here — does not count against concurrency - 'SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING', + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', ]; export const PRE_ACTIVE_STATUSES: TaskStatusType[] = [ @@ -1316,11 +1316,16 @@ stateDiagram-v2 PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel SUBMITTED --> HYDRATING : Admission passes HYDRATING --> RUNNING : Context assembled + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate + AWAITING_APPROVAL --> RUNNING : Approved / denied (resume) RUNNING --> FINALIZING : Session ends FINALIZING --> COMPLETED : Success FINALIZING --> FAILED : Agent failure ``` +**Relationship to AWAITING_APPROVAL:** `PENDING_UPLOADS` and `AWAITING_APPROVAL` (Cedar HITL) are independent lifecycle stages with no transition path between them. `PENDING_UPLOADS` is pre-pipeline (no compute allocated, no concurrency slot consumed). `AWAITING_APPROVAL` is mid-pipeline (container alive, concurrency slot held, paused on a human decision). A task with presigned attachments may later hit an approval gate: `PENDING_UPLOADS → SUBMITTED → ... → RUNNING → AWAITING_APPROVAL → RUNNING → ...`. The agent's IAM-based attachment downloads are unaffected by approval wait time (no presigned URL expiry). + ### Auto-cancel mechanism Tasks in `PENDING_UPLOADS` for > 30 minutes are auto-cancelled via an **EventBridge scheduled rule** (not DynamoDB TTL — TTL would delete the record, losing audit trail). The rule: diff --git a/docs/design/ORCHESTRATOR.md b/docs/design/ORCHESTRATOR.md index 920a67c0..6f8089df 100644 --- a/docs/design/ORCHESTRATOR.md +++ b/docs/design/ORCHESTRATOR.md @@ -52,9 +52,11 @@ Every task moves through a finite set of states from creation to a terminal outc | State | Description | Duration | |---|---|---| +| `PENDING_UPLOADS` | Presigned-upload task awaiting client file uploads | Minutes (30-min auto-cancel) | | `SUBMITTED` | Task accepted, awaiting orchestration | Milliseconds | | `HYDRATING` | Fetching GitHub data, querying memory, assembling prompt | Seconds | | `RUNNING` | Agent session active in compute environment | Minutes to hours | +| `AWAITING_APPROVAL` | Cedar HITL soft-deny gate fired; paused on human decision | Minutes to hours | | `FINALIZING` | Result inference and cleanup in progress | Seconds | | `COMPLETED` | Terminal. Task finished successfully | - | | `FAILED` | Terminal. Task could not complete | - | @@ -65,20 +67,31 @@ Every task moves through a finite set of states from creation to a terminal outc ```mermaid stateDiagram-v2 - [*] --> SUBMITTED + [*] --> PENDING_UPLOADS : Presigned upload task + [*] --> SUBMITTED : Inline/no-attachment task + PENDING_UPLOADS --> SUBMITTED : confirm-uploads succeeds + PENDING_UPLOADS --> FAILED : Screening blocked + PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel + SUBMITTED --> HYDRATING : Admission passes SUBMITTED --> FAILED : Admission rejected SUBMITTED --> CANCELLED : User cancels HYDRATING --> RUNNING : Session started + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate HYDRATING --> FAILED : Hydration error HYDRATING --> CANCELLED : User cancels + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate RUNNING --> FINALIZING : Session ends RUNNING --> CANCELLED : User cancels RUNNING --> TIMED_OUT : Duration exceeded RUNNING --> FAILED : Session crash + AWAITING_APPROVAL --> RUNNING : Approved or denied (resume) + AWAITING_APPROVAL --> CANCELLED : User cancels mid-approval + AWAITING_APPROVAL --> FAILED : Stranded-approval reconciler + FINALIZING --> COMPLETED : PR or commits found FINALIZING --> FAILED : No useful work FINALIZING --> TIMED_OUT : Idle timeout detected @@ -88,13 +101,21 @@ stateDiagram-v2 | From | To | Trigger | Condition | |---|---|---|---| +| `PENDING_UPLOADS` | `SUBMITTED` | `confirm-uploads` succeeds | All attachments screened and passed | +| `PENDING_UPLOADS` | `FAILED` | Screening blocked | Any attachment fails security screening | +| `PENDING_UPLOADS` | `CANCELLED` | User cancels or auto-cancel | Upload window expired (30 min) or explicit cancel | | `SUBMITTED` | `HYDRATING` | Admission passes | Concurrency slot acquired | | `SUBMITTED` | `FAILED` | Admission rejected | Repo not onboarded, rate/concurrency limit, validation error | | `HYDRATING` | `RUNNING` | Hydration complete | `invoke_agent_runtime` returns session ID | +| `HYDRATING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during hydration | | `HYDRATING` | `FAILED` | Hydration error | GitHub API failure, guardrail blocks content, Bedrock unavailable | +| `RUNNING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during execution | | `RUNNING` | `FINALIZING` | Session ends | Response received or session terminated | | `RUNNING` | `TIMED_OUT` | Max duration exceeded | Wall-clock timer (default 8h, matching AgentCore max) | | `RUNNING` | `FAILED` | Session crash | Heartbeat lost (see Liveness monitoring) | +| `AWAITING_APPROVAL` | `RUNNING` | Approved or denied | Human decision received; agent resumes | +| `AWAITING_APPROVAL` | `CANCELLED` | User cancels | Explicit cancel while awaiting approval | +| `AWAITING_APPROVAL` | `FAILED` | Stranded reconciler | Approval request orphaned (agent died mid-wait) | | `FINALIZING` | `COMPLETED` | Success inferred | PR exists or commits on branch | | `FINALIZING` | `FAILED` | Failure inferred | No commits, no PR, or agent reported error | @@ -104,9 +125,11 @@ Users can cancel a task at any point. The orchestrator's response depends on how | State when cancel arrives | Action | |---|---| +| `PENDING_UPLOADS` | Transition to `CANCELLED`. Clean up S3 objects under the task's attachment prefix. No concurrency slot to release. | | `SUBMITTED` | Transition to `CANCELLED`. No cleanup needed. | | `HYDRATING` | Abort hydration, release concurrency slot, transition to `CANCELLED`. | | `RUNNING` | Call `stop_runtime_session`, wait for confirmation, release concurrency, transition to `CANCELLED`. Partial work on GitHub remains for the user to inspect. | +| `AWAITING_APPROVAL` | Call `stop_runtime_session`, release concurrency slot, transition to `CANCELLED`. The pending approval row transitions to `STRANDED`. | | `FINALIZING` | Let finalization complete. Mark `CANCELLED` only if the terminal state was not yet written. | | Terminal | Reject the cancel request. | diff --git a/docs/src/content/docs/architecture/Attachments.md b/docs/src/content/docs/architecture/Attachments.md index 9c0c651a..de2ecba9 100644 --- a/docs/src/content/docs/architecture/Attachments.md +++ b/docs/src/content/docs/architecture/Attachments.md @@ -1297,7 +1297,7 @@ export const VALID_TRANSITIONS: Record = { ```typescript export const ACTIVE_STATUSES: TaskStatusType[] = [ // PENDING_UPLOADS is NOT here — does not count against concurrency - 'SUBMITTED', 'HYDRATING', 'RUNNING', 'FINALIZING', + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', ]; export const PRE_ACTIVE_STATUSES: TaskStatusType[] = [ @@ -1320,11 +1320,16 @@ stateDiagram-v2 PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel SUBMITTED --> HYDRATING : Admission passes HYDRATING --> RUNNING : Context assembled + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate + AWAITING_APPROVAL --> RUNNING : Approved / denied (resume) RUNNING --> FINALIZING : Session ends FINALIZING --> COMPLETED : Success FINALIZING --> FAILED : Agent failure ``` +**Relationship to AWAITING_APPROVAL:** `PENDING_UPLOADS` and `AWAITING_APPROVAL` (Cedar HITL) are independent lifecycle stages with no transition path between them. `PENDING_UPLOADS` is pre-pipeline (no compute allocated, no concurrency slot consumed). `AWAITING_APPROVAL` is mid-pipeline (container alive, concurrency slot held, paused on a human decision). A task with presigned attachments may later hit an approval gate: `PENDING_UPLOADS → SUBMITTED → ... → RUNNING → AWAITING_APPROVAL → RUNNING → ...`. The agent's IAM-based attachment downloads are unaffected by approval wait time (no presigned URL expiry). + ### Auto-cancel mechanism Tasks in `PENDING_UPLOADS` for > 30 minutes are auto-cancelled via an **EventBridge scheduled rule** (not DynamoDB TTL — TTL would delete the record, losing audit trail). The rule: diff --git a/docs/src/content/docs/architecture/Orchestrator.md b/docs/src/content/docs/architecture/Orchestrator.md index 77b96b6b..15d931ba 100644 --- a/docs/src/content/docs/architecture/Orchestrator.md +++ b/docs/src/content/docs/architecture/Orchestrator.md @@ -56,9 +56,11 @@ Every task moves through a finite set of states from creation to a terminal outc | State | Description | Duration | |---|---|---| +| `PENDING_UPLOADS` | Presigned-upload task awaiting client file uploads | Minutes (30-min auto-cancel) | | `SUBMITTED` | Task accepted, awaiting orchestration | Milliseconds | | `HYDRATING` | Fetching GitHub data, querying memory, assembling prompt | Seconds | | `RUNNING` | Agent session active in compute environment | Minutes to hours | +| `AWAITING_APPROVAL` | Cedar HITL soft-deny gate fired; paused on human decision | Minutes to hours | | `FINALIZING` | Result inference and cleanup in progress | Seconds | | `COMPLETED` | Terminal. Task finished successfully | - | | `FAILED` | Terminal. Task could not complete | - | @@ -69,20 +71,31 @@ Every task moves through a finite set of states from creation to a terminal outc ```mermaid stateDiagram-v2 - [*] --> SUBMITTED + [*] --> PENDING_UPLOADS : Presigned upload task + [*] --> SUBMITTED : Inline/no-attachment task + PENDING_UPLOADS --> SUBMITTED : confirm-uploads succeeds + PENDING_UPLOADS --> FAILED : Screening blocked + PENDING_UPLOADS --> CANCELLED : User cancels or 30-min auto-cancel + SUBMITTED --> HYDRATING : Admission passes SUBMITTED --> FAILED : Admission rejected SUBMITTED --> CANCELLED : User cancels HYDRATING --> RUNNING : Session started + HYDRATING --> AWAITING_APPROVAL : Cedar soft-deny gate HYDRATING --> FAILED : Hydration error HYDRATING --> CANCELLED : User cancels + RUNNING --> AWAITING_APPROVAL : Cedar soft-deny gate RUNNING --> FINALIZING : Session ends RUNNING --> CANCELLED : User cancels RUNNING --> TIMED_OUT : Duration exceeded RUNNING --> FAILED : Session crash + AWAITING_APPROVAL --> RUNNING : Approved or denied (resume) + AWAITING_APPROVAL --> CANCELLED : User cancels mid-approval + AWAITING_APPROVAL --> FAILED : Stranded-approval reconciler + FINALIZING --> COMPLETED : PR or commits found FINALIZING --> FAILED : No useful work FINALIZING --> TIMED_OUT : Idle timeout detected @@ -92,13 +105,21 @@ stateDiagram-v2 | From | To | Trigger | Condition | |---|---|---|---| +| `PENDING_UPLOADS` | `SUBMITTED` | `confirm-uploads` succeeds | All attachments screened and passed | +| `PENDING_UPLOADS` | `FAILED` | Screening blocked | Any attachment fails security screening | +| `PENDING_UPLOADS` | `CANCELLED` | User cancels or auto-cancel | Upload window expired (30 min) or explicit cancel | | `SUBMITTED` | `HYDRATING` | Admission passes | Concurrency slot acquired | | `SUBMITTED` | `FAILED` | Admission rejected | Repo not onboarded, rate/concurrency limit, validation error | | `HYDRATING` | `RUNNING` | Hydration complete | `invoke_agent_runtime` returns session ID | +| `HYDRATING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during hydration | | `HYDRATING` | `FAILED` | Hydration error | GitHub API failure, guardrail blocks content, Bedrock unavailable | +| `RUNNING` | `AWAITING_APPROVAL` | Cedar soft-deny gate fires | Tool call triggers a soft-deny policy rule during execution | | `RUNNING` | `FINALIZING` | Session ends | Response received or session terminated | | `RUNNING` | `TIMED_OUT` | Max duration exceeded | Wall-clock timer (default 8h, matching AgentCore max) | | `RUNNING` | `FAILED` | Session crash | Heartbeat lost (see Liveness monitoring) | +| `AWAITING_APPROVAL` | `RUNNING` | Approved or denied | Human decision received; agent resumes | +| `AWAITING_APPROVAL` | `CANCELLED` | User cancels | Explicit cancel while awaiting approval | +| `AWAITING_APPROVAL` | `FAILED` | Stranded reconciler | Approval request orphaned (agent died mid-wait) | | `FINALIZING` | `COMPLETED` | Success inferred | PR exists or commits on branch | | `FINALIZING` | `FAILED` | Failure inferred | No commits, no PR, or agent reported error | @@ -108,9 +129,11 @@ Users can cancel a task at any point. The orchestrator's response depends on how | State when cancel arrives | Action | |---|---| +| `PENDING_UPLOADS` | Transition to `CANCELLED`. Clean up S3 objects under the task's attachment prefix. No concurrency slot to release. | | `SUBMITTED` | Transition to `CANCELLED`. No cleanup needed. | | `HYDRATING` | Abort hydration, release concurrency slot, transition to `CANCELLED`. | | `RUNNING` | Call `stop_runtime_session`, wait for confirmation, release concurrency, transition to `CANCELLED`. Partial work on GitHub remains for the user to inspect. | +| `AWAITING_APPROVAL` | Call `stop_runtime_session`, release concurrency slot, transition to `CANCELLED`. The pending approval row transitions to `STRANDED`. | | `FINALIZING` | Let finalization complete. Mark `CANCELLED` only if the terminal state was not yet written. | | Terminal | Reject the cancel request. | diff --git a/yarn.lock b/yarn.lock index 814102fe..f3085af5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -673,7 +673,7 @@ "@smithy/util-waiter" "^4.2.14" tslib "^2.6.2" -"@aws-sdk/client-s3@^3.1021.0": +"@aws-sdk/client-s3@3.1040.0", "@aws-sdk/client-s3@^3.1021.0": version "3.1040.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.1040.0.tgz#96fa3975b815e6cd6d0855a7bd4a72adf7dc1016" integrity sha512-Ldfby1xDrlZwNY2NxP9pwdVrf8sqHbGBKP1UkoG/oWcePGlGhjY8iVwy8hRy9f1EQfHVFWIFunwHaPQxhYTnWQ== @@ -1904,6 +1904,21 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/s3-presigned-post@3.1040.0": + version "3.1040.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/s3-presigned-post/-/s3-presigned-post-3.1040.0.tgz#3b03fb5ffa5b5476780ffd04337cdb938af20445" + integrity sha512-Ycnktl5kAqgM7ottZIujR+/MsmmZ/iiaKobQXjtZhy/QO9nwnChxHJuOruuuhI5+f/NMbhb0G8aeUexK/D6iQA== + dependencies: + "@aws-sdk/client-s3" "3.1040.0" + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/util-format-url" "^3.972.10" + "@smithy/middleware-endpoint" "^4.4.32" + "@smithy/signature-v4" "^5.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-hex-encoding" "^4.2.2" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + "@aws-sdk/s3-request-presigner@^3.1021.0": version "3.1040.0" resolved "https://registry.yarnpkg.com/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1040.0.tgz#c81d96024325436dd4ff0f412ca5c22d2674a6e6" @@ -2618,6 +2633,13 @@ "@emnapi/wasi-threads" "1.2.0" tslib "^2.4.0" +"@emnapi/runtime@^1.2.0": + version "1.10.0" + resolved "https://registry.yarnpkg.com/@emnapi/runtime/-/runtime-1.10.0.tgz#4b260c0d3534204e98c6110b8db1a987d26ec87c" + integrity sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA== + dependencies: + tslib "^2.4.0" + "@emnapi/runtime@^1.4.3": version "1.9.1" resolved "https://registry.yarnpkg.com/@emnapi/runtime/-/runtime-1.9.1.tgz#115ff2a0d589865be6bd8e9d701e499c473f2a8d" @@ -2918,6 +2940,13 @@ resolved "https://registry.yarnpkg.com/@img/colour/-/colour-1.1.0.tgz#b0c2c2fa661adf75effd6b4964497cd80010bb9d" integrity sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ== +"@img/sharp-darwin-arm64@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz#ef5b5a07862805f1e8145a377c8ba6e98813ca08" + integrity sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ== + optionalDependencies: + "@img/sharp-libvips-darwin-arm64" "1.0.4" + "@img/sharp-darwin-arm64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz#6e0732dcade126b6670af7aa17060b926835ea86" @@ -2925,6 +2954,13 @@ optionalDependencies: "@img/sharp-libvips-darwin-arm64" "1.2.4" +"@img/sharp-darwin-x64@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz#e03d3451cd9e664faa72948cc70a403ea4063d61" + integrity sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q== + optionalDependencies: + "@img/sharp-libvips-darwin-x64" "1.0.4" + "@img/sharp-darwin-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz#19bc1dd6eba6d5a96283498b9c9f401180ee9c7b" @@ -2932,21 +2968,41 @@ optionalDependencies: "@img/sharp-libvips-darwin-x64" "1.2.4" +"@img/sharp-libvips-darwin-arm64@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz#447c5026700c01a993c7804eb8af5f6e9868c07f" + integrity sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg== + "@img/sharp-libvips-darwin-arm64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz#2894c0cb87d42276c3889942e8e2db517a492c43" integrity sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g== +"@img/sharp-libvips-darwin-x64@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz#e0456f8f7c623f9dbfbdc77383caa72281d86062" + integrity sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ== + "@img/sharp-libvips-darwin-x64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz#e63681f4539a94af9cd17246ed8881734386f8cc" integrity sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg== +"@img/sharp-libvips-linux-arm64@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz#979b1c66c9a91f7ff2893556ef267f90ebe51704" + integrity sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA== + "@img/sharp-libvips-linux-arm64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz#b1b288b36864b3bce545ad91fa6dadcf1a4ad318" integrity sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw== +"@img/sharp-libvips-linux-arm@1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz#99f922d4e15216ec205dcb6891b721bfd2884197" + integrity sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g== + "@img/sharp-libvips-linux-arm@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz#b9260dd1ebe6f9e3bdbcbdcac9d2ac125f35852d" @@ -2962,26 +3018,53 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz#880b4678009e5a2080af192332b00b0aaf8a48de" integrity sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA== +"@img/sharp-libvips-linux-s390x@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz#f8a5eb1f374a082f72b3f45e2fb25b8118a8a5ce" + integrity sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA== + "@img/sharp-libvips-linux-s390x@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz#74f343c8e10fad821b38f75ced30488939dc59ec" integrity sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ== +"@img/sharp-libvips-linux-x64@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz#d4c4619cdd157774906e15770ee119931c7ef5e0" + integrity sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw== + "@img/sharp-libvips-linux-x64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz#df4183e8bd8410f7d61b66859a35edeab0a531ce" integrity sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw== +"@img/sharp-libvips-linuxmusl-arm64@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz#166778da0f48dd2bded1fa3033cee6b588f0d5d5" + integrity sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA== + "@img/sharp-libvips-linuxmusl-arm64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz#c8d6b48211df67137541007ee8d1b7b1f8ca8e06" integrity sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw== +"@img/sharp-libvips-linuxmusl-x64@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz#93794e4d7720b077fcad3e02982f2f1c246751ff" + integrity sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw== + "@img/sharp-libvips-linuxmusl-x64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz#be11c75bee5b080cbee31a153a8779448f919f75" integrity sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg== +"@img/sharp-linux-arm64@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz#edb0697e7a8279c9fc829a60fc35644c4839bb22" + integrity sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA== + optionalDependencies: + "@img/sharp-libvips-linux-arm64" "1.0.4" + "@img/sharp-linux-arm64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz#7aa7764ef9c001f15e610546d42fce56911790cc" @@ -2989,6 +3072,13 @@ optionalDependencies: "@img/sharp-libvips-linux-arm64" "1.2.4" +"@img/sharp-linux-arm@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz#422c1a352e7b5832842577dc51602bcd5b6f5eff" + integrity sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ== + optionalDependencies: + "@img/sharp-libvips-linux-arm" "1.0.5" + "@img/sharp-linux-arm@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz#5fb0c3695dd12522d39c3ff7a6bc816461780a0d" @@ -3010,6 +3100,13 @@ optionalDependencies: "@img/sharp-libvips-linux-riscv64" "1.2.4" +"@img/sharp-linux-s390x@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz#f5c077926b48e97e4a04d004dfaf175972059667" + integrity sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q== + optionalDependencies: + "@img/sharp-libvips-linux-s390x" "1.0.4" + "@img/sharp-linux-s390x@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz#93eac601b9f329bb27917e0e19098c722d630df7" @@ -3017,6 +3114,13 @@ optionalDependencies: "@img/sharp-libvips-linux-s390x" "1.2.4" +"@img/sharp-linux-x64@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz#d806e0afd71ae6775cc87f0da8f2d03a7c2209cb" + integrity sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA== + optionalDependencies: + "@img/sharp-libvips-linux-x64" "1.0.4" + "@img/sharp-linux-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz#55abc7cd754ffca5002b6c2b719abdfc846819a8" @@ -3024,6 +3128,13 @@ optionalDependencies: "@img/sharp-libvips-linux-x64" "1.2.4" +"@img/sharp-linuxmusl-arm64@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz#252975b915894fb315af5deea174651e208d3d6b" + integrity sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g== + optionalDependencies: + "@img/sharp-libvips-linuxmusl-arm64" "1.0.4" + "@img/sharp-linuxmusl-arm64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz#d6515ee971bb62f73001a4829b9d865a11b77086" @@ -3031,6 +3142,13 @@ optionalDependencies: "@img/sharp-libvips-linuxmusl-arm64" "1.2.4" +"@img/sharp-linuxmusl-x64@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz#3f4609ac5d8ef8ec7dadee80b560961a60fd4f48" + integrity sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw== + optionalDependencies: + "@img/sharp-libvips-linuxmusl-x64" "1.0.4" + "@img/sharp-linuxmusl-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz#d97978aec7c5212f999714f2f5b736457e12ee9f" @@ -3038,6 +3156,13 @@ optionalDependencies: "@img/sharp-libvips-linuxmusl-x64" "1.2.4" +"@img/sharp-wasm32@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz#6f44f3283069d935bb5ca5813153572f3e6f61a1" + integrity sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg== + dependencies: + "@emnapi/runtime" "^1.2.0" + "@img/sharp-wasm32@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz#2f15803aa626f8c59dd7c9d0bbc766f1ab52cfa0" @@ -3050,11 +3175,21 @@ resolved "https://registry.yarnpkg.com/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz#3706e9e3ac35fddfc1c87f94e849f1b75307ce0a" integrity sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g== +"@img/sharp-win32-ia32@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz#1a0c839a40c5351e9885628c85f2e5dfd02b52a9" + integrity sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ== + "@img/sharp-win32-ia32@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz#0b71166599b049e032f085fb9263e02f4e4788de" integrity sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg== +"@img/sharp-win32-x64@0.33.5": + version "0.33.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz#56f00962ff0c4e0eb93d34a047d29fa995e3e342" + integrity sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg== + "@img/sharp-win32-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz#a81ffb00e69267cd0a1d626eaedb8a8430b2b2f8" @@ -5115,6 +5250,13 @@ dependencies: undici-types "~7.18.0" +"@types/pdf-parse@^1.1.4": + version "1.1.5" + resolved "https://registry.yarnpkg.com/@types/pdf-parse/-/pdf-parse-1.1.5.tgz#a0959022604457169177622b512ed03b975f10e2" + integrity sha512-kBfrSXsloMnUJOKi25s3+hRmkycHfLK6A09eRGqF/N8BkQoPUmaCr+q8Cli5FnfohEz/rsv82zAiPz/LXtOGhA== + dependencies: + "@types/node" "*" + "@types/sax@^1.2.1": version "1.2.7" resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.7.tgz#ba5fe7df9aa9c89b6dff7688a19023dd2963091d" @@ -6070,11 +6212,27 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" -color-name@~1.1.4: +color-name@^1.0.0, color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-string@^1.9.0: + version "1.9.1" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4" + integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg== + dependencies: + color-name "^1.0.0" + simple-swizzle "^0.2.2" + +color@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/color/-/color-4.2.3.tgz#d781ecb5e57224ee43ea9627560107c0e0c6463a" + integrity sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A== + dependencies: + color-convert "^2.0.1" + color-string "^1.9.0" + comma-separated-tokens@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" @@ -6310,7 +6468,7 @@ destr@^2.0.5: resolved "https://registry.yarnpkg.com/destr/-/destr-2.0.5.tgz#7d112ff1b925fb8d2079fac5bdb4a90973b51fdb" integrity sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA== -detect-libc@^2.1.2: +detect-libc@^2.0.3, detect-libc@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.2.tgz#689c5dcdc1900ef5583a4cb9f6d7b473742074ad" integrity sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ== @@ -7000,7 +7158,7 @@ fast-wrap-ansi@^0.1.3: dependencies: fast-string-width "^1.1.0" -fast-xml-builder@^1.1.5: +fast-xml-builder@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz#abd2363145a7625d9789ad96da375fabe3cff28c" integrity sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q== @@ -7009,14 +7167,15 @@ fast-xml-builder@^1.1.5: xml-naming "^0.1.0" fast-xml-parser@5.5.8, fast-xml-parser@5.7.2, fast-xml-parser@5.7.3, fast-xml-parser@^5.7.0: - version "5.7.2" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.7.2.tgz#fecd0b054c6c132fc03dab994a413da781e0eb9f" - integrity sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w== + version "5.8.0" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.8.0.tgz#64d71f0f8d4bf23621dffd762aef7e98c1884fc1" + integrity sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg== dependencies: "@nodable/entities" "^2.1.0" - fast-xml-builder "^1.1.5" + fast-xml-builder "^1.2.0" path-expression-matcher "^1.5.0" - strnum "^2.2.3" + strnum "^2.3.0" + xml-naming "^0.1.0" fb-watchman@^2.0.2: version "2.0.2" @@ -7742,6 +7901,11 @@ is-arrayish@^0.2.1: resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== +is-arrayish@^0.3.1: + version "0.3.4" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.4.tgz#1ee5553818511915685d33bb13d31bf854e5059d" + integrity sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA== + is-async-function@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/is-async-function/-/is-async-function-2.1.1.tgz#3e69018c8e04e73b738793d020bfe884b9fd3523" @@ -9325,10 +9489,10 @@ muggle-string@^0.4.1: resolved "https://registry.yarnpkg.com/muggle-string/-/muggle-string-0.4.1.tgz#3b366bd43b32f809dc20659534dd30e7c8a0d328" integrity sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ== -nanoid@^3.3.11: - version "3.3.11" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.11.tgz#4f4f112cefbe303202f2199838128936266d185b" - integrity sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w== +nanoid@^3.3.12: + version "3.3.12" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.12.tgz#ab3d912e217a6d0a514f00a72a16543a28982c05" + integrity sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ== napi-postinstall@^0.3.0: version "0.3.4" @@ -9362,6 +9526,11 @@ nlcst-to-string@^4.0.0: dependencies: "@types/nlcst" "^2.0.0" +node-ensure@^0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7" + integrity sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw== + node-fetch-native@^1.6.7: version "1.6.7" resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.7.tgz#9d09ca63066cc48423211ed4caf5d70075d76a71" @@ -9726,6 +9895,13 @@ path-scurry@^1.11.1: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" +pdf-parse@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/pdf-parse/-/pdf-parse-1.1.4.tgz#97bca6f46758130dafb1fdd9df905efd07581f4a" + integrity sha512-XRIRcLgk6ZnUbsHsYXExMw+krrPE81hJ6FQPLdBNhhBefqIQKXu/WeTgNBGSwPrfU0v+UCEwn7AoAUOsVKHFvQ== + dependencies: + node-ensure "^0.0.0" + piccolore@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/piccolore/-/piccolore-0.1.3.tgz#ef33f6180e6a37b35fe12a45765a900171cfaedc" @@ -9779,11 +9955,11 @@ postcss-selector-parser@^6.1.1: util-deprecate "^1.0.2" postcss@^8.4.38, postcss@^8.5.10, postcss@^8.5.6: - version "8.5.12" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.12.tgz#cd0c0f667f7cb0521e2313234ea6e707a9ec1ddb" - integrity sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA== + version "8.5.15" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.15.tgz#d1eaf677a324e9ec02196da2d3fecf4a0b9a735c" + integrity sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A== dependencies: - nanoid "^3.3.11" + nanoid "^3.3.12" picocolors "^1.1.1" source-map-js "^1.2.1" @@ -10269,6 +10445,11 @@ semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.2, semver@^7.7.1, semve resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.4.tgz#28464e36060e991fa7a11d0279d2d3f3b57a7e8a" integrity sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA== +semver@^7.6.3: + version "7.8.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.8.0.tgz#ed0661039fcbcda2ce71f01fa6adbefaa77040df" + integrity sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA== + set-function-length@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" @@ -10300,6 +10481,35 @@ set-proto@^1.0.0: es-errors "^1.3.0" es-object-atoms "^1.0.0" +sharp@^0.33.5: + version "0.33.5" + resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.33.5.tgz#13e0e4130cc309d6a9497596715240b2ec0c594e" + integrity sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw== + dependencies: + color "^4.2.3" + detect-libc "^2.0.3" + semver "^7.6.3" + optionalDependencies: + "@img/sharp-darwin-arm64" "0.33.5" + "@img/sharp-darwin-x64" "0.33.5" + "@img/sharp-libvips-darwin-arm64" "1.0.4" + "@img/sharp-libvips-darwin-x64" "1.0.4" + "@img/sharp-libvips-linux-arm" "1.0.5" + "@img/sharp-libvips-linux-arm64" "1.0.4" + "@img/sharp-libvips-linux-s390x" "1.0.4" + "@img/sharp-libvips-linux-x64" "1.0.4" + "@img/sharp-libvips-linuxmusl-arm64" "1.0.4" + "@img/sharp-libvips-linuxmusl-x64" "1.0.4" + "@img/sharp-linux-arm" "0.33.5" + "@img/sharp-linux-arm64" "0.33.5" + "@img/sharp-linux-s390x" "0.33.5" + "@img/sharp-linux-x64" "0.33.5" + "@img/sharp-linuxmusl-arm64" "0.33.5" + "@img/sharp-linuxmusl-x64" "0.33.5" + "@img/sharp-wasm32" "0.33.5" + "@img/sharp-win32-ia32" "0.33.5" + "@img/sharp-win32-x64" "0.33.5" + sharp@^0.34.0: version "0.34.5" resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.34.5.tgz#b6f148e4b8c61f1797bde11a9d1cfebbae2c57b0" @@ -10424,6 +10634,13 @@ signal-exit@^4.0.1: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== +simple-swizzle@^0.2.2: + version "0.2.4" + resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.4.tgz#a8d11a45a11600d6a1ecdff6363329e3648c3667" + integrity sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw== + dependencies: + is-arrayish "^0.3.1" + sisteransi@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" @@ -10564,7 +10781,16 @@ string-length@^4.0.2: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -10622,7 +10848,14 @@ stringify-entities@^4.0.0: character-entities-html4 "^2.0.0" character-entities-legacy "^3.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -10656,10 +10889,10 @@ strip-json-comments@^3.1.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== -strnum@^2.2.3: - version "2.2.3" - resolved "https://registry.yarnpkg.com/strnum/-/strnum-2.2.3.tgz#0119fce02749a11bb126a4d686ac5dbdf6e57586" - integrity sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg== +strnum@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/strnum/-/strnum-2.3.0.tgz#81bfbfef53db8c3217ea62a98c026886ec4a2761" + integrity sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q== style-to-js@^1.0.0: version "1.1.21" @@ -11429,7 +11662,16 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -11503,9 +11745,9 @@ yaml-language-server@~1.20.0: yaml "2.7.1" yaml@1.10.3, yaml@2.7.1, yaml@^2.8.2, yaml@^2.8.3: - version "2.8.3" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.3.tgz#a0d6bd2efb3dd03c59370223701834e60409bd7d" - integrity sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg== + version "2.9.0" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.9.0.tgz#78274afd93598a1dfdd6130df6a566defcbf9aa4" + integrity sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA== yargs-parser@^21.1.1: version "21.1.1" From 246f837d29985e988ba13de5f6f4aba765d1fbbe Mon Sep 17 00:00:00 2001 From: bgagent Date: Thu, 21 May 2026 11:41:53 -0500 Subject: [PATCH 07/19] chore(attachments): latest updates --- agent/src/attachments.py | 2 +- cdk/src/handlers/confirm-uploads.ts | 40 ++------------- .../handlers/shared/attachment-screening.ts | 2 +- cdk/src/handlers/shared/context-hydration.ts | 8 +++ cdk/src/handlers/shared/image-tokens.ts | 1 - cdk/src/handlers/shared/orchestrator.ts | 44 ++++++++++++---- .../shared/resolve-url-attachments.ts | 13 +++-- docs/design/ATTACHMENTS.md | 50 +++++++++---------- .../content/docs/architecture/Attachments.md | 50 +++++++++---------- 9 files changed, 107 insertions(+), 103 deletions(-) diff --git a/agent/src/attachments.py b/agent/src/attachments.py index 05bb0d6b..b998cdf8 100644 --- a/agent/src/attachments.py +++ b/agent/src/attachments.py @@ -1,4 +1,4 @@ -"""Attachment download and integrity verification (Phase 3). +"""Attachment download and integrity verification. Downloads attachments from S3 using version-pinned reads and verifies SHA-256 checksums against the orchestrator-provided values. Files are diff --git a/cdk/src/handlers/confirm-uploads.ts b/cdk/src/handlers/confirm-uploads.ts index 38fe27c0..6cab59bd 100644 --- a/cdk/src/handlers/confirm-uploads.ts +++ b/cdk/src/handlers/confirm-uploads.ts @@ -335,21 +335,10 @@ async function screenSingleAttachment( ContentType: att.content_type, })); - // Estimate token cost for images + // Estimate token cost for images (using shared utility) let tokenEstimate: number | undefined; if (isImage) { - try { - const sharp = (await import('sharp')).default; - const metadata = await sharp(screenResult.content).metadata(); - if (metadata.width && metadata.height) { - tokenEstimate = estimateImageTokens(metadata.width, metadata.height); - } - } catch (err) { - logger.warn('Failed to estimate image tokens in confirm-uploads (non-fatal)', { - error: err instanceof Error ? err.message : String(err), - attachment_id: att.attachment_id, - }); - } + tokenEstimate = await estimateImageTokensFromBuffer(screenResult.content); } return createAttachmentRecord({ @@ -494,10 +483,11 @@ async function transitionToSubmitted( request_id: requestId, }); } catch (orchErr) { - logger.error('Failed to invoke orchestrator after confirm-uploads', { - error: String(orchErr), + logger.error('Failed to invoke orchestrator after confirm-uploads — task may be stuck in SUBMITTED until EventBridge retry', { + error: orchErr instanceof Error ? orchErr.message : String(orchErr), task_id: taskId, request_id: requestId, + metric_type: 'orchestrator_invoke_failure', }); } } @@ -730,23 +720,3 @@ async function decrementConcurrency(userId: string): Promise { }); } } - -const MAX_IMAGE_SIDE = 1568; -const MAX_IMAGE_TOKENS = 1568; -const TOKEN_SAFETY_MARGIN = 1.2; -const TILE_SIZE = 28; - -function estimateImageTokens(width: number, height: number): number { - let w = width; - let h = height; - const maxSide = Math.max(w, h); - if (maxSide > MAX_IMAGE_SIDE) { - const scale = MAX_IMAGE_SIDE / maxSide; - w = Math.round(w * scale); - h = Math.round(h * scale); - } - w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; - h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; - const rawTokens = Math.ceil((w * h) / 750); - return Math.min(Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN), MAX_IMAGE_TOKENS); -} diff --git a/cdk/src/handlers/shared/attachment-screening.ts b/cdk/src/handlers/shared/attachment-screening.ts index 5cb91edd..6fda0ad2 100644 --- a/cdk/src/handlers/shared/attachment-screening.ts +++ b/cdk/src/handlers/shared/attachment-screening.ts @@ -17,8 +17,8 @@ * SOFTWARE. */ -import { ApplyGuardrailCommand, type BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; import { createHash } from 'crypto'; +import { ApplyGuardrailCommand, type BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; import sharp from 'sharp'; import { logger } from './logger'; diff --git a/cdk/src/handlers/shared/context-hydration.ts b/cdk/src/handlers/shared/context-hydration.ts index 1e93679b..74e4e3e6 100644 --- a/cdk/src/handlers/shared/context-hydration.ts +++ b/cdk/src/handlers/shared/context-hydration.ts @@ -180,6 +180,14 @@ export class AttachmentBudgetExceededError extends AttachmentError { } } +/** Attachment infrastructure is misconfigured (e.g. missing guardrail env vars). */ +export class AttachmentConfigurationError extends AttachmentError { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'AttachmentConfigurationError'; + } +} + /** Mapping from policy response keys to assessment detail extraction rules. */ const POLICY_EXTRACTORS: ReadonlyArray<{ readonly policyKey: string; diff --git a/cdk/src/handlers/shared/image-tokens.ts b/cdk/src/handlers/shared/image-tokens.ts index 5b2d3e16..a06bd65a 100644 --- a/cdk/src/handlers/shared/image-tokens.ts +++ b/cdk/src/handlers/shared/image-tokens.ts @@ -18,7 +18,6 @@ */ // Image token estimation matching Anthropic's documented resize rules. -// Shared by create-task-core.ts and confirm-uploads.ts. import sharp from 'sharp'; import { logger } from './logger'; diff --git a/cdk/src/handlers/shared/orchestrator.ts b/cdk/src/handlers/shared/orchestrator.ts index 8fede25f..b2d482a7 100644 --- a/cdk/src/handlers/shared/orchestrator.ts +++ b/cdk/src/handlers/shared/orchestrator.ts @@ -21,7 +21,7 @@ import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { S3Client } from '@aws-sdk/client-s3'; import { DynamoDBDocumentClient, GetCommand, PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; import { ulid } from 'ulid'; -import { AttachmentBudgetExceededError, hydrateContext, resolveGitHubToken } from './context-hydration'; +import { AttachmentBudgetExceededError, AttachmentConfigurationError, AttachmentResolutionError, hydrateContext, resolveGitHubToken } from './context-hydration'; import { logger } from './logger'; import { writeMinimalEpisode } from './memory'; import { coerceNumericOrNull } from './numeric'; @@ -43,6 +43,9 @@ const TASK_RETENTION_DAYS = Number(process.env.TASK_RETENTION_DAYS ?? '90'); const MEMORY_ID = process.env.MEMORY_ID; const ATTACHMENTS_BUCKET_NAME = process.env.ATTACHMENTS_BUCKET_NAME; +/** Conservative fallback token count for images where metadata could not be read. */ +const MAX_IMAGE_TOKENS_FALLBACK = 1568; + /** * State tracked across waitForCondition poll cycles. */ @@ -263,14 +266,17 @@ function resolveAttachmentPayloads( for (const att of attachments) { if (att.screening.status !== 'passed') continue; if (!att.s3_key || !att.s3_version_id || !att.checksum_sha256 || !att.size_bytes) { - logger.warn('Skipping attachment with passed screening but missing S3 fields', { - attachment_id: att.attachment_id, - }); - continue; + throw new AttachmentResolutionError( + `Attachment '${att.filename}' (${att.attachment_id}) has passed screening but is missing required ` + + 'storage metadata (s3_key, s3_version_id, checksum_sha256, or size_bytes). ' + + 'This is an internal data integrity error — please re-submit the task.', + ); } - if (att.type === 'image' && att.token_estimate) { - totalImageTokens += att.token_estimate; + if (att.type === 'image') { + // Use actual estimate if available, otherwise apply a conservative default + // to prevent images with unreadable metadata from bypassing the budget check. + totalImageTokens += att.token_estimate ?? MAX_IMAGE_TOKENS_FALLBACK; } payloads.push({ @@ -428,7 +434,7 @@ export async function hydrateAndTransition(task: TaskRecord, blueprintConfig?: B : undefined; if (!screeningConfig) { - throw new AttachmentBudgetExceededError( + throw new AttachmentConfigurationError( 'URL attachments require content screening (Bedrock Guardrail) but screening is not configured. ' + 'Set GUARDRAIL_ID and GUARDRAIL_VERSION environment variables.', ); @@ -441,7 +447,27 @@ export async function hydrateAndTransition(task: TaskRecord, blueprintConfig?: B try { githubToken = await resolveGitHubToken(githubTokenSecretArn); } catch (err) { - logger.warn('Failed to resolve GitHub token for URL attachment fetch', { + // Check if any pending URL attachments target GitHub — if so, fail clearly + // rather than proceeding to get an opaque 401/403 on fetch. + const githubHosts = ['github.com', 'raw.githubusercontent.com', 'api.github.com']; + const pendingUrlAtts = resolvedAttachments.filter(a => a.type === 'url' && a.screening.status === 'pending'); + const hasGitHubUrl = pendingUrlAtts.some(a => { + try { + const hostname = new URL(a.source_url ?? '').hostname.toLowerCase(); + return githubHosts.some(h => hostname === h || hostname.endsWith(`.${h}`)); + } catch { return false; } + }); + + if (hasGitHubUrl) { + throw new AttachmentResolutionError( + 'Failed to retrieve GitHub credentials needed to fetch URL attachment(s). ' + + `Ensure the GitHub token secret (${githubTokenSecretArn}) is accessible. ` + + `Underlying error: ${err instanceof Error ? err.message : String(err)}`, + { cause: err }, + ); + } + + logger.warn('Failed to resolve GitHub token for URL attachment fetch (no GitHub URLs in batch — proceeding)', { task_id: task.task_id, error: err instanceof Error ? err.message : String(err), }); diff --git a/cdk/src/handlers/shared/resolve-url-attachments.ts b/cdk/src/handlers/shared/resolve-url-attachments.ts index 93852fd8..292cde21 100644 --- a/cdk/src/handlers/shared/resolve-url-attachments.ts +++ b/cdk/src/handlers/shared/resolve-url-attachments.ts @@ -51,7 +51,7 @@ const URL_FETCH_TIMEOUT_MS = 10_000; const MAX_FETCH_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB const MAX_REDIRECTS = 2; -/** RFC 1918 + link-local + loopback + IPv6 equivalents. */ +/** RFC 1918 + link-local + loopback + CGN + IPv6 equivalents + IPv4-mapped IPv6. */ const PRIVATE_IP_RANGES = [ // IPv4 { prefix: '10.', mask: null }, @@ -66,11 +66,16 @@ const PRIVATE_IP_RANGES = [ { prefix: '169.254.', mask: null }, { prefix: '127.', mask: null }, { prefix: '0.', mask: null }, + { prefix: '100.64.', mask: null }, // CGN / Shared Address Space (RFC 6598) // IPv6 { prefix: '::1', mask: null }, { prefix: 'fc', mask: null }, { prefix: 'fd', mask: null }, { prefix: 'fe80:', mask: null }, + // IPv4-mapped IPv6 (e.g. ::ffff:169.254.169.254) — prevents SSRF bypass + // by attackers returning mapped addresses from DNS that embed private IPv4. + { prefix: '::ffff:', mask: null }, + { prefix: '0:0:0:0:0:ffff:', mask: null }, ] as const; // --------------------------------------------------------------------------- @@ -124,13 +129,13 @@ async function resolveAndValidate(hostname: string): Promise { try { // Try IPv4 first (more common for HTTP endpoints) addresses = await dns.resolve4(hostname); - } catch { + } catch (ipv4Err) { try { addresses = await dns.resolve6(hostname); - } catch (err) { + } catch (ipv6Err) { throw new AttachmentResolutionError( `DNS resolution failed for '${hostname}'. Check that the URL is correct and the server is reachable.`, - { cause: err }, + { cause: new AggregateError([ipv4Err, ipv6Err], `Both IPv4 and IPv6 resolution failed for '${hostname}'`) }, ); } } diff --git a/docs/design/ATTACHMENTS.md b/docs/design/ATTACHMENTS.md index 6ee5dcd2..ae787502 100644 --- a/docs/design/ATTACHMENTS.md +++ b/docs/design/ATTACHMENTS.md @@ -137,19 +137,17 @@ The `TaskRecord` gains an `attachments?: AttachmentRecord[]` field. This stores ```typescript function createAttachmentRecord(params: CreateAttachmentRecordParams): AttachmentRecord { // Validates invariants: - // - token_estimate required when type === 'image' // - s3_key and s3_version_id required when screening.status === 'passed' // - checksum_sha256 required when screening.status === 'passed' // - size_bytes required when screening.status === 'passed' // - categories non-empty when screening.status === 'blocked' + // Note: token_estimate is best-effort for images (sharp may fail on unusual formats), + // so it is NOT enforced here. The budget check uses a conservative fallback when absent. if (params.screening.status === 'passed') { if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { throw new Error('Passed screening requires s3_key, s3_version_id, checksum_sha256, and size_bytes'); } } - if (params.type === 'image' && params.screening.status === 'passed' && !params.token_estimate) { - throw new Error('Image attachments with passed screening must have token_estimate'); - } return params as AttachmentRecord; } ``` @@ -757,9 +755,9 @@ function estimateImageTokens(width: number, height: number): number { w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; - // Step 3: Token calculation with safety margin, capped - const rawTokens = Math.min(Math.ceil((w * h) / 750), MAX_IMAGE_TOKENS); - return Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN); + // Step 3: Token calculation with safety margin, then capped to hard ceiling + const rawTokens = Math.ceil((w * h) / 750); + return Math.min(Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN), MAX_IMAGE_TOKENS); } ``` @@ -767,12 +765,12 @@ For standard image sizes (after resizing): | Original dimensions | Resized to (pre-pad) | Padded to | Estimated tokens (with margin) | |---|---|---|---| -| 1920x1080 (full screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | -| 3840x2160 (4K screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | +| 1920x1080 (full screenshot) | 1568x882 | 1568x896 | ~1,568 (capped) | +| 3840x2160 (4K screenshot) | 1568x882 | 1568x896 | ~1,568 (capped) | | 800x600 (cropped screenshot) | 800x600 (no resize) | 812x616 | ~800 | -| 4096x4096 (max-size design) | 1568x1568 | 1568x1568 | ~1,882 (capped) | +| 4096x4096 (max-size design) | 1568x1568 | 1568x1568 | ~1,568 (capped) | -**Note:** The 4K screenshot and full HD screenshot produce the **same** token cost after resizing — both scale down to the same dimensions. The token cap at 1568 (+ safety margin) means very large or square images plateau rather than growing linearly. +**Note:** The 4K screenshot and full HD screenshot produce the **same** token cost after resizing — both scale down to the same dimensions. The hard cap at MAX_IMAGE_TOKENS (1568) means very large or square images plateau rather than growing linearly. **Budget enforcement in attachment resolution:** @@ -843,26 +841,26 @@ async def prepare_attachments( prepared = [] for att in attachments: - local_path = attachments_dir / f"{att.attachment_id}_{att.filename}" + # Unique subdirectory per attachment to avoid filename collisions + dest_dir = attachments_dir / att.attachment_id + dest_dir.mkdir(parents=True, exist_ok=True) + local_path = dest_dir / att.filename + bucket, key = parse_s3_uri(att.s3_uri) - try: - # Download the pinned version — prevents TOCTOU between screening and download - await s3_client.download_file( - bucket, key, str(local_path), - ExtraArgs={"VersionId": att.s3_version_id}, - ) - except ClientError as e: - raise AttachmentDownloadError( - f"Failed to download attachment {att.filename}: {e}" - ) from e + # Download the pinned version — prevents TOCTOU between screening and download + response = s3_client.get_object( + Bucket=bucket, Key=key, VersionId=att.s3_version_id, + ) + content = response["Body"].read() # Verify integrity via SHA-256 checksum (always present — required by factory) # hexdigest() returns lowercase hex, matching the format enforced by AttachmentConfig validator - actual_hash = hashlib.sha256(local_path.read_bytes()).hexdigest() + actual_hash = hashlib.sha256(content).hexdigest() if actual_hash != att.checksum_sha256: - raise AttachmentIntegrityError( - f"Checksum mismatch for {att.filename}: " - f"expected {att.checksum_sha256}, got {actual_hash}" + raise RuntimeError( + f"Attachment '{att.filename}' integrity check failed: " + f"expected SHA-256 {att.checksum_sha256}, got {actual_hash}. " + f"The file may have been tampered with." ) prepared.append(PreparedAttachment( diff --git a/docs/src/content/docs/architecture/Attachments.md b/docs/src/content/docs/architecture/Attachments.md index de2ecba9..7604e240 100644 --- a/docs/src/content/docs/architecture/Attachments.md +++ b/docs/src/content/docs/architecture/Attachments.md @@ -141,19 +141,17 @@ The `TaskRecord` gains an `attachments?: AttachmentRecord[]` field. This stores ```typescript function createAttachmentRecord(params: CreateAttachmentRecordParams): AttachmentRecord { // Validates invariants: - // - token_estimate required when type === 'image' // - s3_key and s3_version_id required when screening.status === 'passed' // - checksum_sha256 required when screening.status === 'passed' // - size_bytes required when screening.status === 'passed' // - categories non-empty when screening.status === 'blocked' + // Note: token_estimate is best-effort for images (sharp may fail on unusual formats), + // so it is NOT enforced here. The budget check uses a conservative fallback when absent. if (params.screening.status === 'passed') { if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { throw new Error('Passed screening requires s3_key, s3_version_id, checksum_sha256, and size_bytes'); } } - if (params.type === 'image' && params.screening.status === 'passed' && !params.token_estimate) { - throw new Error('Image attachments with passed screening must have token_estimate'); - } return params as AttachmentRecord; } ``` @@ -761,9 +759,9 @@ function estimateImageTokens(width: number, height: number): number { w = Math.ceil(w / TILE_SIZE) * TILE_SIZE; h = Math.ceil(h / TILE_SIZE) * TILE_SIZE; - // Step 3: Token calculation with safety margin, capped - const rawTokens = Math.min(Math.ceil((w * h) / 750), MAX_IMAGE_TOKENS); - return Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN); + // Step 3: Token calculation with safety margin, then capped to hard ceiling + const rawTokens = Math.ceil((w * h) / 750); + return Math.min(Math.ceil(rawTokens * TOKEN_SAFETY_MARGIN), MAX_IMAGE_TOKENS); } ``` @@ -771,12 +769,12 @@ For standard image sizes (after resizing): | Original dimensions | Resized to (pre-pad) | Padded to | Estimated tokens (with margin) | |---|---|---|---| -| 1920x1080 (full screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | -| 3840x2160 (4K screenshot) | 1568x882 | 1568x896 | ~1,882 (capped) | +| 1920x1080 (full screenshot) | 1568x882 | 1568x896 | ~1,568 (capped) | +| 3840x2160 (4K screenshot) | 1568x882 | 1568x896 | ~1,568 (capped) | | 800x600 (cropped screenshot) | 800x600 (no resize) | 812x616 | ~800 | -| 4096x4096 (max-size design) | 1568x1568 | 1568x1568 | ~1,882 (capped) | +| 4096x4096 (max-size design) | 1568x1568 | 1568x1568 | ~1,568 (capped) | -**Note:** The 4K screenshot and full HD screenshot produce the **same** token cost after resizing — both scale down to the same dimensions. The token cap at 1568 (+ safety margin) means very large or square images plateau rather than growing linearly. +**Note:** The 4K screenshot and full HD screenshot produce the **same** token cost after resizing — both scale down to the same dimensions. The hard cap at MAX_IMAGE_TOKENS (1568) means very large or square images plateau rather than growing linearly. **Budget enforcement in attachment resolution:** @@ -847,26 +845,26 @@ async def prepare_attachments( prepared = [] for att in attachments: - local_path = attachments_dir / f"{att.attachment_id}_{att.filename}" + # Unique subdirectory per attachment to avoid filename collisions + dest_dir = attachments_dir / att.attachment_id + dest_dir.mkdir(parents=True, exist_ok=True) + local_path = dest_dir / att.filename + bucket, key = parse_s3_uri(att.s3_uri) - try: - # Download the pinned version — prevents TOCTOU between screening and download - await s3_client.download_file( - bucket, key, str(local_path), - ExtraArgs={"VersionId": att.s3_version_id}, - ) - except ClientError as e: - raise AttachmentDownloadError( - f"Failed to download attachment {att.filename}: {e}" - ) from e + # Download the pinned version — prevents TOCTOU between screening and download + response = s3_client.get_object( + Bucket=bucket, Key=key, VersionId=att.s3_version_id, + ) + content = response["Body"].read() # Verify integrity via SHA-256 checksum (always present — required by factory) # hexdigest() returns lowercase hex, matching the format enforced by AttachmentConfig validator - actual_hash = hashlib.sha256(local_path.read_bytes()).hexdigest() + actual_hash = hashlib.sha256(content).hexdigest() if actual_hash != att.checksum_sha256: - raise AttachmentIntegrityError( - f"Checksum mismatch for {att.filename}: " - f"expected {att.checksum_sha256}, got {actual_hash}" + raise RuntimeError( + f"Attachment '{att.filename}' integrity check failed: " + f"expected SHA-256 {att.checksum_sha256}, got {actual_hash}. " + f"The file may have been tampered with." ) prepared.append(PreparedAttachment( From c60dd6d129785affa6b74813583771180cc0dbca Mon Sep 17 00:00:00 2001 From: bgagent Date: Mon, 25 May 2026 20:49:03 -0500 Subject: [PATCH 08/19] chore(attachments): update docs and add validation --- cdk/cdk.json | 3 - cdk/src/constructs/task-api.ts | 30 ++++-- cdk/src/constructs/task-orchestrator.ts | 35 ++++--- .../handlers/shared/attachment-screening.ts | 90 +++++++++++++++-- cdk/src/handlers/shared/image-tokens.ts | 3 +- cdk/src/handlers/shared/sharp-loader.ts | 32 ++++++ cdk/src/stacks/agent.ts | 4 + .../shared/attachment-screening.test.ts | 99 +++++++++++++++++++ docs/design/API_CONTRACT.md | 87 ++++++++++++++-- docs/design/ARCHITECTURE.md | 2 +- docs/design/SECURITY.md | 8 +- docs/guides/USER_GUIDE.md | 63 ++++++++++++ .../content/docs/architecture/Api-contract.md | 87 ++++++++++++++-- .../content/docs/architecture/Architecture.md | 2 +- .../src/content/docs/architecture/Security.md | 8 +- docs/src/content/docs/using/Using-the-cli.md | 63 ++++++++++++ 16 files changed, 561 insertions(+), 55 deletions(-) create mode 100644 cdk/src/handlers/shared/sharp-loader.ts create mode 100644 cdk/test/handlers/shared/attachment-screening.test.ts diff --git a/cdk/cdk.json b/cdk/cdk.json index 0008419f..6cf84b04 100644 --- a/cdk/cdk.json +++ b/cdk/cdk.json @@ -1,9 +1,6 @@ { "app": "npx ts-node -P tsconfig.json --prefer-ts-exts src/main.ts", "output": "cdk.out", - "context": { - "blueprintRepo": "krokoko/agent-plugins" - }, "build": "node -e \"process.exit(0)\"", "watch": { "include": ["src/**/*.ts", "test/**/*.ts"], diff --git a/cdk/src/constructs/task-api.ts b/cdk/src/constructs/task-api.ts index a6429e35..451e7ea9 100644 --- a/cdk/src/constructs/task-api.ts +++ b/cdk/src/constructs/task-api.ts @@ -274,8 +274,15 @@ export class TaskApi extends Construct { // ruleset. Exempt the Linear webhook path from CRS entirely: // the route is HMAC-verified in the Lambda, parsed as strict // JSON, never interpolated into SQL/HTML, and rate-limited by - // the priority-3 rule below. CRS still applies to every other - // route (user API, Slack, etc.). + // the priority-3 rule below. + // + // Inline task attachments (base64 in POST /v1/tasks, up to 3 MB + // total per ATTACHMENTS.md) also exceed the CRS 8 KB body cap and + // surface as API Gateway 403 {"message":"Forbidden"} before the + // create-task Lambda runs. Drop only SizeRestrictions_BODY here; + // other CRS rules still apply. Payload size is bounded by API GW + // (10 MB) and validateAttachments() in the handler. + excludedRules: [{ name: 'SizeRestrictions_BODY' }], scopeDownStatement: { notStatement: { statement: { @@ -380,6 +387,13 @@ export class TaskApi extends Construct { ], }; + // sharp ships native bindings; copy from node_modules (Docker bundling) + // instead of esbuild-inlining. Used by create-task / confirm-uploads paths. + const attachmentScreeningBundling: lambda.BundlingOptions = { + ...commonBundling, + nodeModules: ['sharp'], + }; + // --- Lambda handlers --- const createTaskEnv: Record = { ...commonEnv }; if (props.repoTable) { @@ -402,8 +416,10 @@ export class TaskApi extends Construct { runtime: Runtime.NODEJS_24_X, architecture: Architecture.ARM_64, environment: createTaskEnv, - bundling: commonBundling, - memorySize: 256, + bundling: attachmentScreeningBundling, + // Inline attachment screening (sharp) needs headroom; 256 MB caused + // cold-start init failures → API Gateway 502 on POST /tasks. + memorySize: 1024, timeout: Duration.seconds(15), }); @@ -543,7 +559,7 @@ export class TaskApi extends Construct { runtime: Runtime.NODEJS_24_X, architecture: Architecture.ARM_64, environment: confirmUploadsEnv, - bundling: commonBundling, + bundling: attachmentScreeningBundling, memorySize: 2048, timeout: Duration.seconds(180), }); @@ -888,7 +904,9 @@ export class TaskApi extends Construct { runtime: Runtime.NODEJS_24_X, architecture: Architecture.ARM_64, environment: createTaskEnv, - bundling: commonBundling, + bundling: attachmentScreeningBundling, + memorySize: 1024, + timeout: Duration.seconds(15), }); // --- IAM grants for webhook Lambdas --- diff --git a/cdk/src/constructs/task-orchestrator.ts b/cdk/src/constructs/task-orchestrator.ts index b5f4d744..9731b571 100644 --- a/cdk/src/constructs/task-orchestrator.ts +++ b/cdk/src/constructs/task-orchestrator.ts @@ -177,13 +177,30 @@ export class TaskOrchestrator extends Construct { const handlersDir = path.join(__dirname, '..', 'handlers'); const maxConcurrent = props.maxConcurrentTasksPerUser ?? 10; + // Hydration pulls in bedrock-agentcore (bundled), durable-execution, and + // attachment screening (sharp via resolve-url-attachments). 256 MB OOMs + // at cold start / early steps → task stuck in SUBMITTED (async invoke + // retryAttempts: 0 on the alias). + const orchestratorBundling: lambda.BundlingOptions = { + externalModules: [ + '@aws-sdk/client-dynamodb', + '@aws-sdk/client-ecs', + '@aws-sdk/client-lambda', + '@aws-sdk/client-bedrock-runtime', + '@aws-sdk/client-secrets-manager', + '@aws-sdk/lib-dynamodb', + '@aws-sdk/util-dynamodb', + ], + nodeModules: ['sharp'], + }; + this.fn = new lambda.NodejsFunction(this, 'OrchestratorFn', { entry: path.join(handlersDir, 'orchestrate-task.ts'), handler: 'handler', runtime: Runtime.NODEJS_24_X, architecture: Architecture.ARM_64, timeout: Duration.seconds(60), - memorySize: 256, + memorySize: 1024, durableConfig: { executionTimeout: Duration.hours(9), retentionPeriod: Duration.days(14), @@ -212,21 +229,7 @@ export class TaskOrchestrator extends Construct { }), ...(props.attachmentsBucket && { ATTACHMENTS_BUCKET_NAME: props.attachmentsBucket.bucketName }), }, - bundling: { - // Bundle `@aws-sdk/client-bedrock-agentcore` — newer commands (e.g. - // StopRuntimeSessionCommand) are not in the Lambda runtime's pinned - // SDK and throw ` is not a constructor` if externalized. - // See cancel-task silent-failure mode (task-api.ts commonBundling). - externalModules: [ - '@aws-sdk/client-dynamodb', - '@aws-sdk/client-ecs', - '@aws-sdk/client-lambda', - '@aws-sdk/client-bedrock-runtime', - '@aws-sdk/client-secrets-manager', - '@aws-sdk/lib-dynamodb', - '@aws-sdk/util-dynamodb', - ], - }, + bundling: orchestratorBundling, }); // DynamoDB grants diff --git a/cdk/src/handlers/shared/attachment-screening.ts b/cdk/src/handlers/shared/attachment-screening.ts index 6fda0ad2..7ddb92ab 100644 --- a/cdk/src/handlers/shared/attachment-screening.ts +++ b/cdk/src/handlers/shared/attachment-screening.ts @@ -19,8 +19,8 @@ import { createHash } from 'crypto'; import { ApplyGuardrailCommand, type BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; -import sharp from 'sharp'; import { logger } from './logger'; +import { loadSharp } from './sharp-loader'; // --------------------------------------------------------------------------- // Constants @@ -32,6 +32,9 @@ const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503]); export const MAX_ATTACHMENT_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB +/** Bedrock Guardrail image filter max side (docs: 8000x8000). */ +export const MAX_IMAGE_DIMENSION_PX = 8000; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -122,6 +125,8 @@ export async function screenImage( filename: string, config: ScreeningConfig, ): Promise { + await assertImageDecodable(content, contentType, filename); + // Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) let screeningBuffer: Buffer; let screeningFormat: 'png' | 'jpeg'; @@ -132,7 +137,7 @@ export async function screenImage( } else if (contentType === 'image/gif' || contentType === 'image/webp') { // GIF/WebP → PNG. For animated GIFs, extract first frame only. try { - screeningBuffer = await sharp(content, { animated: false }).png().toBuffer(); + screeningBuffer = await stripAndReencodeImage(content, 'image/png'); } catch (convErr) { throw new AttachmentScreeningError( `Image "${filename}" could not be converted from ${contentType} for screening. ` + @@ -183,16 +188,17 @@ export async function screenImage( }; } - // Screening passed — strip EXIF and re-encode. - // Note: NOT calling .withMetadata() — sharp strips all metadata by default - // when withMetadata is omitted. Calling .withMetadata({}) would opt INTO - // metadata preservation, which is the opposite of what we want. + // Screening passed — strip EXIF and re-encode in the declared format. let cleanedContent: Buffer; try { - cleanedContent = await sharp(content) - .rotate() // Apply EXIF orientation before stripping - .toBuffer(); + cleanedContent = await stripAndReencodeImage(content, contentType); } catch (sharpErr) { + logger.warn('Image sanitization failed (sharp)', { + filename, + content_type: contentType, + content_bytes: content.length, + error: sharpErr instanceof Error ? sharpErr.message : String(sharpErr), + }); throw new AttachmentScreeningError( `Image "${filename}" could not be processed for security sanitization. ` + 'Please re-export the image in a standard format and try again.', @@ -326,6 +332,72 @@ async function extractPdfText(content: Buffer, filename: string): Promise { + try { + const sharp = await loadSharp(); + const metadata = await sharp(content, SHARP_INPUT_OPTIONS).metadata(); + const width = metadata.width; + const height = metadata.height; + if (!width || !height) { + throw new AttachmentScreeningError( + `Image "${filename}" could not be decoded (missing dimensions). ` + + 'Please re-export as PNG or JPEG.', + ); + } + if (width > MAX_IMAGE_DIMENSION_PX || height > MAX_IMAGE_DIMENSION_PX) { + throw new AttachmentScreeningError( + `Image "${filename}" is ${width}x${height}px; maximum allowed dimension is ` + + `${MAX_IMAGE_DIMENSION_PX}px. Please resize the image before uploading.`, + ); + } + if (contentType === 'image/jpeg' && metadata.format && metadata.format !== 'jpeg') { + throw new AttachmentScreeningError( + `Image "${filename}" is not a valid JPEG despite the declared content type.`, + ); + } + if (contentType === 'image/png' && metadata.format && metadata.format !== 'png') { + throw new AttachmentScreeningError( + `Image "${filename}" is not a valid PNG despite the declared content type.`, + ); + } + } catch (err) { + if (err instanceof AttachmentScreeningError) { + throw err; + } + throw new AttachmentScreeningError( + `Image "${filename}" could not be decoded. The file may be corrupt or use an unsupported variant. ` + + 'Please re-export as PNG or JPEG.', + { cause: err }, + ); + } +} + +/** + * Strip metadata (EXIF/ICC/XMP), apply orientation, and re-encode. + * Explicit output format avoids ambiguous `.toBuffer()` behaviour in Lambda/libvips. + */ +async function stripAndReencodeImage(content: Buffer, contentType: string): Promise { + const sharp = await loadSharp(); + const pipeline = sharp(content, SHARP_INPUT_OPTIONS).rotate(); + if (contentType === 'image/jpeg') { + return pipeline.jpeg({ mozjpeg: true, force: true }).toBuffer(); + } + return pipeline.png({ compressionLevel: 9, force: true }).toBuffer(); +} + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- diff --git a/cdk/src/handlers/shared/image-tokens.ts b/cdk/src/handlers/shared/image-tokens.ts index a06bd65a..edd40748 100644 --- a/cdk/src/handlers/shared/image-tokens.ts +++ b/cdk/src/handlers/shared/image-tokens.ts @@ -19,8 +19,8 @@ // Image token estimation matching Anthropic's documented resize rules. -import sharp from 'sharp'; import { logger } from './logger'; +import { loadSharp } from './sharp-loader'; const MAX_IMAGE_SIDE = 1568; const MAX_IMAGE_TOKENS = 1568; @@ -58,6 +58,7 @@ export function estimateImageTokens(width: number, height: number): number { */ export async function estimateImageTokensFromBuffer(content: Buffer): Promise { try { + const sharp = await loadSharp(); const metadata = await sharp(content).metadata(); if (metadata.width && metadata.height) { return estimateImageTokens(metadata.width, metadata.height); diff --git a/cdk/src/handlers/shared/sharp-loader.ts b/cdk/src/handlers/shared/sharp-loader.ts new file mode 100644 index 00000000..f37408c7 --- /dev/null +++ b/cdk/src/handlers/shared/sharp-loader.ts @@ -0,0 +1,32 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import type sharp from 'sharp'; + +type SharpModule = typeof sharp; + +let sharpPromise: Promise | undefined; + +/** Lazy-load sharp so Lambdas without attachments avoid native module init at cold start. */ +export function loadSharp(): Promise { + if (!sharpPromise) { + sharpPromise = import('sharp').then(mod => mod.default); + } + return sharpPromise; +} diff --git a/cdk/src/stacks/agent.ts b/cdk/src/stacks/agent.ts index 8cbbae24..bdfc2fd6 100644 --- a/cdk/src/stacks/agent.ts +++ b/cdk/src/stacks/agent.ts @@ -388,6 +388,10 @@ export class AgentStack extends Stack { // the runtime itself. Deferred because the session-tag plumbing is // orthogonal to landing the feature behavior. traceArtifactsBucket.bucket.grantPut(runtime); + // Version-pinned attachment downloads (agent/src/attachments.py uses + // GetObject + VersionId). grantRead expands to s3:GetObject* including + // GetObjectVersion on the versioned attachments bucket. + attachmentsBucket.bucket.grantRead(runtime); const model = new bedrock.BedrockFoundationModel('anthropic.claude-sonnet-4-6', { supportsAgents: true, diff --git a/cdk/test/handlers/shared/attachment-screening.test.ts b/cdk/test/handlers/shared/attachment-screening.test.ts new file mode 100644 index 00000000..0b8d83aa --- /dev/null +++ b/cdk/test/handlers/shared/attachment-screening.test.ts @@ -0,0 +1,99 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import { + AttachmentScreeningError, + screenImage, +} from '../../../src/handlers/shared/attachment-screening'; + +const ARCHITECTURE_PNG = path.join( + __dirname, + '..', + '..', + '..', + '..', + 'cli', + 'autonomous-engine-architecture.png', +); + +function mockBedrockPass(): BedrockRuntimeClient { + return { + send: jest.fn().mockResolvedValue({ + action: 'NONE', + outputs: [], + assessments: [], + }), + } as unknown as BedrockRuntimeClient; +} + +describe('screenImage', () => { + const config = { + bedrockClient: mockBedrockPass(), + guardrailId: 'test-guardrail', + guardrailVersion: '1', + }; + + test('sanitizes a large real-world PNG (architecture diagram fixture)', async () => { + if (!fs.existsSync(ARCHITECTURE_PNG)) { + return; + } + const content = fs.readFileSync(ARCHITECTURE_PNG); + const result = await screenImage( + content, + 'image/png', + 'autonomous-engine-architecture.png', + config, + ); + + expect(result.contentType).toBe('image/png'); + expect(result.content.length).toBeGreaterThan(0); + expect(result.content.length).toBeLessThanOrEqual(content.length); + expect(result.checksum).toMatch(/^[0-9a-f]{64}$/); + expect(result.content).not.toEqual(content); + }); + + test('rejects oversized dimensions before guardrail', async () => { + const send = jest.fn(); + const client = { send } as unknown as BedrockRuntimeClient; + const sharp = await import('sharp'); + const oversized = await sharp.default({ + create: { + width: 8001, + height: 10, + channels: 3, + background: { r: 0, g: 0, b: 0 }, + }, + }) + .png() + .toBuffer(); + + await expect( + screenImage(oversized, 'image/png', 'huge.png', { + bedrockClient: client, + guardrailId: 'g', + guardrailVersion: '1', + }), + ).rejects.toThrow(AttachmentScreeningError); + + expect(send).not.toHaveBeenCalled(); + }); +}); diff --git a/docs/design/API_CONTRACT.md b/docs/design/API_CONTRACT.md index 9e4b2900..219de6a1 100644 --- a/docs/design/API_CONTRACT.md +++ b/docs/design/API_CONTRACT.md @@ -56,6 +56,7 @@ The gateway extracts `user_id` from the authenticated identity and attaches it t | Method | Path | Auth | Description | |--------|------|------|-------------| | `POST` | `/v1/tasks` | Cognito | Create a task | +| `POST` | `/v1/tasks/{task_id}/confirm-uploads` | Cognito | Confirm presigned uploads and trigger screening | | `GET` | `/v1/tasks` | Cognito | List tasks (paginated) | | `GET` | `/v1/tasks/{task_id}` | Cognito | Get task details | | `DELETE` | `/v1/tasks/{task_id}` | Cognito | Cancel a task | @@ -79,7 +80,7 @@ Creates a new task. The orchestrator runs admission control, context hydration, |---|---|---|---| | `repo` | String | Yes | GitHub repository (`owner/repo`) | | `issue_number` | Number | No | GitHub issue number. Title, body, and comments are fetched during hydration. | -| `task_description` | String | No | Free-text description (max 2,000 chars). At least one of `issue_number`, `task_description`, or `pr_number` required. | +| `task_description` | String | No | Free-text description (max 10,000 chars). At least one of `issue_number`, `task_description`, or `pr_number` required. | | `task_type` | String | No | `new_task` (default), `pr_iteration`, or `pr_review` | | `pr_number` | Number | No | PR to iterate on or review. Required when `task_type` is `pr_iteration` or `pr_review`. | | `max_turns` | Number | No | Max agent turns (1-500, default 100) | @@ -91,10 +92,69 @@ Creates a new task. The orchestrator runs admission control, context hydration, | Field | Type | Required | Description | |---|---|---|---| | `type` | String | Yes | `image`, `file`, or `url` | -| `content_type` | String | No | MIME type (for inline data) | -| `data` | String | No | Base64-encoded content (max 10 MB decoded) | -| `url` | String | No | URL to fetch | -| `filename` | String | No | Original filename | +| `content_type` | String | Conditional | MIME type. Required for presigned uploads and URL attachments; auto-detected from magic bytes for inline data. | +| `data` | String | No | Base64-encoded content (max 500 KB decoded per attachment; max 3 MB total inline per request). Mutually exclusive with `url`. | +| `url` | String | No | HTTPS URL to fetch during hydration (max 10 MB, 10s timeout). Required when `type` is `url`. | +| `filename` | String | No | Original filename. Auto-generated if absent. Must not contain path separators or start with `.`. | +| `expected_size_bytes` | Number | Conditional | Declared file size in bytes. Required for presigned uploads (no `data`, no `url`). Used for early budget validation. | + +**Supported MIME types:** + +| Category | MIME types | Extensions | +|---|---|---| +| Images | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | `.png`, `.jpg`, `.gif`, `.webp` | +| Text files | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | + +**Attachment limits:** + +| Limit | Value | +|---|---| +| Max attachments per task | 10 | +| Max inline data per attachment | 500 KB (decoded) | +| Max total inline data per request | 3 MB (decoded) | +| Max size per attachment (presigned/URL) | 10 MB | +| Max total size per task | 50 MB | +| URL scheme | HTTPS only | +| URL fetch timeout | 10 seconds | + +**Upload paths:** + +- **Inline** (≤ 500 KB): Include base64-encoded `data` in the request body. Task is created in `SUBMITTED` status. +- **Presigned** (> 500 KB, up to 10 MB): Omit `data` and `url`, include `expected_size_bytes`. Task is created in `PENDING_UPLOADS` status with presigned POST URLs returned in the response. Upload directly to S3, then call `POST /v1/tasks/{task_id}/confirm-uploads`. +- **URL** (deferred): Include `url`. Content is fetched and screened during context hydration. + +**Response for presigned upload tasks: `202 Accepted`** + +```json +{ + "data": { + "task_id": "01HYX...", + "status": "PENDING_UPLOADS", + "task_expires_at": "2025-03-15T11:00:00Z", + "attachments": [ + { + "attachment_id": "01HYX...", + "filename": "screenshot.png", + "upload_url": "https://s3.amazonaws.com/...", + "upload_fields": { "key": "...", "policy": "...", "x-amz-signature": "..." }, + "upload_expires_at": "2025-03-15T10:40:00Z" + } + ] + } +} +``` + +### Confirm uploads + +``` +POST /v1/tasks/{task_id}/confirm-uploads +``` + +Triggers security screening for presigned-upload attachments and transitions the task from `PENDING_UPLOADS` to `SUBMITTED`. No request body required. + +**Response: `200 OK`** — Task detail with `status: "SUBMITTED"`. + +**Errors:** `400 ATTACHMENT_BLOCKED`, `400 ATTACHMENT_SIZE_MISMATCH`, `404 TASK_NOT_FOUND`, `409 UPLOADS_NOT_CONFIRMED` (task not in `PENDING_UPLOADS` state), `410 UPLOADS_EXPIRED`, `503 ATTACHMENT_SCREENING_UNAVAILABLE`. **Response: `201 Created`** @@ -116,7 +176,7 @@ For PR tasks, `branch_name` is initially `pending:pr_resolution` and resolved to **Idempotency:** Clients may send `Idempotency-Key` (same format as other `POST` requests). The first successful create returns **`201 Created`** with the body shape above. A subsequent request with the same key and the **same authenticated user** returns **`200 OK`** with the same `{ data: ... }` envelope (full `TaskDetail`, reflecting **current** task state), plus response header `Idempotent-Replay: true`. No duplicate task is created and the orchestrator is not invoked again for that replay. If the key is already bound to a task owned by **another** user, the API returns **`409 DUPLICATE_TASK`** without exposing that task (extremely unlikely for high-entropy keys). -**Errors:** `400 VALIDATION_ERROR`, `400 GUARDRAIL_BLOCKED`, `401 UNAUTHORIZED`, `409 DUPLICATE_TASK` (idempotency key collision across users only), `422 REPO_NOT_ONBOARDED`, `429 RATE_LIMIT_EXCEEDED`, `503 SERVICE_UNAVAILABLE`. +**Errors:** `400 VALIDATION_ERROR`, `400 GUARDRAIL_BLOCKED`, `400 ATTACHMENT_BLOCKED`, `400 ATTACHMENT_INVALID_TYPE`, `400 ATTACHMENT_INVALID_CONTENT`, `400 ATTACHMENT_INLINE_TOO_LARGE`, `400 ATTACHMENTS_TOTAL_TOO_LARGE`, `401 UNAUTHORIZED`, `409 DUPLICATE_TASK` (idempotency key collision across users only), `422 REPO_NOT_ONBOARDED`, `429 RATE_LIMIT_EXCEEDED`, `503 SERVICE_UNAVAILABLE`, `503 ATTACHMENT_SCREENING_UNAVAILABLE`. ### Get task @@ -340,6 +400,21 @@ Tasks created via webhook record `channel_source: 'webhook'` with audit metadata | `REPO_NOT_FOUND_OR_NO_ACCESS` | 422 | Repo onboarded but credentials cannot reach it | | `PR_NOT_FOUND_OR_CLOSED` | 422 | PR does not exist, is closed, or is inaccessible | | `INSUFFICIENT_GITHUB_REPO_PERMISSIONS` | 422 | GitHub token lacks required permissions for the task type | +| `ATTACHMENT_BLOCKED` | 400 | Attachment content blocked by security screening | +| `ATTACHMENT_TOO_LARGE` | 400 | Individual attachment exceeds 10 MB size limit | +| `ATTACHMENT_INLINE_TOO_LARGE` | 400 | Inline attachment exceeds 500 KB limit (use presigned upload) | +| `ATTACHMENTS_INLINE_TOTAL_TOO_LARGE` | 400 | Total inline attachment size exceeds 3 MB limit | +| `ATTACHMENTS_TOTAL_TOO_LARGE` | 400 | Total attachment size exceeds 50 MB limit | +| `ATTACHMENT_INVALID_TYPE` | 400 | MIME type not in allowlist | +| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or could not be sanitized | +| `ATTACHMENT_INVALID_FILENAME` | 400 | Filename contains invalid characters or path traversal | +| `ATTACHMENT_SIZE_MISMATCH` | 400 | Uploaded file size does not match declared `expected_size_bytes` | +| `ATTACHMENT_FETCH_FAILED` | 422 | URL attachment could not be fetched (timeout, DNS, SSRF blocked) | +| `ATTACHMENT_BUDGET_EXCEEDED` | 422 | Image attachments exceed token budget | +| `ATTACHMENT_UNSUPPORTED_AGENT` | 422 | Agent runtime version does not support attachments | +| `UPLOADS_NOT_CONFIRMED` | 409 | Task is in PENDING_UPLOADS but confirm-uploads not yet called | +| `UPLOADS_EXPIRED` | 410 | Upload window expired (30 minutes); re-submit the task | +| `ATTACHMENT_SCREENING_UNAVAILABLE` | 503 | Screening service unavailable after retries | | `GITHUB_UNREACHABLE` | 502 | GitHub API unreachable during pre-flight (transient) | | `RATE_LIMIT_EXCEEDED` | 429 | User exceeded rate limit | | `CONCURRENCY_LIMIT_EXCEEDED` | 409 | User at max concurrent tasks | diff --git a/docs/design/ARCHITECTURE.md b/docs/design/ARCHITECTURE.md index 888779c9..0103cc13 100644 --- a/docs/design/ARCHITECTURE.md +++ b/docs/design/ARCHITECTURE.md @@ -26,7 +26,7 @@ flowchart LR ``` 1. **Admission** (deterministic) - The orchestrator validates the request, checks concurrency limits, and loads the repository's Blueprint configuration. -2. **Context hydration** (deterministic) - The platform fetches external data (GitHub issue body, PR diff, review comments), loads memory from past tasks, and assembles the full prompt. For PR tasks, the prompt is screened through Bedrock Guardrails. +2. **Context hydration** (deterministic) - The platform fetches external data (GitHub issue body, PR diff, review comments), loads memory from past tasks, resolves attachments (URL fetch with SSRF protection, screening, upload to S3), and assembles the full prompt. For PR tasks, the prompt is screened through Bedrock Guardrails. 3. **Pre-flight checks** (deterministic) - GitHub API reachability and repository access are verified. Doomed tasks fail fast with a clear reason before consuming compute. 4. **Agent execution** (agentic) - The agent runs in an isolated compute environment: clone repo, create branch, edit code, commit, run tests, create PR. The orchestrator polls for completion without blocking. 5. **Finalization** (deterministic) - The orchestrator infers the result (PR created or not), writes memory, updates task status, and releases concurrency. diff --git a/docs/design/SECURITY.md b/docs/design/SECURITY.md index 84882bc0..1f7eaab2 100644 --- a/docs/design/SECURITY.md +++ b/docs/design/SECURITY.md @@ -44,15 +44,17 @@ Input screening happens at two points in the pipeline, forming a defense-in-dept ### Submission-time screening -- **Input validation** - Required fields, types, and size limits are enforced before any processing. Task descriptions are capped at 2,000 characters. -- **Bedrock Guardrails** - A `PROMPT_ATTACK` content filter at `HIGH` strength screens task descriptions for prompt injection. +- **Input validation** - Required fields, types, and size limits are enforced before any processing. Task descriptions are capped at 10,000 characters. +- **Bedrock Guardrails** - A `PROMPT_ATTACK` content filter at `MEDIUM` input strength screens task descriptions for prompt injection. +- **Attachment screening** - All attachments (images, text files, URLs) pass through security screening before reaching the agent. Images are validated via magic bytes, screened through Bedrock Guardrails (image content blocks), stripped of EXIF/metadata, and re-encoded. Text files and PDFs are extracted and screened through Bedrock Guardrails text content screening. URL attachments undergo SSRF protection (DNS resolution pinning, private IP blocking, redirect validation) and content screening during hydration. See [ATTACHMENTS.md](./ATTACHMENTS.md) for the full screening pipeline. - **Fail-closed** - If the Bedrock API is unavailable, submissions are rejected (HTTP 503). Unscreened content never reaches the agent. ### Hydration-time screening - **PR tasks** (`pr_iteration`, `pr_review`) - The assembled prompt (PR body, review comments, diff, task description) is screened through Bedrock Guardrails before the agent receives it. - **`new_task` with issue content** - The assembled prompt (issue body, comments, task description) is screened. When no issue content is present, hydration-time screening is skipped because the task description was already screened at submission. -- **Fail-closed** - A Bedrock outage during hydration fails the task. A `guardrail_blocked` event is emitted when content is blocked. +- **URL attachments** - URL attachments are fetched during hydration with full SSRF protection (DNS resolution pinning to prevent rebinding attacks, private IP blocking, redirect validation, timeout enforcement). Fetched content is screened through the same pipeline as inline attachments before being stored in S3. +- **Fail-closed** - A Bedrock outage during hydration fails the task. A `guardrail_blocked` event is emitted when content is blocked. Attachment resolution errors (fetch failures, screening blocks, integrity failures) also fail the task — attachments are never silently dropped. ### Tool access control diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index d07fb952..22dc3484 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -402,6 +402,15 @@ node lib/bin/bgagent.js submit --repo owner/repo --review-pr 55 # Review a PR with a specific focus area node lib/bin/bgagent.js submit --repo owner/repo --review-pr 55 --task "Focus on security and error handling" +# Submit with attachments (local files) +node lib/bin/bgagent.js submit --repo owner/repo --task "Fix this bug" \ + --attachment screenshot.png \ + --attachment error.log + +# Submit with a URL attachment +node lib/bin/bgagent.js submit --repo owner/repo --task "Implement this design" \ + --attachment https://figma.com/file/abc123/export.png + # Submit and wait for completion node lib/bin/bgagent.js submit --repo owner/repo --issue 42 --wait ``` @@ -430,6 +439,7 @@ Created: 2026-04-01T00:39:51.271Z | `--task` | Task description text. | | `--pr` | PR number to iterate on. Sets task type to `pr_iteration`. The agent checks out the PR's branch, reads review feedback, and pushes updates. | | `--review-pr` | PR number to review. Sets task type to `pr_review`. The agent checks out the PR's branch, analyzes changes read-only, and posts structured review comments. | +| `--attachment` | Attach a file or URL (repeatable). Local files ≤ 500 KB are sent inline; larger files use presigned upload. URLs are fetched during hydration. See [Attachments](#attachments) below. | | `--max-turns` | Maximum agent turns (1–500). Overrides per-repo Blueprint default. Platform default: 100. | | `--max-budget` | Maximum cost budget in USD (0.01–100). Overrides per-repo Blueprint default. No default limit. | | `--idempotency-key` | Idempotency key for deduplication. | @@ -441,6 +451,59 @@ Created: 2026-04-01T00:39:51.271Z At least one of `--issue`, `--task`, `--pr`, or `--review-pr` is required. The `--pr` and `--review-pr` flags are mutually exclusive. +### Attachments + +Attachments let you provide non-text context to the agent — screenshots of bugs, design mockups, CSV data, log files, or URLs to external resources. Every attachment passes through security screening before reaching the agent. + +**Supported file types:** + +| Category | Types | Extensions | +|---|---|---| +| Images | PNG, JPEG, GIF, WebP | `.png`, `.jpg`, `.gif`, `.webp` | +| Text files | Plain text, CSV, Markdown, JSON, PDF, Log | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | + +**Limits:** + +| Limit | Value | +|---|---| +| Max attachments per task | 10 | +| Max size per attachment | 10 MB | +| Max total size per task | 50 MB | +| URL attachments | HTTPS only | + +**Usage:** + +```bash +# Local file (auto-detects MIME type from content) +node lib/bin/bgagent.js submit --repo owner/repo --task "Fix this layout" \ + --attachment screenshot.png + +# Multiple attachments +node lib/bin/bgagent.js submit --repo owner/repo --task "Analyze these logs" \ + --attachment error.log \ + --attachment metrics.csv + +# URL attachment (fetched during task hydration) +node lib/bin/bgagent.js submit --repo owner/repo --task "Implement this design" \ + --attachment https://example.com/mockup.png +``` + +The CLI automatically routes attachments through the optimal upload path: + +- **Files ≤ 500 KB** are sent inline (base64-encoded in the request body). +- **Files > 500 KB** use presigned upload (uploaded directly to S3, then confirmed). +- **URLs** are validated at submission and fetched during context hydration with SSRF protection. + +**Security screening:** + +All attachments are screened before reaching the agent: + +- **Images**: Magic bytes validation, Bedrock Guardrail content screening (prompt attack detection), EXIF/metadata stripping, re-encoding to remove embedded payloads. +- **Text files**: Magic bytes validation, Bedrock Guardrail text content screening. PDFs have text extracted (max 50 pages) before screening. +- **URLs**: HTTPS-only enforcement, DNS resolution pinning (prevents DNS rebinding/SSRF), private IP blocking, redirect validation, size and timeout limits. + +If any attachment fails screening, the entire task is rejected with a clear error identifying the problematic file. Re-submit without the flagged attachment. + ### Checking task status Run these from the `cli/` directory (same as in **Setup**). diff --git a/docs/src/content/docs/architecture/Api-contract.md b/docs/src/content/docs/architecture/Api-contract.md index dd37e712..1545dee8 100644 --- a/docs/src/content/docs/architecture/Api-contract.md +++ b/docs/src/content/docs/architecture/Api-contract.md @@ -60,6 +60,7 @@ The gateway extracts `user_id` from the authenticated identity and attaches it t | Method | Path | Auth | Description | |--------|------|------|-------------| | `POST` | `/v1/tasks` | Cognito | Create a task | +| `POST` | `/v1/tasks/{task_id}/confirm-uploads` | Cognito | Confirm presigned uploads and trigger screening | | `GET` | `/v1/tasks` | Cognito | List tasks (paginated) | | `GET` | `/v1/tasks/{task_id}` | Cognito | Get task details | | `DELETE` | `/v1/tasks/{task_id}` | Cognito | Cancel a task | @@ -83,7 +84,7 @@ Creates a new task. The orchestrator runs admission control, context hydration, |---|---|---|---| | `repo` | String | Yes | GitHub repository (`owner/repo`) | | `issue_number` | Number | No | GitHub issue number. Title, body, and comments are fetched during hydration. | -| `task_description` | String | No | Free-text description (max 2,000 chars). At least one of `issue_number`, `task_description`, or `pr_number` required. | +| `task_description` | String | No | Free-text description (max 10,000 chars). At least one of `issue_number`, `task_description`, or `pr_number` required. | | `task_type` | String | No | `new_task` (default), `pr_iteration`, or `pr_review` | | `pr_number` | Number | No | PR to iterate on or review. Required when `task_type` is `pr_iteration` or `pr_review`. | | `max_turns` | Number | No | Max agent turns (1-500, default 100) | @@ -95,10 +96,69 @@ Creates a new task. The orchestrator runs admission control, context hydration, | Field | Type | Required | Description | |---|---|---|---| | `type` | String | Yes | `image`, `file`, or `url` | -| `content_type` | String | No | MIME type (for inline data) | -| `data` | String | No | Base64-encoded content (max 10 MB decoded) | -| `url` | String | No | URL to fetch | -| `filename` | String | No | Original filename | +| `content_type` | String | Conditional | MIME type. Required for presigned uploads and URL attachments; auto-detected from magic bytes for inline data. | +| `data` | String | No | Base64-encoded content (max 500 KB decoded per attachment; max 3 MB total inline per request). Mutually exclusive with `url`. | +| `url` | String | No | HTTPS URL to fetch during hydration (max 10 MB, 10s timeout). Required when `type` is `url`. | +| `filename` | String | No | Original filename. Auto-generated if absent. Must not contain path separators or start with `.`. | +| `expected_size_bytes` | Number | Conditional | Declared file size in bytes. Required for presigned uploads (no `data`, no `url`). Used for early budget validation. | + +**Supported MIME types:** + +| Category | MIME types | Extensions | +|---|---|---| +| Images | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | `.png`, `.jpg`, `.gif`, `.webp` | +| Text files | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | + +**Attachment limits:** + +| Limit | Value | +|---|---| +| Max attachments per task | 10 | +| Max inline data per attachment | 500 KB (decoded) | +| Max total inline data per request | 3 MB (decoded) | +| Max size per attachment (presigned/URL) | 10 MB | +| Max total size per task | 50 MB | +| URL scheme | HTTPS only | +| URL fetch timeout | 10 seconds | + +**Upload paths:** + +- **Inline** (≤ 500 KB): Include base64-encoded `data` in the request body. Task is created in `SUBMITTED` status. +- **Presigned** (> 500 KB, up to 10 MB): Omit `data` and `url`, include `expected_size_bytes`. Task is created in `PENDING_UPLOADS` status with presigned POST URLs returned in the response. Upload directly to S3, then call `POST /v1/tasks/{task_id}/confirm-uploads`. +- **URL** (deferred): Include `url`. Content is fetched and screened during context hydration. + +**Response for presigned upload tasks: `202 Accepted`** + +```json +{ + "data": { + "task_id": "01HYX...", + "status": "PENDING_UPLOADS", + "task_expires_at": "2025-03-15T11:00:00Z", + "attachments": [ + { + "attachment_id": "01HYX...", + "filename": "screenshot.png", + "upload_url": "https://s3.amazonaws.com/...", + "upload_fields": { "key": "...", "policy": "...", "x-amz-signature": "..." }, + "upload_expires_at": "2025-03-15T10:40:00Z" + } + ] + } +} +``` + +### Confirm uploads + +``` +POST /v1/tasks/{task_id}/confirm-uploads +``` + +Triggers security screening for presigned-upload attachments and transitions the task from `PENDING_UPLOADS` to `SUBMITTED`. No request body required. + +**Response: `200 OK`** — Task detail with `status: "SUBMITTED"`. + +**Errors:** `400 ATTACHMENT_BLOCKED`, `400 ATTACHMENT_SIZE_MISMATCH`, `404 TASK_NOT_FOUND`, `409 UPLOADS_NOT_CONFIRMED` (task not in `PENDING_UPLOADS` state), `410 UPLOADS_EXPIRED`, `503 ATTACHMENT_SCREENING_UNAVAILABLE`. **Response: `201 Created`** @@ -120,7 +180,7 @@ For PR tasks, `branch_name` is initially `pending:pr_resolution` and resolved to **Idempotency:** Clients may send `Idempotency-Key` (same format as other `POST` requests). The first successful create returns **`201 Created`** with the body shape above. A subsequent request with the same key and the **same authenticated user** returns **`200 OK`** with the same `{ data: ... }` envelope (full `TaskDetail`, reflecting **current** task state), plus response header `Idempotent-Replay: true`. No duplicate task is created and the orchestrator is not invoked again for that replay. If the key is already bound to a task owned by **another** user, the API returns **`409 DUPLICATE_TASK`** without exposing that task (extremely unlikely for high-entropy keys). -**Errors:** `400 VALIDATION_ERROR`, `400 GUARDRAIL_BLOCKED`, `401 UNAUTHORIZED`, `409 DUPLICATE_TASK` (idempotency key collision across users only), `422 REPO_NOT_ONBOARDED`, `429 RATE_LIMIT_EXCEEDED`, `503 SERVICE_UNAVAILABLE`. +**Errors:** `400 VALIDATION_ERROR`, `400 GUARDRAIL_BLOCKED`, `400 ATTACHMENT_BLOCKED`, `400 ATTACHMENT_INVALID_TYPE`, `400 ATTACHMENT_INVALID_CONTENT`, `400 ATTACHMENT_INLINE_TOO_LARGE`, `400 ATTACHMENTS_TOTAL_TOO_LARGE`, `401 UNAUTHORIZED`, `409 DUPLICATE_TASK` (idempotency key collision across users only), `422 REPO_NOT_ONBOARDED`, `429 RATE_LIMIT_EXCEEDED`, `503 SERVICE_UNAVAILABLE`, `503 ATTACHMENT_SCREENING_UNAVAILABLE`. ### Get task @@ -344,6 +404,21 @@ Tasks created via webhook record `channel_source: 'webhook'` with audit metadata | `REPO_NOT_FOUND_OR_NO_ACCESS` | 422 | Repo onboarded but credentials cannot reach it | | `PR_NOT_FOUND_OR_CLOSED` | 422 | PR does not exist, is closed, or is inaccessible | | `INSUFFICIENT_GITHUB_REPO_PERMISSIONS` | 422 | GitHub token lacks required permissions for the task type | +| `ATTACHMENT_BLOCKED` | 400 | Attachment content blocked by security screening | +| `ATTACHMENT_TOO_LARGE` | 400 | Individual attachment exceeds 10 MB size limit | +| `ATTACHMENT_INLINE_TOO_LARGE` | 400 | Inline attachment exceeds 500 KB limit (use presigned upload) | +| `ATTACHMENTS_INLINE_TOTAL_TOO_LARGE` | 400 | Total inline attachment size exceeds 3 MB limit | +| `ATTACHMENTS_TOTAL_TOO_LARGE` | 400 | Total attachment size exceeds 50 MB limit | +| `ATTACHMENT_INVALID_TYPE` | 400 | MIME type not in allowlist | +| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or could not be sanitized | +| `ATTACHMENT_INVALID_FILENAME` | 400 | Filename contains invalid characters or path traversal | +| `ATTACHMENT_SIZE_MISMATCH` | 400 | Uploaded file size does not match declared `expected_size_bytes` | +| `ATTACHMENT_FETCH_FAILED` | 422 | URL attachment could not be fetched (timeout, DNS, SSRF blocked) | +| `ATTACHMENT_BUDGET_EXCEEDED` | 422 | Image attachments exceed token budget | +| `ATTACHMENT_UNSUPPORTED_AGENT` | 422 | Agent runtime version does not support attachments | +| `UPLOADS_NOT_CONFIRMED` | 409 | Task is in PENDING_UPLOADS but confirm-uploads not yet called | +| `UPLOADS_EXPIRED` | 410 | Upload window expired (30 minutes); re-submit the task | +| `ATTACHMENT_SCREENING_UNAVAILABLE` | 503 | Screening service unavailable after retries | | `GITHUB_UNREACHABLE` | 502 | GitHub API unreachable during pre-flight (transient) | | `RATE_LIMIT_EXCEEDED` | 429 | User exceeded rate limit | | `CONCURRENCY_LIMIT_EXCEEDED` | 409 | User at max concurrent tasks | diff --git a/docs/src/content/docs/architecture/Architecture.md b/docs/src/content/docs/architecture/Architecture.md index a67249f8..ffe3ead9 100644 --- a/docs/src/content/docs/architecture/Architecture.md +++ b/docs/src/content/docs/architecture/Architecture.md @@ -30,7 +30,7 @@ flowchart LR ``` 1. **Admission** (deterministic) - The orchestrator validates the request, checks concurrency limits, and loads the repository's Blueprint configuration. -2. **Context hydration** (deterministic) - The platform fetches external data (GitHub issue body, PR diff, review comments), loads memory from past tasks, and assembles the full prompt. For PR tasks, the prompt is screened through Bedrock Guardrails. +2. **Context hydration** (deterministic) - The platform fetches external data (GitHub issue body, PR diff, review comments), loads memory from past tasks, resolves attachments (URL fetch with SSRF protection, screening, upload to S3), and assembles the full prompt. For PR tasks, the prompt is screened through Bedrock Guardrails. 3. **Pre-flight checks** (deterministic) - GitHub API reachability and repository access are verified. Doomed tasks fail fast with a clear reason before consuming compute. 4. **Agent execution** (agentic) - The agent runs in an isolated compute environment: clone repo, create branch, edit code, commit, run tests, create PR. The orchestrator polls for completion without blocking. 5. **Finalization** (deterministic) - The orchestrator infers the result (PR created or not), writes memory, updates task status, and releases concurrency. diff --git a/docs/src/content/docs/architecture/Security.md b/docs/src/content/docs/architecture/Security.md index 7f31ba0a..0da72358 100644 --- a/docs/src/content/docs/architecture/Security.md +++ b/docs/src/content/docs/architecture/Security.md @@ -48,15 +48,17 @@ Input screening happens at two points in the pipeline, forming a defense-in-dept ### Submission-time screening -- **Input validation** - Required fields, types, and size limits are enforced before any processing. Task descriptions are capped at 2,000 characters. -- **Bedrock Guardrails** - A `PROMPT_ATTACK` content filter at `HIGH` strength screens task descriptions for prompt injection. +- **Input validation** - Required fields, types, and size limits are enforced before any processing. Task descriptions are capped at 10,000 characters. +- **Bedrock Guardrails** - A `PROMPT_ATTACK` content filter at `MEDIUM` input strength screens task descriptions for prompt injection. +- **Attachment screening** - All attachments (images, text files, URLs) pass through security screening before reaching the agent. Images are validated via magic bytes, screened through Bedrock Guardrails (image content blocks), stripped of EXIF/metadata, and re-encoded. Text files and PDFs are extracted and screened through Bedrock Guardrails text content screening. URL attachments undergo SSRF protection (DNS resolution pinning, private IP blocking, redirect validation) and content screening during hydration. See [ATTACHMENTS.md](/architecture/attachments) for the full screening pipeline. - **Fail-closed** - If the Bedrock API is unavailable, submissions are rejected (HTTP 503). Unscreened content never reaches the agent. ### Hydration-time screening - **PR tasks** (`pr_iteration`, `pr_review`) - The assembled prompt (PR body, review comments, diff, task description) is screened through Bedrock Guardrails before the agent receives it. - **`new_task` with issue content** - The assembled prompt (issue body, comments, task description) is screened. When no issue content is present, hydration-time screening is skipped because the task description was already screened at submission. -- **Fail-closed** - A Bedrock outage during hydration fails the task. A `guardrail_blocked` event is emitted when content is blocked. +- **URL attachments** - URL attachments are fetched during hydration with full SSRF protection (DNS resolution pinning to prevent rebinding attacks, private IP blocking, redirect validation, timeout enforcement). Fetched content is screened through the same pipeline as inline attachments before being stored in S3. +- **Fail-closed** - A Bedrock outage during hydration fails the task. A `guardrail_blocked` event is emitted when content is blocked. Attachment resolution errors (fetch failures, screening blocks, integrity failures) also fail the task — attachments are never silently dropped. ### Tool access control diff --git a/docs/src/content/docs/using/Using-the-cli.md b/docs/src/content/docs/using/Using-the-cli.md index ada4d059..f1782920 100644 --- a/docs/src/content/docs/using/Using-the-cli.md +++ b/docs/src/content/docs/using/Using-the-cli.md @@ -44,6 +44,15 @@ node lib/bin/bgagent.js submit --repo owner/repo --review-pr 55 # Review a PR with a specific focus area node lib/bin/bgagent.js submit --repo owner/repo --review-pr 55 --task "Focus on security and error handling" +# Submit with attachments (local files) +node lib/bin/bgagent.js submit --repo owner/repo --task "Fix this bug" \ + --attachment screenshot.png \ + --attachment error.log + +# Submit with a URL attachment +node lib/bin/bgagent.js submit --repo owner/repo --task "Implement this design" \ + --attachment https://figma.com/file/abc123/export.png + # Submit and wait for completion node lib/bin/bgagent.js submit --repo owner/repo --issue 42 --wait ``` @@ -72,6 +81,7 @@ Created: 2026-04-01T00:39:51.271Z | `--task` | Task description text. | | `--pr` | PR number to iterate on. Sets task type to `pr_iteration`. The agent checks out the PR's branch, reads review feedback, and pushes updates. | | `--review-pr` | PR number to review. Sets task type to `pr_review`. The agent checks out the PR's branch, analyzes changes read-only, and posts structured review comments. | +| `--attachment` | Attach a file or URL (repeatable). Local files ≤ 500 KB are sent inline; larger files use presigned upload. URLs are fetched during hydration. See [Attachments](#attachments) below. | | `--max-turns` | Maximum agent turns (1–500). Overrides per-repo Blueprint default. Platform default: 100. | | `--max-budget` | Maximum cost budget in USD (0.01–100). Overrides per-repo Blueprint default. No default limit. | | `--idempotency-key` | Idempotency key for deduplication. | @@ -83,6 +93,59 @@ Created: 2026-04-01T00:39:51.271Z At least one of `--issue`, `--task`, `--pr`, or `--review-pr` is required. The `--pr` and `--review-pr` flags are mutually exclusive. +### Attachments + +Attachments let you provide non-text context to the agent — screenshots of bugs, design mockups, CSV data, log files, or URLs to external resources. Every attachment passes through security screening before reaching the agent. + +**Supported file types:** + +| Category | Types | Extensions | +|---|---|---| +| Images | PNG, JPEG, GIF, WebP | `.png`, `.jpg`, `.gif`, `.webp` | +| Text files | Plain text, CSV, Markdown, JSON, PDF, Log | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | + +**Limits:** + +| Limit | Value | +|---|---| +| Max attachments per task | 10 | +| Max size per attachment | 10 MB | +| Max total size per task | 50 MB | +| URL attachments | HTTPS only | + +**Usage:** + +```bash +# Local file (auto-detects MIME type from content) +node lib/bin/bgagent.js submit --repo owner/repo --task "Fix this layout" \ + --attachment screenshot.png + +# Multiple attachments +node lib/bin/bgagent.js submit --repo owner/repo --task "Analyze these logs" \ + --attachment error.log \ + --attachment metrics.csv + +# URL attachment (fetched during task hydration) +node lib/bin/bgagent.js submit --repo owner/repo --task "Implement this design" \ + --attachment https://example.com/mockup.png +``` + +The CLI automatically routes attachments through the optimal upload path: + +- **Files ≤ 500 KB** are sent inline (base64-encoded in the request body). +- **Files > 500 KB** use presigned upload (uploaded directly to S3, then confirmed). +- **URLs** are validated at submission and fetched during context hydration with SSRF protection. + +**Security screening:** + +All attachments are screened before reaching the agent: + +- **Images**: Magic bytes validation, Bedrock Guardrail content screening (prompt attack detection), EXIF/metadata stripping, re-encoding to remove embedded payloads. +- **Text files**: Magic bytes validation, Bedrock Guardrail text content screening. PDFs have text extracted (max 50 pages) before screening. +- **URLs**: HTTPS-only enforcement, DNS resolution pinning (prevents DNS rebinding/SSRF), private IP blocking, redirect validation, size and timeout limits. + +If any attachment fails screening, the entire task is rejected with a clear error identifying the problematic file. Re-submit without the flagged attachment. + ### Checking task status Run these from the `cli/` directory (same as in **Setup**). From 3c5db571a4b131f02ecab4f085fe07a1c07bafe4 Mon Sep 17 00:00:00 2001 From: bgagent Date: Mon, 25 May 2026 20:50:31 -0500 Subject: [PATCH 09/19] chore(attachments): update roadmap --- docs/guides/ROADMAP.md | 2 +- docs/src/content/docs/roadmap/Roadmap.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guides/ROADMAP.md b/docs/guides/ROADMAP.md index 94839162..a4da446c 100644 --- a/docs/guides/ROADMAP.md +++ b/docs/guides/ROADMAP.md @@ -156,7 +156,7 @@ Planned capabilities, grouped by theme. Items are independent and may ship in an | Capability | Description | |------------|-------------| -| **Task attachments (multimodal)** | Implement end-to-end support for the create-task **`attachments`** array (`API_CONTRACT.md`: `image`, `file`, `url` — inline base64 or fetchable URL, size/MIME limits). Flow through validation, guardrails, context hydration, and agent prompt so images (screenshots, mockups), documents, and linked assets reach the model where the channel allows it. Extend **CLI** and **webhook** task creation to populate the same schema. *Multimodal* is the vision/image path; attachments are the unified carrier for all non-text task context. | +| ~~**Task attachments (multimodal)**~~ | **Implemented.** End-to-end support for the `attachments` array: inline base64 (≤ 500 KB), presigned upload (up to 10 MB), and URL fetch with SSRF protection. Images (PNG, JPEG, GIF, WebP) and text files (TXT, CSV, MD, JSON, PDF, LOG) pass through Bedrock Guardrail screening, magic bytes validation, EXIF stripping, and re-encoding. CLI `--attachment` flag, Slack file uploads, and Linear image extraction all feed the same schema. See [ATTACHMENTS.md](../design/ATTACHMENTS.md). | | **Additional git providers** | GitLab (and optionally Bitbucket). Same workflow, provider-specific API adapters. | | **Slack integration** | Submit tasks, check status, receive notifications from Slack. Block Kit rendering. | | **Control panel** | Web UI: task list, task detail with logs/traces, cancel, metrics dashboards, cost attribution. | diff --git a/docs/src/content/docs/roadmap/Roadmap.md b/docs/src/content/docs/roadmap/Roadmap.md index f019b4e2..8cdf8cf4 100644 --- a/docs/src/content/docs/roadmap/Roadmap.md +++ b/docs/src/content/docs/roadmap/Roadmap.md @@ -160,7 +160,7 @@ Planned capabilities, grouped by theme. Items are independent and may ship in an | Capability | Description | |------------|-------------| -| **Task attachments (multimodal)** | Implement end-to-end support for the create-task **`attachments`** array (`API_CONTRACT.md`: `image`, `file`, `url` — inline base64 or fetchable URL, size/MIME limits). Flow through validation, guardrails, context hydration, and agent prompt so images (screenshots, mockups), documents, and linked assets reach the model where the channel allows it. Extend **CLI** and **webhook** task creation to populate the same schema. *Multimodal* is the vision/image path; attachments are the unified carrier for all non-text task context. | +| ~~**Task attachments (multimodal)**~~ | **Implemented.** End-to-end support for the `attachments` array: inline base64 (≤ 500 KB), presigned upload (up to 10 MB), and URL fetch with SSRF protection. Images (PNG, JPEG, GIF, WebP) and text files (TXT, CSV, MD, JSON, PDF, LOG) pass through Bedrock Guardrail screening, magic bytes validation, EXIF stripping, and re-encoding. CLI `--attachment` flag, Slack file uploads, and Linear image extraction all feed the same schema. See [ATTACHMENTS.md](/architecture/attachments). | | **Additional git providers** | GitLab (and optionally Bitbucket). Same workflow, provider-specific API adapters. | | **Slack integration** | Submit tasks, check status, receive notifications from Slack. Block Kit rendering. | | **Control panel** | Web UI: task list, task detail with logs/traces, cancel, metrics dashboards, cost attribution. | From 83d31f7999b7ab8eee82e9a30cbc3660c85b57cb Mon Sep 17 00:00:00 2001 From: bgagent Date: Mon, 25 May 2026 21:46:56 -0500 Subject: [PATCH 10/19] fix(pr): address automated review comments --- cdk/src/constructs/task-api.ts | 98 ++++++++++---- cdk/src/handlers/cleanup-pending-uploads.ts | 11 ++ cdk/src/handlers/confirm-uploads.ts | 120 ++++++++++++++---- .../handlers/shared/attachment-screening.ts | 62 +++++++-- cdk/src/handlers/shared/create-task-core.ts | 12 +- .../shared/resolve-url-attachments.ts | 23 +++- cdk/src/handlers/slack-command-processor.ts | 8 ++ .../shared/attachment-screening.test.ts | 8 ++ cli/src/commands/submit.ts | 33 ++--- docs/design/ATTACHMENTS.md | 4 +- .../content/docs/architecture/Attachments.md | 4 +- 11 files changed, 291 insertions(+), 92 deletions(-) diff --git a/cdk/src/constructs/task-api.ts b/cdk/src/constructs/task-api.ts index 451e7ea9..fde69652 100644 --- a/cdk/src/constructs/task-api.ts +++ b/cdk/src/constructs/task-api.ts @@ -262,37 +262,87 @@ export class TaskApi extends Construct { }, rules: [ { - name: 'AWSManagedRulesCommonRuleSet', + // CRS for task paths that accept large bodies (inline base64 + // attachments up to 3 MB, presigned upload metadata). Excludes + // SizeRestrictions_BODY only; all other CRS rules apply. Payload + // size is bounded by API GW (10 MB) and validateAttachments(). + name: 'AWSManagedRulesCommonRuleSet-TaskPaths', priority: 1, overrideAction: { none: {} }, statement: { managedRuleGroupStatement: { vendorName: 'AWS', name: 'AWSManagedRulesCommonRuleSet', - // Inbound webhook payloads from mature SaaS tools (Linear ships - // full Issue payloads > 8 KB) trip SizeRestrictions_BODY in this - // ruleset. Exempt the Linear webhook path from CRS entirely: - // the route is HMAC-verified in the Lambda, parsed as strict - // JSON, never interpolated into SQL/HTML, and rate-limited by - // the priority-3 rule below. - // - // Inline task attachments (base64 in POST /v1/tasks, up to 3 MB - // total per ATTACHMENTS.md) also exceed the CRS 8 KB body cap and - // surface as API Gateway 403 {"message":"Forbidden"} before the - // create-task Lambda runs. Drop only SizeRestrictions_BODY here; - // other CRS rules still apply. Payload size is bounded by API GW - // (10 MB) and validateAttachments() in the handler. excludedRules: [{ name: 'SizeRestrictions_BODY' }], scopeDownStatement: { - notStatement: { - statement: { - byteMatchStatement: { - fieldToMatch: { uriPath: {} }, - positionalConstraint: 'EXACTLY', - searchString: '/v1/linear/webhook', - textTransformations: [{ priority: 0, type: 'NONE' }], + orStatement: { + statements: [ + { + byteMatchStatement: { + fieldToMatch: { uriPath: {} }, + positionalConstraint: 'STARTS_WITH', + searchString: '/v1/tasks', + textTransformations: [{ priority: 0, type: 'NONE' }], + }, + }, + { + // Linear webhook payloads > 8 KB (HMAC-verified in Lambda, + // rate-limited by priority-4 rule below). + byteMatchStatement: { + fieldToMatch: { uriPath: {} }, + positionalConstraint: 'EXACTLY', + searchString: '/v1/linear/webhook', + textTransformations: [{ priority: 0, type: 'NONE' }], + }, }, - }, + ], + }, + }, + }, + }, + visibilityConfig: { + cloudWatchMetricsEnabled: true, + metricName: 'CommonRuleSetTaskPaths', + sampledRequestsEnabled: true, + }, + }, + { + // Full CRS (including SizeRestrictions_BODY) for all other paths. + name: 'AWSManagedRulesCommonRuleSet', + priority: 2, + overrideAction: { none: {} }, + statement: { + managedRuleGroupStatement: { + vendorName: 'AWS', + name: 'AWSManagedRulesCommonRuleSet', + scopeDownStatement: { + andStatement: { + statements: [ + { + notStatement: { + statement: { + byteMatchStatement: { + fieldToMatch: { uriPath: {} }, + positionalConstraint: 'STARTS_WITH', + searchString: '/v1/tasks', + textTransformations: [{ priority: 0, type: 'NONE' }], + }, + }, + }, + }, + { + notStatement: { + statement: { + byteMatchStatement: { + fieldToMatch: { uriPath: {} }, + positionalConstraint: 'EXACTLY', + searchString: '/v1/linear/webhook', + textTransformations: [{ priority: 0, type: 'NONE' }], + }, + }, + }, + }, + ], }, }, }, @@ -305,7 +355,7 @@ export class TaskApi extends Construct { }, { name: 'AWSManagedRulesKnownBadInputsRuleSet', - priority: 2, + priority: 3, overrideAction: { none: {} }, statement: { managedRuleGroupStatement: { @@ -321,7 +371,7 @@ export class TaskApi extends Construct { }, { name: 'RateLimitRule', - priority: 3, + priority: 4, action: { block: {} }, statement: { rateBasedStatement: { diff --git a/cdk/src/handlers/cleanup-pending-uploads.ts b/cdk/src/handlers/cleanup-pending-uploads.ts index 23a3f7fe..1b8d68c4 100644 --- a/cdk/src/handlers/cleanup-pending-uploads.ts +++ b/cdk/src/handlers/cleanup-pending-uploads.ts @@ -272,5 +272,16 @@ export async function handler(): Promise { cancelled, raced, errored, + metric_type: errored > 0 ? 'pending_upload_cleanup_errors' : undefined, }); + + // If ALL tasks errored and none were cancelled, throw so EventBridge sees + // a Lambda failure and CloudWatch alarms fire. Partial success (some cancelled, + // some errored) is acceptable — the next scheduled run will retry the failed ones. + if (errored > 0 && cancelled === 0 && raced === 0) { + throw new Error( + `All ${errored} expired PENDING_UPLOADS task(s) failed to process. ` + + 'Investigate DynamoDB/S3 connectivity — abandoned tasks will not auto-cancel until resolved.', + ); + } } diff --git a/cdk/src/handlers/confirm-uploads.ts b/cdk/src/handlers/confirm-uploads.ts index 6cab59bd..9b8163b7 100644 --- a/cdk/src/handlers/confirm-uploads.ts +++ b/cdk/src/handlers/confirm-uploads.ts @@ -151,6 +151,16 @@ export async function handler(event: APIGatewayProxyEvent, context: Context): Pr }); } + // 6b. Pre-check concurrency before expensive screening (fail-fast). + // The actual atomic increment happens in transitionToSubmitted after screening + // passes. This read-only check avoids wasting Bedrock calls when the user is + // already at their concurrency limit. + const preCheckAdmitted = await preCheckConcurrency(task.user_id); + if (!preCheckAdmitted) { + return errorResponse(429, ErrorCode.RATE_LIMIT_EXCEEDED, + 'User concurrency limit reached. Wait for a running task to finish or cancel one, then retry.', requestId); + } + // 7. Screen attachments in parallel with bounded concurrency const screeningConfig = await buildScreeningConfig(); if (!screeningConfig) { @@ -303,6 +313,13 @@ async function screenSingleAttachment( } const content = Buffer.from(await getResult.Body.transformToByteArray()); + if (content.length !== sizeBytes) { + throw new AttachmentScreeningError( + `Upload for '${att.filename}' size mismatch (expected ${sizeBytes} bytes, read ${content.length}). ` + + 'Please re-upload the file and try again.', + ); + } + // Screen based on type const isImage = att.type === 'image'; const screenResult = isImage @@ -443,6 +460,9 @@ async function transitionToSubmitted( } return errorResponse(404, ErrorCode.TASK_NOT_FOUND, 'Task not found.', requestId); } + // Roll back concurrency counter on any other DDB error (throttling, + // network timeout, etc.) to prevent permanent slot leaks. + await decrementConcurrency(task.user_id); throw err; } @@ -471,6 +491,7 @@ async function transitionToSubmitted( } // Invoke orchestrator (fire-and-forget) + let orchestratorInvokeFailed = false; if (lambdaClient && process.env.ORCHESTRATOR_FUNCTION_ARN) { try { await lambdaClient.send(new InvokeCommand({ @@ -483,7 +504,8 @@ async function transitionToSubmitted( request_id: requestId, }); } catch (orchErr) { - logger.error('Failed to invoke orchestrator after confirm-uploads — task may be stuck in SUBMITTED until EventBridge retry', { + orchestratorInvokeFailed = true; + logger.error('Failed to invoke orchestrator after confirm-uploads — task will be picked up by StrandedTaskReconciler', { error: orchErr instanceof Error ? orchErr.message : String(orchErr), task_id: taskId, request_id: requestId, @@ -499,7 +521,12 @@ async function transitionToSubmitted( attachments: finalAttachments, updated_at: now, }; - return successResponse(200, toTaskDetail(updatedTask), requestId); + const responseBody = toTaskDetail(updatedTask); + if (orchestratorInvokeFailed) { + (responseBody as any).warning = 'Task was submitted successfully but orchestration dispatch failed. ' + + 'The task will be picked up automatically within minutes by the background reconciler.'; + } + return successResponse(200, responseBody, requestId); } // --------------------------------------------------------------------------- @@ -574,26 +601,32 @@ async function failTaskOnScreening( async function cleanupAllAttachments(task: TaskRecord, taskId: string): Promise { if (!task.attachments || task.attachments.length === 0) return; - const keys = task.attachments.map(att => - `${ATTACHMENT_OBJECT_KEY_PREFIX}${task.user_id}/${taskId}/${att.attachment_id}/${att.filename}`, - ); + // Include VersionId when available — in a versioned bucket, DeleteObjects + // without VersionId only creates a delete marker, leaving the actual content + // accessible until the 7-day noncurrent lifecycle runs. + const objects = task.attachments.map(att => ({ + Key: `${ATTACHMENT_OBJECT_KEY_PREFIX}${task.user_id}/${taskId}/${att.attachment_id}/${att.filename}`, + ...(att.s3_version_id && att.s3_version_id !== 'unversioned' && { VersionId: att.s3_version_id }), + })); try { const result = await s3Client.send(new DeleteObjectsCommand({ Bucket: ATTACHMENTS_BUCKET, - Delete: { Objects: keys.map(Key => ({ Key })) }, + Delete: { Objects: objects }, })); if (result.Errors && result.Errors.length > 0) { logger.error('Partial cleanup failure in confirm-uploads', { task_id: taskId, failedKeys: result.Errors.map(e => e.Key), + metric_type: 'cleanup_failure_blocked_content', }); } } catch (err) { logger.error('S3 cleanup failed in confirm-uploads — 90-day lifecycle is safety net', { task_id: taskId, - keys, + object_count: objects.length, error: String(err), + metric_type: 'cleanup_failure_blocked_content', }); } } @@ -673,6 +706,30 @@ async function buildScreeningConfig(): Promise { }; } +/** + * Non-mutating read to check if the user is at their concurrency limit. + * Used as a fast pre-check before expensive screening; the actual atomic + * increment happens in checkConcurrency() during transitionToSubmitted. + */ +async function preCheckConcurrency(userId: string): Promise { + try { + const result = await ddb.send(new GetCommand({ + TableName: CONCURRENCY_TABLE_NAME, + Key: { user_id: userId }, + })); + const activeCount = (result.Item?.active_count as number) ?? 0; + return activeCount < MAX_CONCURRENT; + } catch (err) { + // On error reading concurrency, allow the request to proceed — + // the atomic check in transitionToSubmitted is the authoritative gate. + logger.warn('Pre-check concurrency read failed — allowing request to proceed', { + user_id: userId, + error: err instanceof Error ? err.message : String(err), + }); + return true; + } +} + async function checkConcurrency(userId: string): Promise { try { await ddb.send(new UpdateCommand({ @@ -697,26 +754,37 @@ async function checkConcurrency(userId: string): Promise { } async function decrementConcurrency(userId: string): Promise { - try { - await ddb.send(new UpdateCommand({ - TableName: CONCURRENCY_TABLE_NAME, - Key: { user_id: userId }, - UpdateExpression: 'SET active_count = active_count - :one, updated_at = :now', - ConditionExpression: 'attribute_exists(active_count) AND active_count > :zero', - ExpressionAttributeValues: { - ':one': 1, - ':zero': 0, - ':now': new Date().toISOString(), - }, - })); - } catch (err: any) { - if (err.name === 'ConditionalCheckFailedException') { - // Counter already at 0 or doesn't exist — nothing to roll back + const maxAttempts = 3; + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + await ddb.send(new UpdateCommand({ + TableName: CONCURRENCY_TABLE_NAME, + Key: { user_id: userId }, + UpdateExpression: 'SET active_count = active_count - :one, updated_at = :now', + ConditionExpression: 'attribute_exists(active_count) AND active_count > :zero', + ExpressionAttributeValues: { + ':one': 1, + ':zero': 0, + ':now': new Date().toISOString(), + }, + })); return; + } catch (err: any) { + if (err.name === 'ConditionalCheckFailedException') { + // Counter already at 0 or doesn't exist — nothing to roll back + return; + } + if (attempt < maxAttempts - 1) { + // Retry transient DDB errors (throttling, network) with backoff + await new Promise(resolve => setTimeout(resolve, 100 * Math.pow(2, attempt))); + continue; + } + logger.error('Failed to decrement concurrency counter after retries (leak possible)', { + user_id: userId, + attempts: maxAttempts, + error: err instanceof Error ? err.message : String(err), + metric_type: 'concurrency_counter_leak', + }); } - logger.error('Failed to decrement concurrency counter (leak possible)', { - user_id: userId, - error: err instanceof Error ? err.message : String(err), - }); } } diff --git a/cdk/src/handlers/shared/attachment-screening.ts b/cdk/src/handlers/shared/attachment-screening.ts index 7ddb92ab..611062c0 100644 --- a/cdk/src/handlers/shared/attachment-screening.ts +++ b/cdk/src/handlers/shared/attachment-screening.ts @@ -35,6 +35,9 @@ export const MAX_ATTACHMENT_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB /** Bedrock Guardrail image filter max side (docs: 8000x8000). */ export const MAX_IMAGE_DIMENSION_PX = 8000; +const PNG_FILE_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const JPEG_FILE_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]); + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -125,6 +128,7 @@ export async function screenImage( filename: string, config: ScreeningConfig, ): Promise { + assertImageUploadBytes(content, contentType, filename); await assertImageDecodable(content, contentType, filename); // Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) @@ -338,6 +342,47 @@ async function extractPdfText(content: Buffer, filename: string): Promise 0 && s3Client && ATTACHMENTS_BUCKET) { - // Build screening config — fail-closed if guardrail is not configured + // Build screening config — fail-closed if guardrail is not configured. + // Reject inline AND presigned attachments upfront: presigned attachments will + // need screening in confirm-uploads, so rejecting early prevents the user from + // wasting time uploading files that can never be screened. if (!bedrockClient || !process.env.GUARDRAIL_ID || !process.env.GUARDRAIL_VERSION) { - const hasInline = validatedAttachments.some(a => a.delivery === 'inline'); - if (hasInline) { - logger.error('Inline attachment submitted but guardrail is not configured (fail-closed)', { + const hasScreenable = validatedAttachments.some(a => a.delivery === 'inline' || a.delivery === 'presigned'); + if (hasScreenable) { + logger.error('Attachment submitted but guardrail is not configured (fail-closed)', { request_id: requestId, + delivery_types: [...new Set(validatedAttachments.map(a => a.delivery))], }); return errorResponse(503, ErrorCode.ATTACHMENT_SCREENING_UNAVAILABLE, 'Attachment content screening is not configured. Please contact your administrator.', requestId); diff --git a/cdk/src/handlers/shared/resolve-url-attachments.ts b/cdk/src/handlers/shared/resolve-url-attachments.ts index 292cde21..70f99081 100644 --- a/cdk/src/handlers/shared/resolve-url-attachments.ts +++ b/cdk/src/handlers/shared/resolve-url-attachments.ts @@ -33,7 +33,6 @@ * Tests: cdk/test/handlers/shared/resolve-url-attachments.test.ts */ -import { createHash } from 'crypto'; import { promises as dns } from 'dns'; import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { screenImage, screenTextFile, AttachmentScreeningError, type ScreeningConfig } from './attachment-screening'; @@ -41,6 +40,7 @@ import { AttachmentResolutionError } from './context-hydration'; import { estimateImageTokensFromBuffer } from './image-tokens'; import { logger } from './logger'; import { createAttachmentRecord, type AttachmentRecord } from './types'; +import { isAllowedMimeType, validateMagicBytes } from './validation'; import { ATTACHMENT_OBJECT_KEY_PREFIX } from '../../constructs/attachments-bucket'; // --------------------------------------------------------------------------- @@ -383,6 +383,23 @@ export async function resolveUrlAttachments( const isImage = finalContentType.startsWith('image/'); const resolvedContentType = att.content_type || finalContentType; + // Validate content-type is in the allowlist (attacker controls the response header) + const attachmentType = isImage ? 'image' : 'file'; + if (!isAllowedMimeType(resolvedContentType, attachmentType)) { + throw new AttachmentResolutionError( + `URL attachment '${att.filename}' returned unsupported content type '${resolvedContentType}'. ` + + 'Only supported image and text file types are allowed.', + ); + } + + // Validate magic bytes match declared content type (prevents polyglot/masquerade) + if (!validateMagicBytes(content, resolvedContentType)) { + throw new AttachmentResolutionError( + `URL attachment '${att.filename}' content does not match declared type '${resolvedContentType}'. ` + + 'The file may be corrupt or masquerading as a different type.', + ); + } + // Screen the fetched content let screenResult; try { @@ -417,8 +434,8 @@ export async function resolveUrlAttachments( ContentType: resolvedContentType, })); - // Compute checksum - const checksum = createHash('sha256').update(screenResult.content).digest('hex'); + // Use checksum from screening (already computed over the cleaned content) + const checksum = screenResult.checksum; // Estimate token cost for images let tokenEstimate: number | undefined; diff --git a/cdk/src/handlers/slack-command-processor.ts b/cdk/src/handlers/slack-command-processor.ts index 682c3718..f9a72ec2 100644 --- a/cdk/src/handlers/slack-command-processor.ts +++ b/cdk/src/handlers/slack-command-processor.ts @@ -441,6 +441,14 @@ async function extractSlackFileAttachments( const buffer = Buffer.from(await response.arrayBuffer()); + // Post-download size validation: Slack's declared file.size may differ + // from the actual download (e.g., server-side processing, bug, or manipulation). + if (buffer.length > SLACK_FILE_MAX_SIZE_BYTES) { + const sizeMb = (buffer.length / (1024 * 1024)).toFixed(1); + errors.push(`\`${file.name}\` (downloaded size ${sizeMb} MB exceeds 10 MB limit)`); + continue; + } + attachments.push({ type: isImage ? 'image' : 'file', content_type: mime, diff --git a/cdk/test/handlers/shared/attachment-screening.test.ts b/cdk/test/handlers/shared/attachment-screening.test.ts index 0b8d83aa..37b3a03b 100644 --- a/cdk/test/handlers/shared/attachment-screening.test.ts +++ b/cdk/test/handlers/shared/attachment-screening.test.ts @@ -21,6 +21,7 @@ import * as fs from 'fs'; import * as path from 'path'; import type { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; import { + assertImageUploadBytes, AttachmentScreeningError, screenImage, } from '../../../src/handlers/shared/attachment-screening'; @@ -45,6 +46,13 @@ function mockBedrockPass(): BedrockRuntimeClient { } as unknown as BedrockRuntimeClient; } +describe('assertImageUploadBytes', () => { + test('rejects non-PNG bytes for image/png', () => { + expect(() => assertImageUploadBytes(Buffer.from('not a png'), 'image/png', 'x.png')) + .toThrow(AttachmentScreeningError); + }); +}); + describe('screenImage', () => { const config = { bedrockClient: mockBedrockPass(), diff --git a/cli/src/commands/submit.ts b/cli/src/commands/submit.ts index 23030a72..fd2bbf36 100644 --- a/cli/src/commands/submit.ts +++ b/cli/src/commands/submit.ts @@ -310,41 +310,28 @@ function resolveAttachmentArg(arg: string): Attachment { /** * Upload a local file to S3 via a presigned POST (multipart/form-data). - * The presigned fields (policy, signature, etc.) are included as form fields - * before the file content, as required by S3's POST Object API. + * Policy fields from the API must precede the file; use FormData so Node sets + * the boundary and Content-Length correctly for multi-megabyte payloads. */ async function uploadViaPresignedPost( filePath: string, instruction: AttachmentUploadInstruction, ): Promise { const fileData = fs.readFileSync(filePath); - const boundary = `----AttachmentBoundary${Date.now()}${Math.random().toString(36).slice(2)}`; - - // Build multipart/form-data body: presigned fields first, then the file. - const parts: Buffer[] = []; - - for (const [key, value] of Object.entries(instruction.upload_fields)) { - parts.push(Buffer.from( - `--${boundary}\r\nContent-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`, - )); - } - - // The file field must be last per S3 POST Object requirements. const ext = path.extname(filePath).toLowerCase(); const contentType = MIME_BY_EXT[ext] ?? 'application/octet-stream'; - parts.push(Buffer.from( - `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${path.basename(filePath)}"\r\n` - + `Content-Type: ${contentType}\r\n\r\n`, - )); - parts.push(fileData); - parts.push(Buffer.from(`\r\n--${boundary}--\r\n`)); - const body = Buffer.concat(parts); + const form = new FormData(); + for (const [key, value] of Object.entries(instruction.upload_fields)) { + form.append(key, value); + } + // File must be last. Object Content-Type comes from the policy field above, + // not the multipart part headers (per S3 POST Object). + form.append('file', new Blob([fileData], { type: contentType }), path.basename(filePath)); const res = await fetch(instruction.upload_url, { method: 'POST', - headers: { 'Content-Type': `multipart/form-data; boundary=${boundary}` }, - body, + body: form, }); if (!res.ok) { diff --git a/docs/design/ATTACHMENTS.md b/docs/design/ATTACHMENTS.md index ae787502..ca76ee87 100644 --- a/docs/design/ATTACHMENTS.md +++ b/docs/design/ATTACHMENTS.md @@ -232,6 +232,7 @@ sequenceDiagram C->>GW: POST /v1/tasks/{task_id}/confirm-uploads GW->>H: Forward H->>H: Check status == PENDING_UPLOADS (short-circuit if already SUBMITTED) + H->>DB: Pre-check user concurrency (read-only, fail-fast before expensive screening) H->>S3: HeadObject per attachment (verify uploads exist, with retry for 404) loop Each attachment (parallel, bounded concurrency 3) H->>S3: GetObject (stream content for screening) @@ -359,6 +360,7 @@ sequenceDiagram Note over O: During context hydration O->>O: SSRF check (DNS resolve → reject private IPs → connect to resolved IP) O->>O: Fetch URL content (with timeout + size limit) + O->>O: Validate content-type allowlist + magic bytes (prevents polyglot/masquerade) O->>SCR: Screen fetched content (with retry) SCR-->>O: Pass / Blocked alt Screening passed @@ -1578,7 +1580,7 @@ New event types in `TaskEventsTable`: |---|---|---| | Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; Bedrock image screening; EXIF stripping; image re-encoding through `sharp` strips embedded payloads; sharp failure → reject | | Prompt injection via file content | Text file containing adversarial instructions | Magic bytes validation; Bedrock Guardrail text screening with retry (same as task descriptions); content trust tagging as `untrusted-external` | -| SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time | +| SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time; content-type allowlist + magic bytes validation after fetch (prevents attacker-controlled Content-Type header from bypassing type restrictions) | | Data exfiltration via URL attachment | URL pointing to attacker-controlled server (leaks request headers/IP) | No auth headers sent to non-GitHub URLs; minimal request headers; no cookies | | Denial of service via large attachments | Many large base64 payloads | 500 KB inline limit; 3 MB total inline; 10 MB per-attachment; 50 MB total; 10 count limit; 6 MB Lambda payload limit | | Path traversal via filename | `filename: "../../etc/passwd"` | Filename sanitization regex; reject path separators, dots-prefix, null bytes; use `attachment_id` as primary path component | diff --git a/docs/src/content/docs/architecture/Attachments.md b/docs/src/content/docs/architecture/Attachments.md index 7604e240..5f32dd8d 100644 --- a/docs/src/content/docs/architecture/Attachments.md +++ b/docs/src/content/docs/architecture/Attachments.md @@ -236,6 +236,7 @@ sequenceDiagram C->>GW: POST /v1/tasks/{task_id}/confirm-uploads GW->>H: Forward H->>H: Check status == PENDING_UPLOADS (short-circuit if already SUBMITTED) + H->>DB: Pre-check user concurrency (read-only, fail-fast before expensive screening) H->>S3: HeadObject per attachment (verify uploads exist, with retry for 404) loop Each attachment (parallel, bounded concurrency 3) H->>S3: GetObject (stream content for screening) @@ -363,6 +364,7 @@ sequenceDiagram Note over O: During context hydration O->>O: SSRF check (DNS resolve → reject private IPs → connect to resolved IP) O->>O: Fetch URL content (with timeout + size limit) + O->>O: Validate content-type allowlist + magic bytes (prevents polyglot/masquerade) O->>SCR: Screen fetched content (with retry) SCR-->>O: Pass / Blocked alt Screening passed @@ -1582,7 +1584,7 @@ New event types in `TaskEventsTable`: |---|---|---| | Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; Bedrock image screening; EXIF stripping; image re-encoding through `sharp` strips embedded payloads; sharp failure → reject | | Prompt injection via file content | Text file containing adversarial instructions | Magic bytes validation; Bedrock Guardrail text screening with retry (same as task descriptions); content trust tagging as `untrusted-external` | -| SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time | +| SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time; content-type allowlist + magic bytes validation after fetch (prevents attacker-controlled Content-Type header from bypassing type restrictions) | | Data exfiltration via URL attachment | URL pointing to attacker-controlled server (leaks request headers/IP) | No auth headers sent to non-GitHub URLs; minimal request headers; no cookies | | Denial of service via large attachments | Many large base64 payloads | 500 KB inline limit; 3 MB total inline; 10 MB per-attachment; 50 MB total; 10 count limit; 6 MB Lambda payload limit | | Path traversal via filename | `filename: "../../etc/passwd"` | Filename sanitization regex; reject path separators, dots-prefix, null bytes; use `attachment_id` as primary path component | From d0bc34c9b04d3ffcdea8cf85203cbcf0794dbb26 Mon Sep 17 00:00:00 2001 From: bgagent Date: Mon, 25 May 2026 23:37:54 -0500 Subject: [PATCH 11/19] chore(attachments): simplify validation and fix some runtime issues detected during testing --- agent/tests/test_attachments.py | 185 ++++++++++++++++ cdk/package.json | 2 +- cdk/src/constructs/task-api.ts | 11 +- cdk/src/constructs/task-orchestrator.ts | 7 +- cdk/src/handlers/cleanup-pending-uploads.ts | 5 +- cdk/src/handlers/confirm-uploads.ts | 30 ++- .../handlers/shared/attachment-screening.ts | 207 ++++++++---------- cdk/src/handlers/shared/create-task-core.ts | 20 +- cdk/src/handlers/shared/image-tokens.ts | 25 +-- .../shared/resolve-url-attachments.ts | 112 +++++++++- cdk/src/handlers/shared/sharp-loader.ts | 32 --- cdk/src/handlers/shared/validation.ts | 24 +- .../handlers/cleanup-pending-uploads.test.ts | 198 +++++++++++++++++ .../shared/attachment-screening.test.ts | 184 ++++++++++++++-- .../shared/resolve-url-attachments.test.ts | 114 ++++++++++ cdk/test/handlers/shared/validation.test.ts | 7 +- cli/src/commands/submit.ts | 44 +++- docs/design/API_CONTRACT.md | 2 +- docs/design/ATTACHMENTS.md | 120 +++------- docs/design/SECURITY.md | 2 +- docs/guides/USER_GUIDE.md | 4 +- .../content/docs/architecture/Api-contract.md | 2 +- .../content/docs/architecture/Attachments.md | 120 +++------- .../src/content/docs/architecture/Security.md | 2 +- docs/src/content/docs/using/Using-the-cli.md | 4 +- yarn.lock | 186 +--------------- 26 files changed, 1029 insertions(+), 620 deletions(-) create mode 100644 agent/tests/test_attachments.py delete mode 100644 cdk/src/handlers/shared/sharp-loader.ts create mode 100644 cdk/test/handlers/cleanup-pending-uploads.test.ts create mode 100644 cdk/test/handlers/shared/resolve-url-attachments.test.ts diff --git a/agent/tests/test_attachments.py b/agent/tests/test_attachments.py new file mode 100644 index 00000000..c779c478 --- /dev/null +++ b/agent/tests/test_attachments.py @@ -0,0 +1,185 @@ +"""Unit tests for attachments.py — download and integrity verification.""" + +import hashlib +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import ValidationError + +from attachments import ATTACHMENTS_DIR, PreparedAttachment, download_attachments +from models import AttachmentConfig + + +def _make_config( + content: bytes = b"hello world", + filename: str = "test.txt", + attachment_id: str = "ATT001", +) -> tuple[AttachmentConfig, bytes]: + """Create an AttachmentConfig with matching checksum.""" + checksum = hashlib.sha256(content).hexdigest() + config = AttachmentConfig( + attachment_id=attachment_id, + type="file", + content_type="text/plain", + filename=filename, + s3_uri="s3://test-bucket/attachments/user-1/task-1/ATT001/test.txt", + s3_version_id="v1", + size_bytes=len(content), + checksum_sha256=checksum, + ) + return config, content + + +class TestPreparedAttachment: + def test_frozen_model(self): + att = PreparedAttachment( + attachment_id="ATT001", + type="file", + content_type="text/plain", + filename="test.txt", + local_path="/tmp/test.txt", + size_bytes=100, + ) + with pytest.raises(ValidationError): + att.filename = "other.txt" + + def test_rejects_extra_fields(self): + with pytest.raises(ValidationError): + PreparedAttachment( + attachment_id="ATT001", + type="file", + content_type="text/plain", + filename="test.txt", + local_path="/tmp/test.txt", + size_bytes=100, + extra_field="bad", + ) + + +class TestDownloadAttachments: + def test_empty_list_returns_empty(self, tmp_path): + result = download_attachments([], str(tmp_path)) + assert result == [] + + @patch("boto3.client") + def test_successful_download_and_verify(self, mock_client, tmp_path): + config, content = _make_config() + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + mock_s3.get_object.return_value = {"Body": MagicMock(read=lambda: content)} + + result = download_attachments([config], str(tmp_path)) + + assert len(result) == 1 + assert result[0].filename == "test.txt" + assert result[0].size_bytes == len(content) + assert Path(result[0].local_path).exists() + assert Path(result[0].local_path).read_bytes() == content + + @patch("boto3.client") + def test_passes_version_id_to_s3(self, mock_client, tmp_path): + config, content = _make_config() + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + mock_s3.get_object.return_value = {"Body": MagicMock(read=lambda: content)} + + download_attachments([config], str(tmp_path)) + + mock_s3.get_object.assert_called_once_with( + Bucket="test-bucket", + Key="attachments/user-1/task-1/ATT001/test.txt", + VersionId="v1", + ) + + @patch("boto3.client") + def test_checksum_mismatch_raises(self, mock_client, tmp_path): + config, _ = _make_config() + tampered_content = b"tampered content" + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + mock_s3.get_object.return_value = { + "Body": MagicMock(read=lambda: tampered_content) + } + + with pytest.raises(RuntimeError, match="integrity check failed"): + download_attachments([config], str(tmp_path)) + + @patch("boto3.client") + def test_size_mismatch_raises(self, mock_client, tmp_path): + content = b"hello world" + checksum = hashlib.sha256(content).hexdigest() + + # Config says size is 5, but content is 11 bytes + config = AttachmentConfig( + attachment_id="ATT001", + type="file", + content_type="text/plain", + filename="test.txt", + s3_uri="s3://test-bucket/attachments/user-1/task-1/ATT001/test.txt", + s3_version_id="v1", + size_bytes=5, + checksum_sha256=checksum, + ) + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + mock_s3.get_object.return_value = {"Body": MagicMock(read=lambda: content)} + + with pytest.raises(RuntimeError, match="size mismatch"): + download_attachments([config], str(tmp_path)) + + @patch("boto3.client") + def test_file_written_read_only(self, mock_client, tmp_path): + config, content = _make_config() + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + mock_s3.get_object.return_value = {"Body": MagicMock(read=lambda: content)} + + result = download_attachments([config], str(tmp_path)) + + local_path = Path(result[0].local_path) + mode = os.stat(str(local_path)).st_mode & 0o777 + assert mode == 0o444 + + @patch("boto3.client") + def test_creates_per_attachment_subdirectory(self, mock_client, tmp_path): + config, content = _make_config() + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + mock_s3.get_object.return_value = {"Body": MagicMock(read=lambda: content)} + + result = download_attachments([config], str(tmp_path)) + + # File should be under .attachments// + local_path = Path(result[0].local_path) + assert local_path.parent.name == config.attachment_id + assert local_path.parent.parent.name == ATTACHMENTS_DIR + + @patch("boto3.client") + def test_multiple_attachments_all_verified(self, mock_client, tmp_path): + configs_and_contents = [ + _make_config(b"content 1", "file1.txt", "ATT001"), + _make_config(b"content 2", "file2.txt", "ATT002"), + ] + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + # Return different content for each call + mock_s3.get_object.side_effect = [ + {"Body": MagicMock(read=lambda: configs_and_contents[0][1])}, + {"Body": MagicMock(read=lambda: configs_and_contents[1][1])}, + ] + + result = download_attachments( + [c[0] for c in configs_and_contents], str(tmp_path) + ) + assert len(result) == 2 + assert result[0].filename == "file1.txt" + assert result[1].filename == "file2.txt" diff --git a/cdk/package.json b/cdk/package.json index 765ad06e..4a450795 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -33,7 +33,6 @@ "cdk-nag": "^2.37.55", "constructs": "^10.3.0", "pdf-parse": "^1.1.1", - "sharp": "^0.33.5", "js-yaml": "^4.1.1", "ulid": "^3.0.2" }, @@ -64,6 +63,7 @@ "resolutions": { "eslint-plugin-import/minimatch": "^3.1.2" }, + "optionalDependencies": {}, "engines": { "node": ">= 20.x <= 24.x" }, diff --git a/cdk/src/constructs/task-api.ts b/cdk/src/constructs/task-api.ts index fde69652..6ecd745a 100644 --- a/cdk/src/constructs/task-api.ts +++ b/cdk/src/constructs/task-api.ts @@ -437,11 +437,10 @@ export class TaskApi extends Construct { ], }; - // sharp ships native bindings; copy from node_modules (Docker bundling) - // instead of esbuild-inlining. Used by create-task / confirm-uploads paths. + // pdf-parse is used for PDF attachment screening (text extraction). const attachmentScreeningBundling: lambda.BundlingOptions = { ...commonBundling, - nodeModules: ['sharp'], + nodeModules: ['pdf-parse'], }; // --- Lambda handlers --- @@ -467,9 +466,7 @@ export class TaskApi extends Construct { architecture: Architecture.ARM_64, environment: createTaskEnv, bundling: attachmentScreeningBundling, - // Inline attachment screening (sharp) needs headroom; 256 MB caused - // cold-start init failures → API Gateway 502 on POST /tasks. - memorySize: 1024, + memorySize: 512, timeout: Duration.seconds(15), }); @@ -610,7 +607,7 @@ export class TaskApi extends Construct { architecture: Architecture.ARM_64, environment: confirmUploadsEnv, bundling: attachmentScreeningBundling, - memorySize: 2048, + memorySize: 1024, timeout: Duration.seconds(180), }); diff --git a/cdk/src/constructs/task-orchestrator.ts b/cdk/src/constructs/task-orchestrator.ts index 9731b571..f913b0b3 100644 --- a/cdk/src/constructs/task-orchestrator.ts +++ b/cdk/src/constructs/task-orchestrator.ts @@ -178,9 +178,8 @@ export class TaskOrchestrator extends Construct { const maxConcurrent = props.maxConcurrentTasksPerUser ?? 10; // Hydration pulls in bedrock-agentcore (bundled), durable-execution, and - // attachment screening (sharp via resolve-url-attachments). 256 MB OOMs - // at cold start / early steps → task stuck in SUBMITTED (async invoke - // retryAttempts: 0 on the alias). + // attachment screening (URL resolution). pdf-parse is needed for PDF text + // extraction during screening. const orchestratorBundling: lambda.BundlingOptions = { externalModules: [ '@aws-sdk/client-dynamodb', @@ -191,7 +190,7 @@ export class TaskOrchestrator extends Construct { '@aws-sdk/lib-dynamodb', '@aws-sdk/util-dynamodb', ], - nodeModules: ['sharp'], + nodeModules: ['pdf-parse'], }; this.fn = new lambda.NodejsFunction(this, 'OrchestratorFn', { diff --git a/cdk/src/handlers/cleanup-pending-uploads.ts b/cdk/src/handlers/cleanup-pending-uploads.ts index 1b8d68c4..60f649f2 100644 --- a/cdk/src/handlers/cleanup-pending-uploads.ts +++ b/cdk/src/handlers/cleanup-pending-uploads.ts @@ -128,15 +128,16 @@ async function cancelExpiredTask(task: ExpiredTask): Promise { Key: { task_id: { S: task.task_id } }, UpdateExpression: 'SET #s = :cancelled, updated_at = :now, completed_at = :now, ' - + 'error_message = :err, status_created_at = :sca', + + 'error_message = :err, status_created_at = :sca, #ttl = :ttl', ConditionExpression: '#s = :expected', - ExpressionAttributeNames: { '#s': 'status' }, + ExpressionAttributeNames: { '#s': 'status', '#ttl': 'ttl' }, ExpressionAttributeValues: { ':cancelled': { S: 'CANCELLED' }, ':expected': { S: 'PENDING_UPLOADS' }, ':now': { S: now }, ':err': { S: errorMessage }, ':sca': { S: `CANCELLED#${now}` }, + ':ttl': { N: String(Math.floor(Date.now() / 1000) + TASK_RETENTION_DAYS * 86400) }, }, })); } catch (err: any) { diff --git a/cdk/src/handlers/confirm-uploads.ts b/cdk/src/handlers/confirm-uploads.ts index 9b8163b7..983dc8d7 100644 --- a/cdk/src/handlers/confirm-uploads.ts +++ b/cdk/src/handlers/confirm-uploads.ts @@ -355,7 +355,7 @@ async function screenSingleAttachment( // Estimate token cost for images (using shared utility) let tokenEstimate: number | undefined; if (isImage) { - tokenEstimate = await estimateImageTokensFromBuffer(screenResult.content); + tokenEstimate = estimateImageTokensFromBuffer(screenResult.content, att.content_type); } return createAttachmentRecord({ @@ -545,11 +545,12 @@ async function failTaskOnScreening( await ddb.send(new UpdateCommand({ TableName: TABLE_NAME, Key: { task_id: taskId }, - UpdateExpression: 'SET #s = :failed, #sca = :status_created_at, error_message = :err, updated_at = :now', + UpdateExpression: 'SET #s = :failed, #sca = :status_created_at, error_message = :err, updated_at = :now, #ttl = :ttl', ConditionExpression: '#s = :pending_uploads', ExpressionAttributeNames: { '#s': 'status', '#sca': 'status_created_at', + '#ttl': 'ttl', }, ExpressionAttributeValues: { ':failed': TaskStatus.FAILED, @@ -557,6 +558,7 @@ async function failTaskOnScreening( ':status_created_at': `${TaskStatus.FAILED}#${now}`, ':err': `Attachment '${filename}' blocked: ${reason}`, ':now': now, + ':ttl': computeTtlEpoch(TASK_RETENTION_DAYS), }, })); } catch (err: any) { @@ -717,16 +719,28 @@ async function preCheckConcurrency(userId: string): Promise { TableName: CONCURRENCY_TABLE_NAME, Key: { user_id: userId }, })); - const activeCount = (result.Item?.active_count as number) ?? 0; + const activeCount = (result?.Item?.active_count as number) ?? 0; return activeCount < MAX_CONCURRENT; - } catch (err) { - // On error reading concurrency, allow the request to proceed — - // the atomic check in transitionToSubmitted is the authoritative gate. - logger.warn('Pre-check concurrency read failed — allowing request to proceed', { + } catch (err: any) { + // Only swallow DDB throttling errors — these are transient and the atomic + // check in transitionToSubmitted is the authoritative gate. + const throttleErrors = ['ProvisionedThroughputExceededException', 'RequestLimitExceeded', 'ThrottlingException']; + if (throttleErrors.includes(err?.name)) { + logger.warn('Pre-check concurrency throttled — allowing request to proceed', { + user_id: userId, + error: err.message, + }); + return true; + } + // Non-throttling errors (misconfigured table, IAM, network partition) + // should propagate — no point running expensive screening if infra is broken. + logger.error('Pre-check concurrency failed (non-throttling error)', { user_id: userId, error: err instanceof Error ? err.message : String(err), + error_name: err?.name, + metric_type: 'precheck_concurrency_failure', }); - return true; + throw err; } } diff --git a/cdk/src/handlers/shared/attachment-screening.ts b/cdk/src/handlers/shared/attachment-screening.ts index 611062c0..91aabdaf 100644 --- a/cdk/src/handlers/shared/attachment-screening.ts +++ b/cdk/src/handlers/shared/attachment-screening.ts @@ -20,7 +20,6 @@ import { createHash } from 'crypto'; import { ApplyGuardrailCommand, type BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; import { logger } from './logger'; -import { loadSharp } from './sharp-loader'; // --------------------------------------------------------------------------- // Constants @@ -63,10 +62,6 @@ export interface ScreenedAttachment { // Retry utility // --------------------------------------------------------------------------- -/** - * Retry with exponential backoff for transient Bedrock errors. - * Non-retryable errors (4xx except 429, validation errors) propagate immediately. - */ async function retryWithBackoff( fn: () => Promise, opts: { maxRetries: number; baseDelayMs: number; context: string }, @@ -116,11 +111,11 @@ function sleep(ms: number): Promise { /** * Screen an image attachment through the Bedrock Guardrail. * - * Flow: validate → convert GIF/WebP to PNG (Bedrock only accepts png|jpeg) → - * screen → strip EXIF / re-encode on pass. + * Only PNG and JPEG are accepted. Raw bytes are passed directly to Bedrock + * (no re-encoding or metadata stripping required). * - * @returns ScreenedAttachment with cleaned content (EXIF-stripped, re-encoded) and checksum. - * @throws Error on sharp failure or guardrail unavailability (fail-closed). + * @returns ScreenedAttachment with original content and checksum. + * @throws AttachmentScreeningError for unsupported formats or corrupt files. */ export async function screenImage( content: Buffer, @@ -128,43 +123,11 @@ export async function screenImage( filename: string, config: ScreeningConfig, ): Promise { + assertSupportedImageFormat(contentType, filename); assertImageUploadBytes(content, contentType, filename); - await assertImageDecodable(content, contentType, filename); - - // Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) - let screeningBuffer: Buffer; - let screeningFormat: 'png' | 'jpeg'; - - if (contentType === 'image/jpeg') { - screeningBuffer = content; - screeningFormat = 'jpeg'; - } else if (contentType === 'image/gif' || contentType === 'image/webp') { - // GIF/WebP → PNG. For animated GIFs, extract first frame only. - try { - screeningBuffer = await stripAndReencodeImage(content, 'image/png'); - } catch (convErr) { - throw new AttachmentScreeningError( - `Image "${filename}" could not be converted from ${contentType} for screening. ` + - 'The file may be corrupt. Please re-export or use a PNG/JPEG format.', - { cause: convErr }, - ); - } - screeningFormat = 'png'; + assertImageDimensionsWithinLimits(content, contentType, filename); - // Post-conversion size check: PNG expansion of compressed GIF/WebP can exceed limit. - if (screeningBuffer.length > MAX_ATTACHMENT_SIZE_BYTES) { - throw new AttachmentScreeningError( - `Image "${filename}" is ${contentType} and its PNG conversion for screening ` + - `exceeds the ${MAX_ATTACHMENT_SIZE_BYTES / (1024 * 1024)} MB limit ` + - `(${(screeningBuffer.length / (1024 * 1024)).toFixed(1)} MB after conversion). ` + - 'Please convert to JPEG or reduce image dimensions before uploading.', - ); - } - } else { - // PNG: use as-is - screeningBuffer = content; - screeningFormat = 'png'; - } + const screeningFormat: 'png' | 'jpeg' = contentType === 'image/jpeg' ? 'jpeg' : 'png'; // Screen through Bedrock Guardrail with retry const result = await retryWithBackoff( @@ -175,44 +138,27 @@ export async function screenImage( content: [{ image: { format: screeningFormat, - source: { bytes: screeningBuffer }, + source: { bytes: content }, }, }], })), { maxRetries: MAX_RETRIES, baseDelayMs: BASE_DELAY_MS, context: `image_screening:${filename}` }, ); + const checksum = computeSha256(content); + if (result.action === 'GUARDRAIL_INTERVENED') { const categories = extractBlockedCategories(result.assessments); return { - content: screeningBuffer, + content, contentType, - checksum: computeSha256(screeningBuffer), + checksum, screening: { status: 'blocked', categories }, }; } - // Screening passed — strip EXIF and re-encode in the declared format. - let cleanedContent: Buffer; - try { - cleanedContent = await stripAndReencodeImage(content, contentType); - } catch (sharpErr) { - logger.warn('Image sanitization failed (sharp)', { - filename, - content_type: contentType, - content_bytes: content.length, - error: sharpErr instanceof Error ? sharpErr.message : String(sharpErr), - }); - throw new AttachmentScreeningError( - `Image "${filename}" could not be processed for security sanitization. ` + - 'Please re-export the image in a standard format and try again.', - { cause: sharpErr }, - ); - } - - const checksum = computeSha256(cleanedContent); return { - content: cleanedContent, + content, contentType, checksum, screening: { status: 'passed' }, @@ -337,14 +283,23 @@ async function extractPdfText(content: Buffer, filename: string): Promise content.length) return undefined; + const height = content.readUInt16BE(offset + 5); + const width = content.readUInt16BE(offset + 7); + return { width, height }; + } + // Skip non-SOF markers + if (marker === 0xd9 || marker === 0xda) { + // End of image or start of scan — stop searching + return undefined; + } + if (offset + 3 >= content.length) return undefined; + const segmentLength = content.readUInt16BE(offset + 2); + offset += 2 + segmentLength; + } + return undefined; +} + +function assertImageDimensionsWithinLimits( content: Buffer, contentType: string, filename: string, -): Promise { - try { - const sharp = await loadSharp(); - const metadata = await sharp(content, SHARP_INPUT_OPTIONS).metadata(); - const width = metadata.width; - const height = metadata.height; - if (!width || !height) { - throw new AttachmentScreeningError( - `Image "${filename}" could not be decoded (missing dimensions). ` + - 'Please re-export as PNG or JPEG.', - ); - } - if (width > MAX_IMAGE_DIMENSION_PX || height > MAX_IMAGE_DIMENSION_PX) { +): void { + let dims: { width: number; height: number } | undefined; + + if (contentType === 'image/png') { + dims = readPngDimensions(content); + if (!dims) { throw new AttachmentScreeningError( - `Image "${filename}" is ${width}x${height}px; maximum allowed dimension is ` + - `${MAX_IMAGE_DIMENSION_PX}px. Please resize the image before uploading.`, + `Image "${filename}" upload is not a valid PNG file (missing IHDR). ` + + 'The upload may be incomplete or corrupted — please try submitting again.', ); } - } catch (err) { - if (err instanceof AttachmentScreeningError) { - throw err; + } else if (contentType === 'image/jpeg') { + dims = readJpegDimensions(content); + if (!dims) { + // Non-fatal: if we can't parse dimensions, let Bedrock handle rejection + return; } - const detail = err instanceof Error ? err.message : String(err); - logger.warn('Image decode failed (sharp)', { - filename, - content_type: contentType, - content_bytes: content.length, - error: detail, - }); - throw new AttachmentScreeningError( - `Image "${filename}" could not be decoded. The file may be corrupt or use an unsupported variant. ` + - 'Please re-export as PNG or JPEG.', - { cause: err }, - ); + } else { + return; } -} -/** - * Strip metadata (EXIF/ICC/XMP), apply orientation, and re-encode. - * Explicit output format avoids ambiguous `.toBuffer()` behaviour in Lambda/libvips. - */ -async function stripAndReencodeImage(content: Buffer, contentType: string): Promise { - const sharp = await loadSharp(); - const pipeline = sharp(content, SHARP_INPUT_OPTIONS).rotate(); - if (contentType === 'image/jpeg') { - return pipeline.jpeg({ mozjpeg: true, force: true }).toBuffer(); + if (dims.width > MAX_IMAGE_DIMENSION_PX || dims.height > MAX_IMAGE_DIMENSION_PX) { + throw new AttachmentScreeningError( + `Image "${filename}" is ${dims.width}x${dims.height}px; maximum allowed dimension is ` + + `${MAX_IMAGE_DIMENSION_PX}px. Please resize the image before uploading.`, + ); } - return pipeline.png({ compressionLevel: 9, force: true }).toBuffer(); } // --------------------------------------------------------------------------- diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index 893b54c2..0a27958d 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -435,7 +435,7 @@ export async function createTaskCore( // Estimate image token cost (best-effort, non-blocking) const tokenEstimate = inlineAtt.type === 'image' - ? await estimateImageTokensFromBuffer(screenResult.content) + ? estimateImageTokensFromBuffer(screenResult.content, inlineAtt.content_type) : undefined; attachmentRecords.push(createAttachmentRecord({ @@ -660,9 +660,21 @@ export async function createTaskCore( // 9. Return created task if (hasPresignedAttachments && s3Client && ATTACHMENTS_BUCKET) { // Generate presigned POST policies for presigned attachments - const uploadInstructions = await generateUploadInstructions( - taskRecord, validatedAttachments, context.userId, taskId, s3Client, - ); + let uploadInstructions: AttachmentUploadInstruction[]; + try { + uploadInstructions = await generateUploadInstructions( + taskRecord, validatedAttachments, context.userId, taskId, s3Client, + ); + } catch (presignErr) { + logger.error('Failed to generate presigned upload instructions — task orphaned in PENDING_UPLOADS', { + task_id: taskId, + error: presignErr instanceof Error ? presignErr.message : String(presignErr), + request_id: requestId, + metric_type: 'presigned_post_generation_failure', + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, + 'Failed to generate upload instructions. Please try again.', requestId); + } const taskExpiresAt = new Date(Date.now() + 30 * 60 * 1000).toISOString(); // 30 min auto-cancel window return successResponse(202, { ...toTaskDetail(taskRecord), diff --git a/cdk/src/handlers/shared/image-tokens.ts b/cdk/src/handlers/shared/image-tokens.ts index edd40748..44a81e7c 100644 --- a/cdk/src/handlers/shared/image-tokens.ts +++ b/cdk/src/handlers/shared/image-tokens.ts @@ -19,8 +19,7 @@ // Image token estimation matching Anthropic's documented resize rules. -import { logger } from './logger'; -import { loadSharp } from './sharp-loader'; +import { readPngDimensions, readJpegDimensions } from './attachment-screening'; const MAX_IMAGE_SIDE = 1568; const MAX_IMAGE_TOKENS = 1568; @@ -53,21 +52,17 @@ export function estimateImageTokens(width: number, height: number): number { } /** - * Estimate image tokens from a buffer by reading dimensions via sharp. + * Estimate image tokens from a buffer by reading dimensions from headers. + * Uses pure buffer parsing (PNG IHDR / JPEG SOF markers) — no native deps. * Returns undefined if dimensions cannot be determined (non-fatal). */ -export async function estimateImageTokensFromBuffer(content: Buffer): Promise { - try { - const sharp = await loadSharp(); - const metadata = await sharp(content).metadata(); - if (metadata.width && metadata.height) { - return estimateImageTokens(metadata.width, metadata.height); - } - } catch (err) { - logger.warn('Failed to estimate image tokens (non-fatal)', { - error: err instanceof Error ? err.message : String(err), - content_length: content.length, - }); +export function estimateImageTokensFromBuffer(content: Buffer, contentType?: string): number | undefined { + const dims = contentType === 'image/jpeg' + ? readJpegDimensions(content) + : readPngDimensions(content) ?? readJpegDimensions(content); + + if (dims) { + return estimateImageTokens(dims.width, dims.height); } return undefined; } diff --git a/cdk/src/handlers/shared/resolve-url-attachments.ts b/cdk/src/handlers/shared/resolve-url-attachments.ts index 70f99081..669b9531 100644 --- a/cdk/src/handlers/shared/resolve-url-attachments.ts +++ b/cdk/src/handlers/shared/resolve-url-attachments.ts @@ -34,6 +34,7 @@ */ import { promises as dns } from 'dns'; +import * as https from 'https'; import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { screenImage, screenTextFile, AttachmentScreeningError, type ScreeningConfig } from './attachment-screening'; import { AttachmentResolutionError } from './context-hydration'; @@ -69,6 +70,7 @@ const PRIVATE_IP_RANGES = [ { prefix: '100.64.', mask: null }, // CGN / Shared Address Space (RFC 6598) // IPv6 { prefix: '::1', mask: null }, + { prefix: '::', mask: (ip: string) => ip === '::' }, // Unspecified address (could route to localhost) { prefix: 'fc', mask: null }, { prefix: 'fd', mask: null }, { prefix: 'fe80:', mask: null }, @@ -98,7 +100,7 @@ export interface ResolveUrlAttachmentsOptions { * Check if an IP address belongs to a private/internal range. * Returns a reason string if private, undefined if public. */ -function isPrivateIp(ip: string): string | undefined { +export function isPrivateIp(ip: string): string | undefined { const normalized = ip.toLowerCase(); for (const range of PRIVATE_IP_RANGES) { @@ -173,6 +175,76 @@ function buildPinnedUrl(originalUrl: URL, resolvedIp: string): URL { return pinned; } +/** + * Perform an HTTPS request using Node.js native https module with proper + * TLS servername for DNS pinning. This is necessary because global fetch() + * uses the URL hostname for SNI — when connecting to a resolved IP, the + * certificate check fails unless servername is overridden. + * + * Returns a standard Response object for compatibility with the existing code. + */ +async function pinnedHttpsRequest( + pinnedUrl: URL, + originalHostname: string, + options: { headers: Record; signal: AbortSignal }, +): Promise { + return new Promise((resolve, reject) => { + const hostname = pinnedUrl.hostname.replace(/^\[|\]$/g, ''); // strip IPv6 brackets + const port = Number(pinnedUrl.port || '443'); + + const agent = new https.Agent({ + servername: originalHostname, + }); + + const req = https.request( + { + hostname, + port, + path: pinnedUrl.pathname + pinnedUrl.search, + method: 'GET', + headers: options.headers, + agent, + }, + (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + const body = Buffer.concat(chunks); + const responseHeaders = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value) responseHeaders.set(key, Array.isArray(value) ? value[0] : value); + } + resolve(new Response(body, { + status: res.statusCode ?? 500, + statusText: res.statusMessage, + headers: responseHeaders, + })); + agent.destroy(); + }); + res.on('error', (err) => { + agent.destroy(); + reject(err); + }); + }, + ); + + req.on('error', (err) => { + agent.destroy(); + reject(err); + }); + + if (options.signal) { + options.signal.addEventListener('abort', () => { + req.destroy(); + agent.destroy(); + reject(new DOMException('The operation was aborted.', 'AbortError')); + }); + } + + req.end(); + }); +} + /** * Fetch a URL with SSRF protections: DNS pinning, private IP rejection, * redirect validation, timeout, and size limit. @@ -210,6 +282,7 @@ async function ssrfSafeFetch( } let currentUrl = url; + let currentHostname = parsedUrl.hostname; let redirectCount = 0; let response: Response; @@ -219,12 +292,11 @@ async function ssrfSafeFetch( const timeout = setTimeout(() => controller.abort(), URL_FETCH_TIMEOUT_MS); try { - // Use pinnedUrl (resolved IP) for the actual connection, with Host header - // preserving the real hostname for TLS SNI and virtual-host routing. - response = await fetch(pinnedUrl.toString(), { + // Use pinnedHttpsRequest with proper TLS servername for DNS pinning. + // Connects to the resolved IP while using the original hostname for SNI. + response = await pinnedHttpsRequest(pinnedUrl, currentHostname, { headers, signal: controller.signal, - redirect: 'manual', // Handle redirects manually for IP re-validation }); } catch (err: any) { if (err.name === 'AbortError') { @@ -265,6 +337,7 @@ async function ssrfSafeFetch( } const redirectIp = await resolveAndValidate(redirectUrl.hostname); currentUrl = redirectUrl.toString(); + currentHostname = redirectUrl.hostname; // Pin the redirect target to its resolved IP pinnedUrl = buildPinnedUrl(redirectUrl, redirectIp); @@ -427,12 +500,27 @@ export async function resolveUrlAttachments( // Upload screened content to S3 const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${userId}/${taskId}/${att.attachment_id}/${att.filename}`; - const putResult = await options.s3Client.send(new PutObjectCommand({ - Bucket: options.bucketName, - Key: s3Key, - Body: screenResult.content, - ContentType: resolvedContentType, - })); + let putResult; + try { + putResult = await options.s3Client.send(new PutObjectCommand({ + Bucket: options.bucketName, + Key: s3Key, + Body: screenResult.content, + ContentType: resolvedContentType, + })); + } catch (s3Err) { + logger.error('S3 upload failed for URL attachment', { + attachment_id: att.attachment_id, + filename: att.filename, + s3_key: s3Key, + error: s3Err instanceof Error ? s3Err.message : String(s3Err), + metric_type: 'url_attachment_upload_failure', + }); + throw new AttachmentResolutionError( + `URL attachment '${att.filename}' could not be stored. Please try again later.`, + { cause: s3Err }, + ); + } // Use checksum from screening (already computed over the cleaned content) const checksum = screenResult.checksum; @@ -440,7 +528,7 @@ export async function resolveUrlAttachments( // Estimate token cost for images let tokenEstimate: number | undefined; if (isImage) { - tokenEstimate = await estimateImageTokensFromBuffer(screenResult.content); + tokenEstimate = estimateImageTokensFromBuffer(screenResult.content, resolvedContentType); } resolved.set(att.attachment_id, createAttachmentRecord({ diff --git a/cdk/src/handlers/shared/sharp-loader.ts b/cdk/src/handlers/shared/sharp-loader.ts deleted file mode 100644 index f37408c7..00000000 --- a/cdk/src/handlers/shared/sharp-loader.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * MIT No Attribution - * - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -import type sharp from 'sharp'; - -type SharpModule = typeof sharp; - -let sharpPromise: Promise | undefined; - -/** Lazy-load sharp so Lambdas without attachments avoid native module init at cold start. */ -export function loadSharp(): Promise { - if (!sharpPromise) { - sharpPromise = import('sharp').then(mod => mod.default); - } - return sharpPromise; -} diff --git a/cdk/src/handlers/shared/validation.ts b/cdk/src/handlers/shared/validation.ts index 56dad904..53e6320c 100644 --- a/cdk/src/handlers/shared/validation.ts +++ b/cdk/src/handlers/shared/validation.ts @@ -269,12 +269,10 @@ type _AssertAttachmentExhaustive = Exclude(ATTACHMENT_TYPE_LIST); -/** Allowed image MIME types (Bedrock vision-supported formats). */ +/** Allowed image MIME types (PNG and JPEG only — passed directly to Bedrock). */ const ALLOWED_IMAGE_MIME_TYPES = new Set([ 'image/png', 'image/jpeg', - 'image/gif', - 'image/webp', ]); /** Allowed file MIME types. */ @@ -294,25 +292,14 @@ const ALLOWED_FILE_MIME_TYPES = new Set([ const MAGIC_BYTES: ReadonlyArray<{ readonly mime: string; readonly bytes: readonly number[]; readonly offset?: number }> = [ { mime: 'image/png', bytes: [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A] }, { mime: 'image/jpeg', bytes: [0xFF, 0xD8, 0xFF] }, - { mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] }, // GIF8 (covers GIF87a and GIF89a) { mime: 'application/pdf', bytes: [0x25, 0x50, 0x44, 0x46, 0x2D] }, // %PDF- ]; -// RIFF....WEBP requires checking bytes 0-3 (RIFF) and 8-11 (WEBP) -function isWebP(data: Buffer): boolean { - if (data.length < 12) return false; - return data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46 - && data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50; -} - /** * Validate content against declared MIME type using magic bytes. * For text types, checks for valid UTF-8 and no null bytes. */ export function validateMagicBytes(data: Buffer, contentType: string): boolean { - // WebP has a split signature - if (contentType === 'image/webp') return isWebP(data); - // Check against known binary signatures const sig = MAGIC_BYTES.find(s => s.mime === contentType); if (sig) { @@ -338,7 +325,6 @@ export function validateMagicBytes(data: Buffer, contentType: string): boolean { * Detect MIME type from magic bytes (for inline attachments without content_type). */ export function detectMimeTypeFromMagicBytes(data: Buffer): string | null { - if (isWebP(data)) return 'image/webp'; for (const sig of MAGIC_BYTES) { if (data.length >= sig.bytes.length) { const offset = sig.offset ?? 0; @@ -401,8 +387,6 @@ function generateFilename(type: string, contentType: string, index: number): str const MIME_TO_EXTENSION: Record = { 'image/png': 'png', 'image/jpeg': 'jpg', - 'image/gif': 'gif', - 'image/webp': 'webp', 'text/plain': 'txt', 'text/csv': 'csv', 'text/markdown': 'md', @@ -498,8 +482,12 @@ export function validateAttachments( return { valid: false, error: `attachments[${i}]: detected content_type '${detected}' not allowed for type '${attType}'` }; } resolvedContentType = detected; + } else if (attType === 'url') { + // URL attachments: content_type is determined at fetch time during hydration. + // Use a placeholder — resolve-url-attachments.ts validates after download. + resolvedContentType = 'application/octet-stream'; } else { - return { valid: false, error: `attachments[${i}]: content_type is required for presigned uploads and URL attachments` }; + return { valid: false, error: `attachments[${i}]: content_type is required for presigned uploads` }; } // Magic bytes check against declared content_type (for inline data with declared type) diff --git a/cdk/test/handlers/cleanup-pending-uploads.test.ts b/cdk/test/handlers/cleanup-pending-uploads.test.ts new file mode 100644 index 00000000..815a7763 --- /dev/null +++ b/cdk/test/handlers/cleanup-pending-uploads.test.ts @@ -0,0 +1,198 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// Mock DynamoDB and S3 clients +const mockDdbSend = jest.fn(); +const mockS3Send = jest.fn(); + +jest.mock('@aws-sdk/client-dynamodb', () => ({ + DynamoDBClient: jest.fn(() => ({ send: mockDdbSend })), + QueryCommand: jest.fn((input: any) => ({ input, _type: 'Query' })), + UpdateItemCommand: jest.fn((input: any) => ({ input, _type: 'UpdateItem' })), + PutItemCommand: jest.fn((input: any) => ({ input, _type: 'PutItem' })), +})); + +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(() => ({ send: mockS3Send })), + ListObjectsV2Command: jest.fn((input: any) => ({ input, _type: 'ListObjectsV2' })), + DeleteObjectsCommand: jest.fn((input: any) => ({ input, _type: 'DeleteObjects' })), +})); + +// Set env vars before import +process.env.TASK_TABLE_NAME = 'TaskTable'; +process.env.TASK_EVENTS_TABLE_NAME = 'EventsTable'; +process.env.ATTACHMENTS_BUCKET_NAME = 'test-attachments-bucket'; +process.env.PENDING_UPLOAD_TIMEOUT_SECONDS = '1800'; +process.env.TASK_RETENTION_DAYS = '90'; + +import { handler } from '../../src/handlers/cleanup-pending-uploads'; + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe('cleanup-pending-uploads handler', () => { + test('does nothing when no expired tasks found', async () => { + mockDdbSend.mockResolvedValueOnce({ Items: [] }); + + await handler(); + + expect(mockDdbSend).toHaveBeenCalledTimes(1); + expect(mockS3Send).not.toHaveBeenCalled(); + }); + + test('cancels expired task with conditional write and sets TTL', async () => { + const thirtyFiveMinAgo = new Date(Date.now() - 35 * 60 * 1000).toISOString(); + + // findExpiredPendingUploads returns one task + mockDdbSend.mockResolvedValueOnce({ + Items: [{ + task_id: { S: 'TASK001' }, + user_id: { S: 'user-123' }, + created_at: { S: thirtyFiveMinAgo }, + }], + }); + + // cancelExpiredTask — DDB update succeeds + mockDdbSend.mockResolvedValueOnce({}); + // write event (best-effort) + mockDdbSend.mockResolvedValueOnce({}); + + // cleanupTaskAttachments — S3 list returns no objects + mockS3Send.mockResolvedValueOnce({ Contents: [] }); + + await handler(); + + // Verify the UpdateItem includes TTL + const updateCall = mockDdbSend.mock.calls[1][0]; + expect(updateCall.input.UpdateExpression).toContain('#ttl = :ttl'); + expect(updateCall.input.ExpressionAttributeValues[':ttl']).toBeDefined(); + expect(updateCall.input.ExpressionAttributeValues[':ttl'].N).toBeDefined(); + // TTL should be ~90 days from now + const ttlValue = Number(updateCall.input.ExpressionAttributeValues[':ttl'].N); + const expectedMin = Math.floor(Date.now() / 1000) + 89 * 86400; + const expectedMax = Math.floor(Date.now() / 1000) + 91 * 86400; + expect(ttlValue).toBeGreaterThan(expectedMin); + expect(ttlValue).toBeLessThan(expectedMax); + }); + + test('handles race condition with confirm-uploads (ConditionalCheckFailedException)', async () => { + const thirtyFiveMinAgo = new Date(Date.now() - 35 * 60 * 1000).toISOString(); + + mockDdbSend.mockResolvedValueOnce({ + Items: [{ + task_id: { S: 'TASK001' }, + user_id: { S: 'user-123' }, + created_at: { S: thirtyFiveMinAgo }, + }], + }); + + // Transition fails — confirm-uploads won the race + const condErr = new Error('The conditional request failed'); + condErr.name = 'ConditionalCheckFailedException'; + mockDdbSend.mockRejectedValueOnce(condErr); + + await handler(); + + // S3 cleanup should NOT be called when race was lost + expect(mockS3Send).not.toHaveBeenCalled(); + // No event should be written + expect(mockDdbSend).toHaveBeenCalledTimes(2); // query + failed update + }); + + test('cleans up S3 objects after successful cancellation', async () => { + const thirtyFiveMinAgo = new Date(Date.now() - 35 * 60 * 1000).toISOString(); + + mockDdbSend.mockResolvedValueOnce({ + Items: [{ + task_id: { S: 'TASK001' }, + user_id: { S: 'user-123' }, + created_at: { S: thirtyFiveMinAgo }, + }], + }); + + // cancelExpiredTask succeeds + mockDdbSend.mockResolvedValueOnce({}); + // write event + mockDdbSend.mockResolvedValueOnce({}); + + // S3 list returns objects + mockS3Send.mockResolvedValueOnce({ + Contents: [ + { Key: 'attachments/user-123/TASK001/ATT001/image.png' }, + { Key: 'attachments/user-123/TASK001/ATT002/doc.pdf' }, + ], + }); + // S3 delete succeeds + mockS3Send.mockResolvedValueOnce({ Deleted: [{}, {}] }); + + await handler(); + + // Verify S3 delete was called with the right keys + const deleteCall = mockS3Send.mock.calls[1][0]; + expect(deleteCall.input.Bucket).toBe('test-attachments-bucket'); + expect(deleteCall.input.Delete.Objects).toEqual([ + { Key: 'attachments/user-123/TASK001/ATT001/image.png' }, + { Key: 'attachments/user-123/TASK001/ATT002/doc.pdf' }, + ]); + }); + + test('throws when ALL tasks error (triggers CloudWatch alarm)', async () => { + const thirtyFiveMinAgo = new Date(Date.now() - 35 * 60 * 1000).toISOString(); + + mockDdbSend.mockResolvedValueOnce({ + Items: [{ + task_id: { S: 'TASK001' }, + user_id: { S: 'user-123' }, + created_at: { S: thirtyFiveMinAgo }, + }], + }); + + // Non-conditional DDB error (infra failure) + const infraErr = new Error('Service unavailable'); + infraErr.name = 'InternalServerError'; + mockDdbSend.mockRejectedValueOnce(infraErr); + + await expect(handler()).rejects.toThrow('All 1 expired PENDING_UPLOADS task(s) failed to process'); + }); + + test('does not throw on partial success (some cancelled, some errored)', async () => { + const thirtyFiveMinAgo = new Date(Date.now() - 35 * 60 * 1000).toISOString(); + + mockDdbSend.mockResolvedValueOnce({ + Items: [ + { task_id: { S: 'TASK001' }, user_id: { S: 'user-123' }, created_at: { S: thirtyFiveMinAgo } }, + { task_id: { S: 'TASK002' }, user_id: { S: 'user-456' }, created_at: { S: thirtyFiveMinAgo } }, + ], + }); + + // First task: cancel succeeds + mockDdbSend.mockResolvedValueOnce({}); + mockDdbSend.mockResolvedValueOnce({}); // event + mockS3Send.mockResolvedValueOnce({ Contents: [] }); // no S3 objects + + // Second task: cancel fails with infra error + const infraErr = new Error('Timeout'); + infraErr.name = 'RequestTimeout'; + mockDdbSend.mockRejectedValueOnce(infraErr); + + // Should NOT throw — partial success is acceptable + await expect(handler()).resolves.toBeUndefined(); + }); +}); diff --git a/cdk/test/handlers/shared/attachment-screening.test.ts b/cdk/test/handlers/shared/attachment-screening.test.ts index 37b3a03b..fb9f310a 100644 --- a/cdk/test/handlers/shared/attachment-screening.test.ts +++ b/cdk/test/handlers/shared/attachment-screening.test.ts @@ -23,6 +23,8 @@ import type { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; import { assertImageUploadBytes, AttachmentScreeningError, + readJpegDimensions, + readPngDimensions, screenImage, } from '../../../src/handlers/shared/attachment-screening'; @@ -46,11 +48,111 @@ function mockBedrockPass(): BedrockRuntimeClient { } as unknown as BedrockRuntimeClient; } +function mockBedrockBlock(): BedrockRuntimeClient { + return { + send: jest.fn().mockResolvedValue({ + action: 'GUARDRAIL_INTERVENED', + outputs: [], + assessments: [{ contentPolicy: { filters: [{ type: 'SEXUAL' }] } }], + }), + } as unknown as BedrockRuntimeClient; +} + +// Minimal valid PNG: 1x1 pixel with valid IHDR (CRC is not checked by our parser) +function minimalPng(): Buffer { + // PNG signature + IHDR chunk (length=13, "IHDR", width=1, height=1, bit_depth=8, color_type=2, zeros for rest + dummy CRC) + return Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, // PNG signature + 0x00, 0x00, 0x00, 0x0d, // IHDR length = 13 + 0x49, 0x48, 0x44, 0x52, // "IHDR" + 0x00, 0x00, 0x00, 0x01, // width = 1 + 0x00, 0x00, 0x00, 0x01, // height = 1 + 0x08, 0x02, 0x00, 0x00, 0x00, // bit_depth=8, color_type=2(RGB), compression, filter, interlace + 0x00, 0x00, 0x00, 0x00, // CRC (dummy — not validated by dimension parser) + ]); +} + +// Minimal valid JPEG with SOF0 (dimensions 100x50) +function minimalJpeg(width = 100, height = 50): Buffer { + // SOI + APP0 marker (minimal) + SOF0 with dimensions + EOI + const soi = Buffer.from([0xff, 0xd8]); + // SOF0 marker + const sof0 = Buffer.alloc(11); + sof0[0] = 0xff; + sof0[1] = 0xc0; + sof0.writeUInt16BE(8, 2); // segment length + sof0[4] = 8; // precision + sof0.writeUInt16BE(height, 5); + sof0.writeUInt16BE(width, 7); + sof0[9] = 1; // num components + const eoi = Buffer.from([0xff, 0xd9]); + return Buffer.concat([soi, sof0, eoi]); +} + describe('assertImageUploadBytes', () => { test('rejects non-PNG bytes for image/png', () => { expect(() => assertImageUploadBytes(Buffer.from('not a png'), 'image/png', 'x.png')) .toThrow(AttachmentScreeningError); }); + + test('rejects non-JPEG bytes for image/jpeg', () => { + expect(() => assertImageUploadBytes(Buffer.from('not a jpeg'), 'image/jpeg', 'x.jpg')) + .toThrow(AttachmentScreeningError); + }); + + test('rejects empty buffer', () => { + expect(() => assertImageUploadBytes(Buffer.alloc(0), 'image/png', 'empty.png')) + .toThrow(AttachmentScreeningError); + }); + + test('accepts valid PNG signature', () => { + const png = minimalPng(); + expect(() => assertImageUploadBytes(png, 'image/png', 'valid.png')).not.toThrow(); + }); + + test('accepts valid JPEG signature', () => { + const jpeg = minimalJpeg(); + expect(() => assertImageUploadBytes(jpeg, 'image/jpeg', 'valid.jpg')).not.toThrow(); + }); +}); + +describe('readPngDimensions', () => { + test('reads IHDR from architecture diagram fixture', () => { + if (!fs.existsSync(ARCHITECTURE_PNG)) { + return; + } + const content = fs.readFileSync(ARCHITECTURE_PNG); + expect(readPngDimensions(content)).toEqual({ width: 3454, height: 1442 }); + }); + + test('reads dimensions from minimal PNG', () => { + const png = minimalPng(); + expect(readPngDimensions(png)).toEqual({ width: 1, height: 1 }); + }); + + test('returns undefined when IHDR chunk is missing', () => { + const sig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + expect(readPngDimensions(Buffer.concat([sig, Buffer.from('FAKE')]))).toBeUndefined(); + }); + + test('returns undefined for too-short buffer', () => { + expect(readPngDimensions(Buffer.alloc(10))).toBeUndefined(); + }); +}); + +describe('readJpegDimensions', () => { + test('reads dimensions from SOF0 marker', () => { + const jpeg = minimalJpeg(800, 600); + expect(readJpegDimensions(jpeg)).toEqual({ width: 800, height: 600 }); + }); + + test('returns undefined for non-JPEG data', () => { + expect(readJpegDimensions(Buffer.from('not jpeg'))).toBeUndefined(); + }); + + test('returns undefined for too-short buffer', () => { + expect(readJpegDimensions(Buffer.from([0xff, 0xd8]))).toBeUndefined(); + }); }); describe('screenImage', () => { @@ -60,39 +162,61 @@ describe('screenImage', () => { guardrailVersion: '1', }; - test('sanitizes a large real-world PNG (architecture diagram fixture)', async () => { + test('passes raw PNG to Bedrock and returns original content', async () => { if (!fs.existsSync(ARCHITECTURE_PNG)) { return; } const content = fs.readFileSync(ARCHITECTURE_PNG); - const result = await screenImage( - content, - 'image/png', - 'autonomous-engine-architecture.png', - config, - ); + const result = await screenImage(content, 'image/png', 'test.png', config); expect(result.contentType).toBe('image/png'); - expect(result.content.length).toBeGreaterThan(0); - expect(result.content.length).toBeLessThanOrEqual(content.length); + // No re-encoding — content is passed through as-is + expect(result.content).toBe(content); expect(result.checksum).toMatch(/^[0-9a-f]{64}$/); - expect(result.content).not.toEqual(content); + expect(result.screening.status).toBe('passed'); + }); + + test('passes raw JPEG to Bedrock', async () => { + const jpeg = minimalJpeg(); + const result = await screenImage(jpeg, 'image/jpeg', 'test.jpg', config); + + expect(result.contentType).toBe('image/jpeg'); + expect(result.content).toBe(jpeg); + expect(result.screening.status).toBe('passed'); + }); + + test('rejects GIF format', async () => { + const gif = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]); // GIF89a + await expect( + screenImage(gif, 'image/gif', 'anim.gif', config), + ).rejects.toThrow(AttachmentScreeningError); + }); + + test('rejects WebP format', async () => { + const webp = Buffer.alloc(12); + webp.write('RIFF', 0); + webp.write('WEBP', 8); + await expect( + screenImage(webp, 'image/webp', 'photo.webp', config), + ).rejects.toThrow(AttachmentScreeningError); }); - test('rejects oversized dimensions before guardrail', async () => { + test('rejects oversized PNG dimensions before guardrail call', async () => { + // Build a PNG with IHDR declaring 9000x100 dimensions + const sig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const ihdrLen = Buffer.alloc(4); + ihdrLen.writeUInt32BE(13, 0); + const ihdrType = Buffer.from('IHDR'); + const ihdrData = Buffer.alloc(13); + ihdrData.writeUInt32BE(9000, 0); // width > 8000 + ihdrData.writeUInt32BE(100, 4); + ihdrData[8] = 8; + ihdrData[9] = 2; + const crcBuf = Buffer.alloc(4); + const oversized = Buffer.concat([sig, ihdrLen, ihdrType, ihdrData, crcBuf]); + const send = jest.fn(); const client = { send } as unknown as BedrockRuntimeClient; - const sharp = await import('sharp'); - const oversized = await sharp.default({ - create: { - width: 8001, - height: 10, - channels: 3, - background: { r: 0, g: 0, b: 0 }, - }, - }) - .png() - .toBuffer(); await expect( screenImage(oversized, 'image/png', 'huge.png', { @@ -102,6 +226,22 @@ describe('screenImage', () => { }), ).rejects.toThrow(AttachmentScreeningError); + // Bedrock should never be called for oversized images expect(send).not.toHaveBeenCalled(); }); + + test('returns blocked status when guardrail intervenes', async () => { + const png = minimalPng(); + const blockConfig = { + bedrockClient: mockBedrockBlock(), + guardrailId: 'test-guardrail', + guardrailVersion: '1', + }; + + const result = await screenImage(png, 'image/png', 'blocked.png', blockConfig); + expect(result.screening.status).toBe('blocked'); + if (result.screening.status === 'blocked') { + expect(result.screening.categories).toContain('SEXUAL'); + } + }); }); diff --git a/cdk/test/handlers/shared/resolve-url-attachments.test.ts b/cdk/test/handlers/shared/resolve-url-attachments.test.ts new file mode 100644 index 00000000..8990c51c --- /dev/null +++ b/cdk/test/handlers/shared/resolve-url-attachments.test.ts @@ -0,0 +1,114 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { isPrivateIp } from '../../../src/handlers/shared/resolve-url-attachments'; + +describe('isPrivateIp', () => { + describe('IPv4 private ranges', () => { + test('blocks 10.x.x.x (RFC 1918 Class A)', () => { + expect(isPrivateIp('10.0.0.1')).toBeDefined(); + expect(isPrivateIp('10.255.255.255')).toBeDefined(); + }); + + test('blocks 172.16-31.x.x (RFC 1918 Class B)', () => { + expect(isPrivateIp('172.16.0.1')).toBeDefined(); + expect(isPrivateIp('172.31.255.255')).toBeDefined(); + }); + + test('allows 172.15.x.x and 172.32.x.x (outside RFC 1918)', () => { + expect(isPrivateIp('172.15.0.1')).toBeUndefined(); + expect(isPrivateIp('172.32.0.1')).toBeUndefined(); + }); + + test('blocks 192.168.x.x (RFC 1918 Class C)', () => { + expect(isPrivateIp('192.168.0.1')).toBeDefined(); + expect(isPrivateIp('192.168.255.255')).toBeDefined(); + }); + + test('blocks 169.254.x.x (link-local)', () => { + expect(isPrivateIp('169.254.169.254')).toBeDefined(); // AWS metadata + expect(isPrivateIp('169.254.0.1')).toBeDefined(); + }); + + test('blocks 127.x.x.x (loopback)', () => { + expect(isPrivateIp('127.0.0.1')).toBeDefined(); + expect(isPrivateIp('127.255.255.255')).toBeDefined(); + }); + + test('blocks 0.x.x.x (current network)', () => { + expect(isPrivateIp('0.0.0.0')).toBeDefined(); + expect(isPrivateIp('0.1.2.3')).toBeDefined(); + }); + + test('blocks 100.64.x.x (CGN / RFC 6598)', () => { + expect(isPrivateIp('100.64.0.1')).toBeDefined(); + expect(isPrivateIp('100.64.255.255')).toBeDefined(); + }); + + test('allows public IPv4 addresses', () => { + expect(isPrivateIp('8.8.8.8')).toBeUndefined(); + expect(isPrivateIp('1.1.1.1')).toBeUndefined(); + expect(isPrivateIp('203.0.113.1')).toBeUndefined(); + expect(isPrivateIp('100.63.255.255')).toBeUndefined(); // Just below CGN + }); + }); + + describe('IPv6 private ranges', () => { + test('blocks ::1 (loopback)', () => { + expect(isPrivateIp('::1')).toBeDefined(); + }); + + test('blocks :: (unspecified address)', () => { + expect(isPrivateIp('::')).toBeDefined(); + }); + + test('blocks fc/fd prefixes (ULA)', () => { + expect(isPrivateIp('fc00::1')).toBeDefined(); + expect(isPrivateIp('fd12:3456:789a::1')).toBeDefined(); + }); + + test('blocks fe80: (link-local)', () => { + expect(isPrivateIp('fe80::1')).toBeDefined(); + expect(isPrivateIp('fe80::abcd:ef01')).toBeDefined(); + }); + + test('blocks IPv4-mapped IPv6 (::ffff:x.x.x.x)', () => { + expect(isPrivateIp('::ffff:169.254.169.254')).toBeDefined(); + expect(isPrivateIp('::ffff:10.0.0.1')).toBeDefined(); + expect(isPrivateIp('::ffff:127.0.0.1')).toBeDefined(); + }); + + test('blocks expanded IPv4-mapped IPv6 (0:0:0:0:0:ffff:x)', () => { + expect(isPrivateIp('0:0:0:0:0:ffff:169.254.169.254')).toBeDefined(); + }); + + test('allows public IPv6 addresses', () => { + expect(isPrivateIp('2001:4860:4860::8888')).toBeUndefined(); // Google DNS + expect(isPrivateIp('2606:4700:4700::1111')).toBeUndefined(); // Cloudflare DNS + }); + }); + + describe('case insensitivity', () => { + test('handles uppercase IPv6', () => { + expect(isPrivateIp('FC00::1')).toBeDefined(); + expect(isPrivateIp('FE80::1')).toBeDefined(); + expect(isPrivateIp('::FFFF:10.0.0.1')).toBeDefined(); + }); + }); +}); diff --git a/cdk/test/handlers/shared/validation.test.ts b/cdk/test/handlers/shared/validation.test.ts index 83e93641..13475e6a 100644 --- a/cdk/test/handlers/shared/validation.test.ts +++ b/cdk/test/handlers/shared/validation.test.ts @@ -638,8 +638,11 @@ describe('isAllowedMimeType', () => { test('allows valid image types', () => { expect(isAllowedMimeType('image/png', 'image')).toBe(true); expect(isAllowedMimeType('image/jpeg', 'image')).toBe(true); - expect(isAllowedMimeType('image/gif', 'image')).toBe(true); - expect(isAllowedMimeType('image/webp', 'image')).toBe(true); + }); + + test('rejects GIF and WebP image types', () => { + expect(isAllowedMimeType('image/gif', 'image')).toBe(false); + expect(isAllowedMimeType('image/webp', 'image')).toBe(false); }); test('allows valid file types', () => { diff --git a/cli/src/commands/submit.ts b/cli/src/commands/submit.ts index fd2bbf36..00d3ffa5 100644 --- a/cli/src/commands/submit.ts +++ b/cli/src/commands/submit.ts @@ -313,29 +313,51 @@ function resolveAttachmentArg(arg: string): Attachment { * Policy fields from the API must precede the file; use FormData so Node sets * the boundary and Content-Length correctly for multi-megabyte payloads. */ +/** Upload timeout: 2 minutes for large files. */ +const UPLOAD_TIMEOUT_MS = 120_000; + async function uploadViaPresignedPost( filePath: string, instruction: AttachmentUploadInstruction, ): Promise { const fileData = fs.readFileSync(filePath); - const ext = path.extname(filePath).toLowerCase(); - const contentType = MIME_BY_EXT[ext] ?? 'application/octet-stream'; const form = new FormData(); for (const [key, value] of Object.entries(instruction.upload_fields)) { form.append(key, value); } - // File must be last. Object Content-Type comes from the policy field above, - // not the multipart part headers (per S3 POST Object). - form.append('file', new Blob([fileData], { type: contentType }), path.basename(filePath)); + // File must be last. S3 POST Object uses Content-Type from the policy field, + // not the multipart part — do not set a part Content-Type (breaks some clients). + form.append('file', new Blob([fileData]), path.basename(filePath)); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), UPLOAD_TIMEOUT_MS); - const res = await fetch(instruction.upload_url, { - method: 'POST', - body: form, - }); + let res: Response; + try { + res = await fetch(instruction.upload_url, { + method: 'POST', + body: form, + signal: controller.signal, + }); + } catch (err: any) { + if (err.name === 'AbortError') { + throw new CliError( + `Upload timed out for ${instruction.filename} after ${UPLOAD_TIMEOUT_MS / 1000}s. ` + + 'Check your network connection and try again.', + ); + } + throw new CliError( + `Upload failed for ${instruction.filename}: ${err.message ?? String(err)}`, + ); + } finally { + clearTimeout(timeout); + } - if (!res.ok) { - const text = await res.text().catch(() => ''); + // S3 POST Object returns 204 on success. Some error conditions return 200 + // with an XML error body (e.g., KMS failures). Check for XML error pattern. + const text = await res.text().catch(() => ''); + if (!res.ok || (res.status === 200 && text.includes(''))) { throw new CliError( `Presigned upload failed for ${instruction.filename}: HTTP ${res.status}${text ? ` — ${text.slice(0, 200)}` : ''}`, ); diff --git a/docs/design/API_CONTRACT.md b/docs/design/API_CONTRACT.md index 219de6a1..29ec2efd 100644 --- a/docs/design/API_CONTRACT.md +++ b/docs/design/API_CONTRACT.md @@ -102,7 +102,7 @@ Creates a new task. The orchestrator runs admission control, context hydration, | Category | MIME types | Extensions | |---|---|---| -| Images | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | `.png`, `.jpg`, `.gif`, `.webp` | +| Images | `image/png`, `image/jpeg` | `.png`, `.jpg` | | Text files | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | **Attachment limits:** diff --git a/docs/design/ATTACHMENTS.md b/docs/design/ATTACHMENTS.md index ca76ee87..a2f9d924 100644 --- a/docs/design/ATTACHMENTS.md +++ b/docs/design/ATTACHMENTS.md @@ -141,7 +141,7 @@ function createAttachmentRecord(params: CreateAttachmentRecordParams): Attachmen // - checksum_sha256 required when screening.status === 'passed' // - size_bytes required when screening.status === 'passed' // - categories non-empty when screening.status === 'blocked' - // Note: token_estimate is best-effort for images (sharp may fail on unusual formats), + // Note: token_estimate is best-effort for images (dimension parsing may fail on unusual formats), // so it is NOT enforced here. The budget check uses a conservative fallback when absent. if (params.screening.status === 'passed') { if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { @@ -194,7 +194,7 @@ The `` segment ensures uniqueness even if multiple attachments sh | Max total inline data per request | 3 MB decoded | Hard cap on total base64-decoded bytes in a single request. Even with base64 overhead (~4 MB encoded) plus JSON fields, this stays under the 6 MB Lambda payload limit. | | Max total size per task | 50 MB | Prevents abuse; bounds total screening and transfer time | | Max task_description length | 10,000 chars | Increased from 2,000. **This is a standalone API change that affects all tasks** (not just attachment tasks). Rationale: (a) attachments need rich explanatory context ("implement this design per the attached mockup, paying attention to the header layout"), (b) multiple users have reported the 2K limit as a friction point for complex task descriptions even without attachments, (c) the guardrail screening cost increase is minimal (text screening is cheap), (d) DynamoDB item size impact is negligible (~8 KB vs ~2 KB for the description field). **Requires updating [API_CONTRACT.md](./API_CONTRACT.md) line 82 in tandem.** | -| Allowed image MIME types | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | Bedrock vision-supported formats | +| Allowed image MIME types | `image/png`, `image/jpeg` | Bedrock-supported formats; GIF/WebP removed to eliminate native image processing dependency | | Allowed file MIME types | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | Useful for code/data context; no executables | | Max URL fetch size | 10 MB | Same per-attachment limit for fetched content | | URL fetch timeout | 10 seconds | Prevent SSRF-style long-poll attacks | @@ -244,7 +244,6 @@ sequenceDiagram H-->>C: 400 ATTACHMENT_BLOCKED { attachment_id, reason } end end - H->>H: Strip EXIF from images, re-upload cleaned version H->>DB: Update TaskRecord (status: SUBMITTED, screening: passed) with condition: status = PENDING_UPLOADS H->>H: Async-invoke Orchestrator H-->>C: 200 OK { task_id, status: SUBMITTED } @@ -328,8 +327,7 @@ sequenceDiagram H->>H: Cleanup already-uploaded S3 objects H-->>C: 400 ATTACHMENT_BLOCKED end - H->>H: Strip EXIF (images) / sanitize; on sharp failure → ATTACHMENT_INVALID_CONTENT - H->>S3: PutObject (cleaned content) + H->>S3: PutObject (screened content) H->>H: Build AttachmentRecord end H->>DB: Write TaskRecord (status: SUBMITTED, with attachment metadata) @@ -364,7 +362,6 @@ sequenceDiagram O->>SCR: Screen fetched content (with retry) SCR-->>O: Pass / Blocked alt Screening passed - O->>O: Strip EXIF (images) / sanitize; on sharp failure → fail task O->>S3: PutObject O->>O: Update AttachmentRecord else Screening blocked or fetch failed @@ -420,76 +417,27 @@ This prevents a single transient Bedrock hiccup from failing an entire task afte ```mermaid flowchart TD - I[Image attachment] --> MB[Magic bytes: verify image signature] + I[Image attachment] --> FMT{PNG or JPEG?} + FMT -->|No: GIF/WebP| RJ[REJECTED: unsupported format] + FMT -->|Yes| MB[Magic bytes: verify image signature] MB -->|Invalid| R[REJECTED: not a valid image] - MB -->|Valid| V[Validate: decodable, dimensions ≤ 8192x8192] - V --> G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] + MB -->|Valid| D[Dimension check: parse PNG IHDR / JPEG SOF] + D -->|> 8000px| OV[REJECTED: oversized] + D -->|OK| G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] G -->|INTERVENED| B[BLOCKED: content policy violation] - G -->|NONE| M[EXIF strip + re-encode via sharp] - M -->|sharp error| SE[REJECTED: ATTACHMENT_INVALID_CONTENT] - M -->|Success| P[PASSED] + G -->|NONE| P[PASSED: original bytes stored as-is] G -->|Error after retries| F[FAIL CLOSED: 503] ``` -**Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the image processing pipeline. +**Supported formats:** Only `image/png` and `image/jpeg` are accepted. GIF and WebP are rejected at the validation layer. This eliminates the need for a native image processing library (sharp/libvips) — raw image bytes are passed directly to Bedrock for screening. -**Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks. **Prerequisite:** Verify that the installed version of `@aws-sdk/client-bedrock-runtime` in `cdk/package.json` supports image content blocks in `ApplyGuardrail`. If it does not, a dependency upgrade is required before Phase 1. +**Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the screening pipeline. -**Format limitation:** The Bedrock `GuardrailImageBlock` API **only supports `png` and `jpeg` formats**. GIF and WebP images must be converted to PNG via `sharp` before screening. This conversion happens before the `ApplyGuardrailCommand` call (not after — the screened content must be the same content that reaches the agent): +**Dimension checks:** Image dimensions are read from PNG IHDR chunks and JPEG SOF markers using pure buffer parsing (no native dependencies). Images exceeding 8000px on either side are rejected before the Bedrock call. -```typescript -// Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) -let screeningBuffer: Buffer; -let screeningFormat: 'png' | 'jpeg'; - -if (contentType === 'image/jpeg') { - screeningBuffer = imageBuffer; - screeningFormat = 'jpeg'; -} else if (['image/gif', 'image/webp'].includes(contentType)) { - // GIF/WebP → PNG. For animated GIFs, extract first frame only (prevents OOM). - screeningBuffer = await sharp(imageBuffer, { animated: false }).png().toBuffer(); - screeningFormat = 'png'; - - // Post-conversion size check: PNG expansion of compressed GIF/WebP can exceed 10 MB. - // Fail with a clear error rather than letting a downstream size check reject it opaquely. - if (screeningBuffer.length > MAX_ATTACHMENT_SIZE_BYTES) { - throw new AttachmentResolutionError( - `Image "${filename}" is ${contentType} and its PNG conversion for screening ` + - `exceeds the ${MAX_ATTACHMENT_SIZE_BYTES / (1024 * 1024)} MB limit ` + - `(${(screeningBuffer.length / (1024 * 1024)).toFixed(1)} MB after conversion). ` + - `Please convert to JPEG or reduce image dimensions before uploading.` - ); - } -} else { - // PNG: use as-is - screeningBuffer = imageBuffer; - screeningFormat = 'png'; -} - -const result = await retryWithBackoff(() => - bedrockClient.send(new ApplyGuardrailCommand({ - guardrailIdentifier: GUARDRAIL_ID, - guardrailVersion: GUARDRAIL_VERSION, - source: 'INPUT', - content: [{ - image: { - format: screeningFormat, - source: { bytes: screeningBuffer }, - }, - }], - })), - { maxRetries: 3, baseDelayMs: 200, retryableErrors: [429, 500, 502, 503] }, -); -``` - -The GIF/WebP → PNG conversion uses `{ animated: false }` to extract only the first frame from animated GIFs, preventing unbounded memory usage. The post-conversion size check catches cases where a highly-compressed GIF/WebP expands beyond 10 MB as PNG — the user gets a clear error with remediation (convert to JPEG or reduce dimensions). The conversion error path (`ATTACHMENT_INVALID_CONTENT`) is shared with the EXIF stripping pipeline. - -**EXIF stripping + re-encoding:** After screening passes, the image is processed through `sharp`: -1. Strip all EXIF/IPTC/XMP metadata (GPS coordinates, device info, timestamps — prevents PII leakage). -2. Re-encode the image in the same format. This strips any non-image data that may have been appended to the file (steganography payloads, polyglot trailing data). -3. The re-encoded image is what gets stored in S3 and delivered to the agent. +**Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks with `png` and `jpeg` formats. Raw image bytes are passed directly — no re-encoding or format conversion needed. -**sharp failure handling:** If `sharp` cannot process an image (corrupt image that passes magic bytes check, OOM on large image, library bug), the attachment is **rejected** with `ATTACHMENT_INVALID_CONTENT` and message: "Image could not be processed for security sanitization. Please re-export the image in a standard format and try again." This preserves the security guarantee — no un-sanitized images reach the agent. Fail-closed, not fail-open. +**No EXIF stripping:** Images are stored as-is after screening passes. EXIF metadata (GPS, camera info) is not stripped since attachments are uploaded by the task submitter for their own agent. This trade-off eliminates the native `sharp`/`libvips` dependency, which caused cross-platform build issues and Lambda ARM64 decode failures. ### File screening @@ -515,8 +463,6 @@ flowchart TD | `text/*` | Valid UTF-8, no null bytes in first 8 KB | | `image/png` | `\x89PNG\r\n\x1a\n` | | `image/jpeg` | `\xFF\xD8\xFF` | -| `image/gif` | `GIF87a` or `GIF89a` | -| `image/webp` | `RIFF....WEBP` | A file claiming to be `text/plain` but starting with `MZ` (PE executable) or `PK` (ZIP) is rejected immediately. @@ -672,7 +618,7 @@ const resolvedAttachments = await resolveAttachments( `resolveAttachments()` handles: 1. **Inline/presigned attachments (already screened, already in S3):** Validate S3 key exists, compute `s3_uri` for the agent. -2. **URL attachments (not yet fetched):** Fetch (with SSRF protections), screen (with retry), sanitize (EXIF strip / re-encode), upload to S3, compute `s3_uri`. On any failure, throw `AttachmentResolutionError` which fails the task. +2. **URL attachments (not yet fetched):** Fetch (with SSRF protections), screen (with retry), upload to S3, compute `s3_uri`. On any failure, throw `AttachmentResolutionError` which fails the task. 3. **Token budget accounting:** Estimate token cost of image attachments and deduct from the available prompt budget (see [Token budget](#token-budget-accounting)). ### Payload changes @@ -782,7 +728,7 @@ async function resolveAttachments(attachments, ...) { for (const att of attachments) { if (att.type === 'image') { - // getImageDimensions uses sharp.metadata() on the S3 content. + // getImageDimensions parses PNG IHDR / JPEG SOF markers from the buffer. // If dimensions cannot be determined (corrupt image, unsupported format variant), // throw AttachmentResolutionError — never default to (0,0) or skip the estimate. let width: number, height: number; @@ -1027,7 +973,7 @@ When a user mentions `@Shoof` in a message that contains file uploads or image a |---|---| | File too large | "Task not created. `{filename}` is too large ({size} MB, max 10 MB). Please reduce the file size or remove it and try again." | | Screening blocked | "Task not created. `{filename}` was blocked by content screening ({categories}). Please remove this file and try again." | -| Unsupported MIME type | "Task not created. `{filename}` has unsupported type `{mime}`. Supported: images (png, jpeg, gif, webp) and text files (txt, csv, json, md, pdf, log)." | +| Unsupported MIME type | "Task not created. `{filename}` has unsupported type `{mime}`. Supported: images (png, jpeg) and text files (txt, csv, json, md, pdf, log)." | | S3 upload failure | "Task not created. Failed to process `{filename}`. Please try again." | | Multiple failures | "Task not created. 2 attachment errors: `{file1}` (blocked by content screening), `{file2}` (too large, 15 MB > 10 MB limit). Fix or remove these files and try again." | @@ -1399,12 +1345,11 @@ New construct at `cdk/src/constructs/attachments-bucket.ts`. Follows `TraceArtif ### New: `ConfirmUploadsFunction` A separate Lambda for the `confirm-uploads` endpoint, with: -- **Memory:** 2048 MB (must hold up to 10 MB raw image + decompressed pixel buffer for `sharp` re-encoding; `sharp` decompresses a 4096x4096 RGBA image to ~64 MB in memory) -- **Timeout:** 180 seconds (3 minutes). Attachments are screened in **parallel with bounded concurrency of 3**. Worst-case: 4 batches of ~45s each (S3 read + Bedrock screen with retries + sharp re-encode + S3 write). The 180s budget accommodates Bedrock retry delays. +- **Memory:** 1024 MB (holds up to 10 MB raw image bytes in memory for Bedrock screening; no native image processing) +- **Timeout:** 180 seconds (3 minutes). Attachments are screened in **parallel with bounded concurrency of 3**. Worst-case: 4 batches of ~45s each (S3 read + Bedrock screen with retries + S3 write). The 180s budget accommodates Bedrock retry delays. - **Internal deadline timer:** The handler sets a deadline at `Lambda timeout - 15 seconds` (165s). If the screening loop has not completed by this deadline, remaining unscreened attachments are aborted and the handler returns a 503 with `Retry-After: 30` header and body: "Attachment screening did not complete within the time limit. Reduce the number or size of attachments and try again, or retry after 30 seconds (already-screened attachments will be skipped on retry)." The `Retry-After` header enables clients to implement automatic backoff. On retry, the per-attachment screening state (above) ensures only unscreened attachments are re-processed, so retries make forward progress. This prevents opaque Lambda timeout errors. -- **Per-attachment screening state with atomic DynamoDB + S3 ordering:** Each attachment's screening pipeline follows a strict order: (1) screen content, (2) sanitize/re-encode via sharp, (3) PutObject to S3 (the cleaned version), (4) update the attachment's `screening` status to `passed` in DynamoDB (with `s3_version_id` from the PutObject response). The DynamoDB write is the **commit point** — if any prior step fails, the attachment remains in `pending` status. On retry (after a timeout or Lambda restart), the handler skips attachments with `screening.status === 'passed'` (already committed to both S3 and DynamoDB). Attachments still in `pending` are re-processed from step 1 — this is safe because S3 PutObject is idempotent and the version ID from the new put supersedes any orphaned partial upload. This ordering ensures no attachment is marked as `passed` in DynamoDB without the corresponding cleaned content being in S3. -- **Bundled dependencies:** `sharp` (for EXIF stripping + re-encoding), `pdf-parse` (for PDF text extraction) -- **Cold-start validation:** On module initialization, the handler verifies `sharp` loads correctly (e.g., `sharp(Buffer.alloc(1)).metadata()` wrapped in try-catch). If the native module fails to load (architecture mismatch, missing binary), the handler short-circuits all requests with 503: "Attachment processing is temporarily unavailable. Please try again later." This prevents opaque `Runtime.ImportModuleError` failures from reaching users. +- **Per-attachment screening state with atomic DynamoDB + S3 ordering:** Each attachment's screening pipeline follows a strict order: (1) screen content via Bedrock Guardrail, (2) PutObject to S3, (3) update the attachment's `screening` status to `passed` in DynamoDB (with `s3_version_id` from the PutObject response). The DynamoDB write is the **commit point** — if any prior step fails, the attachment remains in `pending` status. On retry (after a timeout or Lambda restart), the handler skips attachments with `screening.status === 'passed'` (already committed to both S3 and DynamoDB). Attachments still in `pending` are re-processed from step 1 — this is safe because S3 PutObject is idempotent and the version ID from the new put supersedes any orphaned partial upload. +- **Bundled dependencies:** `pdf-parse` (for PDF text extraction) Separating this from the create-task Lambda keeps the common path (task creation without attachments) lean and fast. @@ -1523,7 +1468,7 @@ New error codes for attachment-related failures: | `ATTACHMENTS_INLINE_TOTAL_TOO_LARGE` | 400 | Total inline attachment size exceeds 3 MB limit | | `ATTACHMENTS_TOTAL_TOO_LARGE` | 400 | Total attachment size exceeds 50 MB limit | | `ATTACHMENT_INVALID_TYPE` | 400 | MIME type not in allowlist | -| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or could not be sanitized (sharp failure) | +| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or image dimensions exceed limits | | `ATTACHMENT_INVALID_FILENAME` | 400 | Filename contains invalid characters or path traversal | | `ATTACHMENT_SIZE_MISMATCH` | 400 | Uploaded file size does not match declared `expected_size_bytes` (> 10% deviation) | | `ATTACHMENT_FETCH_FAILED` | 422 | URL attachment could not be fetched (timeout, DNS, SSRF blocked) | @@ -1555,7 +1500,6 @@ New error codes for attachment-related failures: | `OrphanedAttachmentNoTask` | — | Objects uploaded before task was persisted to DynamoDB (pre-write failures); no task record to correlate | | `PendingUploadExpired` | — | Tasks that expired in PENDING_UPLOADS (never confirmed) | | `ConfirmUploadsRace` | — | Concurrent confirm-uploads detected (condition check failed) | -| `SharpColdStartFailure` | — | `sharp` native module failed to load on Lambda cold start | | `ScreeningDeadlineExceeded` | — | Confirm-uploads hit internal deadline timer before all attachments screened | ### Task events @@ -1578,14 +1522,14 @@ New event types in `TaskEventsTable`: | Threat | Vector | Mitigation | |---|---|---| -| Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; Bedrock image screening; EXIF stripping; image re-encoding through `sharp` strips embedded payloads; sharp failure → reject | +| Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; dimension checks; Bedrock Guardrail image screening (detects harmful visual content); only PNG/JPEG accepted (no executable formats) | | Prompt injection via file content | Text file containing adversarial instructions | Magic bytes validation; Bedrock Guardrail text screening with retry (same as task descriptions); content trust tagging as `untrusted-external` | | SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time; content-type allowlist + magic bytes validation after fetch (prevents attacker-controlled Content-Type header from bypassing type restrictions) | | Data exfiltration via URL attachment | URL pointing to attacker-controlled server (leaks request headers/IP) | No auth headers sent to non-GitHub URLs; minimal request headers; no cookies | | Denial of service via large attachments | Many large base64 payloads | 500 KB inline limit; 3 MB total inline; 10 MB per-attachment; 50 MB total; 10 count limit; 6 MB Lambda payload limit | | Path traversal via filename | `filename: "../../etc/passwd"` | Filename sanitization regex; reject path separators, dots-prefix, null bytes; use `attachment_id` as primary path component | | Zip bomb / decompression bomb | Compressed content that expands massively | No archive types in MIME allowlist; PDF text extraction capped at 50 pages and 1 MB output | -| Polyglot files | File with valid image header + appended executable | Magic bytes validation at upload; image re-encoding strips trailing data | +| Polyglot files | File with valid image header + appended executable | Magic bytes validation at upload; only PNG/JPEG image types accepted; Bedrock screens the actual image content | | Presigned URL abuse | Leaked presigned POST policy used to upload different content | Content-Type fixed in presigned POST policy; `content-length-range` enforced by S3 (rejects > 10 MB before writing); 10-minute expiry; screening runs after upload regardless; size verified via HeadObject (defense-in-depth) | | S3 object replacement (TOCTOU) | Client uploads benign content (passes screening), then replaces object with malicious content before agent downloads | S3 versioning enabled; `VersionId` pinned at screening time and stored in `AttachmentRecord`; agent downloads with pinned `VersionId`; `noncurrentVersionExpiration: 7 days` prevents storage bloat while allowing long-running tasks to complete | | Upload slot exhaustion | Create many tasks in PENDING_UPLOADS, never confirm | 30-minute EventBridge auto-cancel with S3 cleanup; PENDING_UPLOADS does not count against concurrency; rate limiting on task creation already exists | @@ -1613,7 +1557,7 @@ This is correct even for inline uploads from authenticated users. The content of | S3 PUT/GET | ~$0.00001 | 10 PUTs + 10 GETs | | Bedrock Guardrail (image) | ~$0.01-0.05 per image | Depends on image size and guardrail config | | Bedrock Guardrail (text) | ~$0.001-0.01 per file | Same as existing text screening | -| Lambda compute (confirm-uploads) | ~$0.01-0.05 | 30-180s at 2048 MB for screening + re-encoding | +| Lambda compute (confirm-uploads) | ~$0.005-0.03 | 30-180s at 1024 MB for screening (no image re-encoding) | | Lambda compute (create-task, inline) | ~$0.001 | Additional 1-3s at 256 MB for small inline attachments | | Lambda compute (auto-cancel rule) | ~$0.0001 | 5-minute schedule, mostly no-op | | Data transfer (URL fetch) | ~$0.001 | Outbound fetch within region is free; cross-region is negligible | @@ -1640,8 +1584,8 @@ The implementation is ordered to deliver value incrementally while maintaining s 11. Add attachment validation to `validation.ts` (sync: schema, limits, magic bytes, content_type detection, filename generation; async: SSRF DNS pre-check with differentiated error messages) 12. Increase `MAX_TASK_DESCRIPTION_LENGTH` to 10,000 (standalone API change — update API_CONTRACT.md) 13. Add Bedrock screening retry logic (3 retries, exponential backoff) -14. Add GIF/WebP → PNG conversion before Bedrock screening (Bedrock only supports png|jpeg), with post-conversion size check -15. Add inline upload path to `create-task-core.ts` (base64 → magic bytes → screen with retry → EXIF strip/re-encode → S3 with versioning; sharp failure → ATTACHMENT_INVALID_CONTENT) +14. ~~Add GIF/WebP → PNG conversion~~ — Removed: only PNG/JPEG accepted (no native image dependency) +15. Add inline upload path to `create-task-core.ts` (base64 → magic bytes → dimension check → screen with retry → S3 with versioning) 16. Add SHA-256 checksum computation at upload time (stored in AttachmentRecord, required by factory) 17. Add partial failure cleanup with proper `DeleteObjects` error handling and metrics 18. Add `attachments` field to `TaskRecord` @@ -1650,13 +1594,13 @@ The implementation is ordered to deliver value incrementally while maintaining s 21. Increase create-task Lambda memory to 256 MB (from CDK default 128 MB), timeout to 15s (from CDK default 3s) 22. Update `API_CONTRACT.md` (inline limit, task_description limit, note Bedrock format constraints) -**Security included in Phase 1:** magic bytes validation, EXIF stripping, image re-encoding, GIF/WebP conversion (with post-conversion size check), sharp fail-closed, filename sanitization, partial failure cleanup, Bedrock retry, S3 versioning (TOCTOU prevention), SHA-256 integrity. +**Security included in Phase 1:** magic bytes validation, dimension checks, filename sanitization, partial failure cleanup, Bedrock retry, S3 versioning (TOCTOU prevention), SHA-256 integrity. ### Phase 2: Presigned upload + state machine (large attachments) 23. Add `PENDING_UPLOADS` to `task-status.ts` (status, transitions, `PRE_ACTIVE_STATUSES` classification) 24. Update all code paths that assume binary status classification (active vs terminal) — see [Impact on existing code](#impact-on-existing-code-that-assumes-binary-status-classification) -25. Add `confirm-uploads` Lambda (2048 MB, 180s timeout) with parallel screening (concurrency 3), internal deadline timer, per-attachment screening state, and sharp cold-start validation +25. Add `confirm-uploads` Lambda (1024 MB, 180s timeout) with parallel screening (concurrency 3), internal deadline timer, per-attachment screening state 26. Add `POST /v1/tasks/{task_id}/confirm-uploads` API endpoint with concurrent-call safety (early short-circuit, conditional DynamoDB write for both success and failure paths) 27. Move concurrency increment from create-task to confirm-uploads for presigned-upload tasks 28. Add presigned POST policy generation in create-task handler (with `content-length-range` enforcement, `expected_size_bytes` validation, S3 versioning) @@ -1692,5 +1636,5 @@ The implementation is ordered to deliver value incrementally while maintaining s 47. Add alerting on `OrphanedAttachmentCleanupFailure` and `PendingUploadExpired` 48. Add comprehensive integration tests (all upload paths, all failure modes, all channels, concurrent confirm-uploads, auto-cancel racing with confirm-uploads) 49. Add load testing for screening pipeline (sustained 10-attachment tasks at peak rate) -50. Add monitoring for `sharp` native module health (cold-start validation failures) -51. Add monitoring for format conversion size expansion (GIF/WebP → PNG rejection rate) +50. ~~Add monitoring for sharp native module health~~ — Removed: no native image dependencies +51. ~~Add monitoring for format conversion size expansion~~ — Removed: no format conversion diff --git a/docs/design/SECURITY.md b/docs/design/SECURITY.md index 1f7eaab2..e39eeee6 100644 --- a/docs/design/SECURITY.md +++ b/docs/design/SECURITY.md @@ -46,7 +46,7 @@ Input screening happens at two points in the pipeline, forming a defense-in-dept - **Input validation** - Required fields, types, and size limits are enforced before any processing. Task descriptions are capped at 10,000 characters. - **Bedrock Guardrails** - A `PROMPT_ATTACK` content filter at `MEDIUM` input strength screens task descriptions for prompt injection. -- **Attachment screening** - All attachments (images, text files, URLs) pass through security screening before reaching the agent. Images are validated via magic bytes, screened through Bedrock Guardrails (image content blocks), stripped of EXIF/metadata, and re-encoded. Text files and PDFs are extracted and screened through Bedrock Guardrails text content screening. URL attachments undergo SSRF protection (DNS resolution pinning, private IP blocking, redirect validation) and content screening during hydration. See [ATTACHMENTS.md](./ATTACHMENTS.md) for the full screening pipeline. +- **Attachment screening** - All attachments (images, text files, URLs) pass through security screening before reaching the agent. Images (PNG and JPEG only) are validated via magic bytes and dimension checks, then screened through Bedrock Guardrails (image content blocks). Text files and PDFs are extracted and screened through Bedrock Guardrails text content screening. URL attachments undergo SSRF protection (DNS resolution pinning, private IP blocking, redirect validation) and content screening during hydration. See [ATTACHMENTS.md](./ATTACHMENTS.md) for the full screening pipeline. - **Fail-closed** - If the Bedrock API is unavailable, submissions are rejected (HTTP 503). Unscreened content never reaches the agent. ### Hydration-time screening diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index 22dc3484..7888c547 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -459,7 +459,7 @@ Attachments let you provide non-text context to the agent — screenshots of bug | Category | Types | Extensions | |---|---|---| -| Images | PNG, JPEG, GIF, WebP | `.png`, `.jpg`, `.gif`, `.webp` | +| Images | PNG, JPEG | `.png`, `.jpg` | | Text files | Plain text, CSV, Markdown, JSON, PDF, Log | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | **Limits:** @@ -498,7 +498,7 @@ The CLI automatically routes attachments through the optimal upload path: All attachments are screened before reaching the agent: -- **Images**: Magic bytes validation, Bedrock Guardrail content screening (prompt attack detection), EXIF/metadata stripping, re-encoding to remove embedded payloads. +- **Images**: Magic bytes validation, dimension checks (max 8000px per side), Bedrock Guardrail content screening (prompt attack detection). Only PNG and JPEG are accepted. - **Text files**: Magic bytes validation, Bedrock Guardrail text content screening. PDFs have text extracted (max 50 pages) before screening. - **URLs**: HTTPS-only enforcement, DNS resolution pinning (prevents DNS rebinding/SSRF), private IP blocking, redirect validation, size and timeout limits. diff --git a/docs/src/content/docs/architecture/Api-contract.md b/docs/src/content/docs/architecture/Api-contract.md index 1545dee8..32d8fc27 100644 --- a/docs/src/content/docs/architecture/Api-contract.md +++ b/docs/src/content/docs/architecture/Api-contract.md @@ -106,7 +106,7 @@ Creates a new task. The orchestrator runs admission control, context hydration, | Category | MIME types | Extensions | |---|---|---| -| Images | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | `.png`, `.jpg`, `.gif`, `.webp` | +| Images | `image/png`, `image/jpeg` | `.png`, `.jpg` | | Text files | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | **Attachment limits:** diff --git a/docs/src/content/docs/architecture/Attachments.md b/docs/src/content/docs/architecture/Attachments.md index 5f32dd8d..b5a792eb 100644 --- a/docs/src/content/docs/architecture/Attachments.md +++ b/docs/src/content/docs/architecture/Attachments.md @@ -145,7 +145,7 @@ function createAttachmentRecord(params: CreateAttachmentRecordParams): Attachmen // - checksum_sha256 required when screening.status === 'passed' // - size_bytes required when screening.status === 'passed' // - categories non-empty when screening.status === 'blocked' - // Note: token_estimate is best-effort for images (sharp may fail on unusual formats), + // Note: token_estimate is best-effort for images (dimension parsing may fail on unusual formats), // so it is NOT enforced here. The budget check uses a conservative fallback when absent. if (params.screening.status === 'passed') { if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { @@ -198,7 +198,7 @@ The `` segment ensures uniqueness even if multiple attachments sh | Max total inline data per request | 3 MB decoded | Hard cap on total base64-decoded bytes in a single request. Even with base64 overhead (~4 MB encoded) plus JSON fields, this stays under the 6 MB Lambda payload limit. | | Max total size per task | 50 MB | Prevents abuse; bounds total screening and transfer time | | Max task_description length | 10,000 chars | Increased from 2,000. **This is a standalone API change that affects all tasks** (not just attachment tasks). Rationale: (a) attachments need rich explanatory context ("implement this design per the attached mockup, paying attention to the header layout"), (b) multiple users have reported the 2K limit as a friction point for complex task descriptions even without attachments, (c) the guardrail screening cost increase is minimal (text screening is cheap), (d) DynamoDB item size impact is negligible (~8 KB vs ~2 KB for the description field). **Requires updating [API_CONTRACT.md](/architecture/api-contract) line 82 in tandem.** | -| Allowed image MIME types | `image/png`, `image/jpeg`, `image/gif`, `image/webp` | Bedrock vision-supported formats | +| Allowed image MIME types | `image/png`, `image/jpeg` | Bedrock-supported formats; GIF/WebP removed to eliminate native image processing dependency | | Allowed file MIME types | `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/pdf`, `text/x-log` | Useful for code/data context; no executables | | Max URL fetch size | 10 MB | Same per-attachment limit for fetched content | | URL fetch timeout | 10 seconds | Prevent SSRF-style long-poll attacks | @@ -248,7 +248,6 @@ sequenceDiagram H-->>C: 400 ATTACHMENT_BLOCKED { attachment_id, reason } end end - H->>H: Strip EXIF from images, re-upload cleaned version H->>DB: Update TaskRecord (status: SUBMITTED, screening: passed) with condition: status = PENDING_UPLOADS H->>H: Async-invoke Orchestrator H-->>C: 200 OK { task_id, status: SUBMITTED } @@ -332,8 +331,7 @@ sequenceDiagram H->>H: Cleanup already-uploaded S3 objects H-->>C: 400 ATTACHMENT_BLOCKED end - H->>H: Strip EXIF (images) / sanitize; on sharp failure → ATTACHMENT_INVALID_CONTENT - H->>S3: PutObject (cleaned content) + H->>S3: PutObject (screened content) H->>H: Build AttachmentRecord end H->>DB: Write TaskRecord (status: SUBMITTED, with attachment metadata) @@ -368,7 +366,6 @@ sequenceDiagram O->>SCR: Screen fetched content (with retry) SCR-->>O: Pass / Blocked alt Screening passed - O->>O: Strip EXIF (images) / sanitize; on sharp failure → fail task O->>S3: PutObject O->>O: Update AttachmentRecord else Screening blocked or fetch failed @@ -424,76 +421,27 @@ This prevents a single transient Bedrock hiccup from failing an entire task afte ```mermaid flowchart TD - I[Image attachment] --> MB[Magic bytes: verify image signature] + I[Image attachment] --> FMT{PNG or JPEG?} + FMT -->|No: GIF/WebP| RJ[REJECTED: unsupported format] + FMT -->|Yes| MB[Magic bytes: verify image signature] MB -->|Invalid| R[REJECTED: not a valid image] - MB -->|Valid| V[Validate: decodable, dimensions ≤ 8192x8192] - V --> G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] + MB -->|Valid| D[Dimension check: parse PNG IHDR / JPEG SOF] + D -->|> 8000px| OV[REJECTED: oversized] + D -->|OK| G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] G -->|INTERVENED| B[BLOCKED: content policy violation] - G -->|NONE| M[EXIF strip + re-encode via sharp] - M -->|sharp error| SE[REJECTED: ATTACHMENT_INVALID_CONTENT] - M -->|Success| P[PASSED] + G -->|NONE| P[PASSED: original bytes stored as-is] G -->|Error after retries| F[FAIL CLOSED: 503] ``` -**Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the image processing pipeline. +**Supported formats:** Only `image/png` and `image/jpeg` are accepted. GIF and WebP are rejected at the validation layer. This eliminates the need for a native image processing library (sharp/libvips) — raw image bytes are passed directly to Bedrock for screening. -**Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks. **Prerequisite:** Verify that the installed version of `@aws-sdk/client-bedrock-runtime` in `cdk/package.json` supports image content blocks in `ApplyGuardrail`. If it does not, a dependency upgrade is required before Phase 1. +**Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the screening pipeline. -**Format limitation:** The Bedrock `GuardrailImageBlock` API **only supports `png` and `jpeg` formats**. GIF and WebP images must be converted to PNG via `sharp` before screening. This conversion happens before the `ApplyGuardrailCommand` call (not after — the screened content must be the same content that reaches the agent): +**Dimension checks:** Image dimensions are read from PNG IHDR chunks and JPEG SOF markers using pure buffer parsing (no native dependencies). Images exceeding 8000px on either side are rejected before the Bedrock call. -```typescript -// Convert GIF/WebP to PNG before screening (Bedrock only accepts png | jpeg) -let screeningBuffer: Buffer; -let screeningFormat: 'png' | 'jpeg'; - -if (contentType === 'image/jpeg') { - screeningBuffer = imageBuffer; - screeningFormat = 'jpeg'; -} else if (['image/gif', 'image/webp'].includes(contentType)) { - // GIF/WebP → PNG. For animated GIFs, extract first frame only (prevents OOM). - screeningBuffer = await sharp(imageBuffer, { animated: false }).png().toBuffer(); - screeningFormat = 'png'; - - // Post-conversion size check: PNG expansion of compressed GIF/WebP can exceed 10 MB. - // Fail with a clear error rather than letting a downstream size check reject it opaquely. - if (screeningBuffer.length > MAX_ATTACHMENT_SIZE_BYTES) { - throw new AttachmentResolutionError( - `Image "${filename}" is ${contentType} and its PNG conversion for screening ` + - `exceeds the ${MAX_ATTACHMENT_SIZE_BYTES / (1024 * 1024)} MB limit ` + - `(${(screeningBuffer.length / (1024 * 1024)).toFixed(1)} MB after conversion). ` + - `Please convert to JPEG or reduce image dimensions before uploading.` - ); - } -} else { - // PNG: use as-is - screeningBuffer = imageBuffer; - screeningFormat = 'png'; -} - -const result = await retryWithBackoff(() => - bedrockClient.send(new ApplyGuardrailCommand({ - guardrailIdentifier: GUARDRAIL_ID, - guardrailVersion: GUARDRAIL_VERSION, - source: 'INPUT', - content: [{ - image: { - format: screeningFormat, - source: { bytes: screeningBuffer }, - }, - }], - })), - { maxRetries: 3, baseDelayMs: 200, retryableErrors: [429, 500, 502, 503] }, -); -``` - -The GIF/WebP → PNG conversion uses `{ animated: false }` to extract only the first frame from animated GIFs, preventing unbounded memory usage. The post-conversion size check catches cases where a highly-compressed GIF/WebP expands beyond 10 MB as PNG — the user gets a clear error with remediation (convert to JPEG or reduce dimensions). The conversion error path (`ATTACHMENT_INVALID_CONTENT`) is shared with the EXIF stripping pipeline. - -**EXIF stripping + re-encoding:** After screening passes, the image is processed through `sharp`: -1. Strip all EXIF/IPTC/XMP metadata (GPS coordinates, device info, timestamps — prevents PII leakage). -2. Re-encode the image in the same format. This strips any non-image data that may have been appended to the file (steganography payloads, polyglot trailing data). -3. The re-encoded image is what gets stored in S3 and delivered to the agent. +**Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks with `png` and `jpeg` formats. Raw image bytes are passed directly — no re-encoding or format conversion needed. -**sharp failure handling:** If `sharp` cannot process an image (corrupt image that passes magic bytes check, OOM on large image, library bug), the attachment is **rejected** with `ATTACHMENT_INVALID_CONTENT` and message: "Image could not be processed for security sanitization. Please re-export the image in a standard format and try again." This preserves the security guarantee — no un-sanitized images reach the agent. Fail-closed, not fail-open. +**No EXIF stripping:** Images are stored as-is after screening passes. EXIF metadata (GPS, camera info) is not stripped since attachments are uploaded by the task submitter for their own agent. This trade-off eliminates the native `sharp`/`libvips` dependency, which caused cross-platform build issues and Lambda ARM64 decode failures. ### File screening @@ -519,8 +467,6 @@ flowchart TD | `text/*` | Valid UTF-8, no null bytes in first 8 KB | | `image/png` | `\x89PNG\r\n\x1a\n` | | `image/jpeg` | `\xFF\xD8\xFF` | -| `image/gif` | `GIF87a` or `GIF89a` | -| `image/webp` | `RIFF....WEBP` | A file claiming to be `text/plain` but starting with `MZ` (PE executable) or `PK` (ZIP) is rejected immediately. @@ -676,7 +622,7 @@ const resolvedAttachments = await resolveAttachments( `resolveAttachments()` handles: 1. **Inline/presigned attachments (already screened, already in S3):** Validate S3 key exists, compute `s3_uri` for the agent. -2. **URL attachments (not yet fetched):** Fetch (with SSRF protections), screen (with retry), sanitize (EXIF strip / re-encode), upload to S3, compute `s3_uri`. On any failure, throw `AttachmentResolutionError` which fails the task. +2. **URL attachments (not yet fetched):** Fetch (with SSRF protections), screen (with retry), upload to S3, compute `s3_uri`. On any failure, throw `AttachmentResolutionError` which fails the task. 3. **Token budget accounting:** Estimate token cost of image attachments and deduct from the available prompt budget (see [Token budget](#token-budget-accounting)). ### Payload changes @@ -786,7 +732,7 @@ async function resolveAttachments(attachments, ...) { for (const att of attachments) { if (att.type === 'image') { - // getImageDimensions uses sharp.metadata() on the S3 content. + // getImageDimensions parses PNG IHDR / JPEG SOF markers from the buffer. // If dimensions cannot be determined (corrupt image, unsupported format variant), // throw AttachmentResolutionError — never default to (0,0) or skip the estimate. let width: number, height: number; @@ -1031,7 +977,7 @@ When a user mentions `@Shoof` in a message that contains file uploads or image a |---|---| | File too large | "Task not created. `{filename}` is too large ({size} MB, max 10 MB). Please reduce the file size or remove it and try again." | | Screening blocked | "Task not created. `{filename}` was blocked by content screening ({categories}). Please remove this file and try again." | -| Unsupported MIME type | "Task not created. `{filename}` has unsupported type `{mime}`. Supported: images (png, jpeg, gif, webp) and text files (txt, csv, json, md, pdf, log)." | +| Unsupported MIME type | "Task not created. `{filename}` has unsupported type `{mime}`. Supported: images (png, jpeg) and text files (txt, csv, json, md, pdf, log)." | | S3 upload failure | "Task not created. Failed to process `{filename}`. Please try again." | | Multiple failures | "Task not created. 2 attachment errors: `{file1}` (blocked by content screening), `{file2}` (too large, 15 MB > 10 MB limit). Fix or remove these files and try again." | @@ -1403,12 +1349,11 @@ New construct at `cdk/src/constructs/attachments-bucket.ts`. Follows `TraceArtif ### New: `ConfirmUploadsFunction` A separate Lambda for the `confirm-uploads` endpoint, with: -- **Memory:** 2048 MB (must hold up to 10 MB raw image + decompressed pixel buffer for `sharp` re-encoding; `sharp` decompresses a 4096x4096 RGBA image to ~64 MB in memory) -- **Timeout:** 180 seconds (3 minutes). Attachments are screened in **parallel with bounded concurrency of 3**. Worst-case: 4 batches of ~45s each (S3 read + Bedrock screen with retries + sharp re-encode + S3 write). The 180s budget accommodates Bedrock retry delays. +- **Memory:** 1024 MB (holds up to 10 MB raw image bytes in memory for Bedrock screening; no native image processing) +- **Timeout:** 180 seconds (3 minutes). Attachments are screened in **parallel with bounded concurrency of 3**. Worst-case: 4 batches of ~45s each (S3 read + Bedrock screen with retries + S3 write). The 180s budget accommodates Bedrock retry delays. - **Internal deadline timer:** The handler sets a deadline at `Lambda timeout - 15 seconds` (165s). If the screening loop has not completed by this deadline, remaining unscreened attachments are aborted and the handler returns a 503 with `Retry-After: 30` header and body: "Attachment screening did not complete within the time limit. Reduce the number or size of attachments and try again, or retry after 30 seconds (already-screened attachments will be skipped on retry)." The `Retry-After` header enables clients to implement automatic backoff. On retry, the per-attachment screening state (above) ensures only unscreened attachments are re-processed, so retries make forward progress. This prevents opaque Lambda timeout errors. -- **Per-attachment screening state with atomic DynamoDB + S3 ordering:** Each attachment's screening pipeline follows a strict order: (1) screen content, (2) sanitize/re-encode via sharp, (3) PutObject to S3 (the cleaned version), (4) update the attachment's `screening` status to `passed` in DynamoDB (with `s3_version_id` from the PutObject response). The DynamoDB write is the **commit point** — if any prior step fails, the attachment remains in `pending` status. On retry (after a timeout or Lambda restart), the handler skips attachments with `screening.status === 'passed'` (already committed to both S3 and DynamoDB). Attachments still in `pending` are re-processed from step 1 — this is safe because S3 PutObject is idempotent and the version ID from the new put supersedes any orphaned partial upload. This ordering ensures no attachment is marked as `passed` in DynamoDB without the corresponding cleaned content being in S3. -- **Bundled dependencies:** `sharp` (for EXIF stripping + re-encoding), `pdf-parse` (for PDF text extraction) -- **Cold-start validation:** On module initialization, the handler verifies `sharp` loads correctly (e.g., `sharp(Buffer.alloc(1)).metadata()` wrapped in try-catch). If the native module fails to load (architecture mismatch, missing binary), the handler short-circuits all requests with 503: "Attachment processing is temporarily unavailable. Please try again later." This prevents opaque `Runtime.ImportModuleError` failures from reaching users. +- **Per-attachment screening state with atomic DynamoDB + S3 ordering:** Each attachment's screening pipeline follows a strict order: (1) screen content via Bedrock Guardrail, (2) PutObject to S3, (3) update the attachment's `screening` status to `passed` in DynamoDB (with `s3_version_id` from the PutObject response). The DynamoDB write is the **commit point** — if any prior step fails, the attachment remains in `pending` status. On retry (after a timeout or Lambda restart), the handler skips attachments with `screening.status === 'passed'` (already committed to both S3 and DynamoDB). Attachments still in `pending` are re-processed from step 1 — this is safe because S3 PutObject is idempotent and the version ID from the new put supersedes any orphaned partial upload. +- **Bundled dependencies:** `pdf-parse` (for PDF text extraction) Separating this from the create-task Lambda keeps the common path (task creation without attachments) lean and fast. @@ -1527,7 +1472,7 @@ New error codes for attachment-related failures: | `ATTACHMENTS_INLINE_TOTAL_TOO_LARGE` | 400 | Total inline attachment size exceeds 3 MB limit | | `ATTACHMENTS_TOTAL_TOO_LARGE` | 400 | Total attachment size exceeds 50 MB limit | | `ATTACHMENT_INVALID_TYPE` | 400 | MIME type not in allowlist | -| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or could not be sanitized (sharp failure) | +| `ATTACHMENT_INVALID_CONTENT` | 400 | Content does not match declared MIME type (magic bytes mismatch) or image dimensions exceed limits | | `ATTACHMENT_INVALID_FILENAME` | 400 | Filename contains invalid characters or path traversal | | `ATTACHMENT_SIZE_MISMATCH` | 400 | Uploaded file size does not match declared `expected_size_bytes` (> 10% deviation) | | `ATTACHMENT_FETCH_FAILED` | 422 | URL attachment could not be fetched (timeout, DNS, SSRF blocked) | @@ -1559,7 +1504,6 @@ New error codes for attachment-related failures: | `OrphanedAttachmentNoTask` | — | Objects uploaded before task was persisted to DynamoDB (pre-write failures); no task record to correlate | | `PendingUploadExpired` | — | Tasks that expired in PENDING_UPLOADS (never confirmed) | | `ConfirmUploadsRace` | — | Concurrent confirm-uploads detected (condition check failed) | -| `SharpColdStartFailure` | — | `sharp` native module failed to load on Lambda cold start | | `ScreeningDeadlineExceeded` | — | Confirm-uploads hit internal deadline timer before all attachments screened | ### Task events @@ -1582,14 +1526,14 @@ New event types in `TaskEventsTable`: | Threat | Vector | Mitigation | |---|---|---| -| Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; Bedrock image screening; EXIF stripping; image re-encoding through `sharp` strips embedded payloads; sharp failure → reject | +| Malicious image (steganography, exploit payload) | Inline upload or URL | Magic bytes validation; dimension checks; Bedrock Guardrail image screening (detects harmful visual content); only PNG/JPEG accepted (no executable formats) | | Prompt injection via file content | Text file containing adversarial instructions | Magic bytes validation; Bedrock Guardrail text screening with retry (same as task descriptions); content trust tagging as `untrusted-external` | | SSRF via URL attachment | URL pointing to internal network | HTTPS-only; DNS resolution with manual connect to resolved IP (prevents rebinding TOCTOU); redirect validation; private IP blocking; applied at fetch time; content-type allowlist + magic bytes validation after fetch (prevents attacker-controlled Content-Type header from bypassing type restrictions) | | Data exfiltration via URL attachment | URL pointing to attacker-controlled server (leaks request headers/IP) | No auth headers sent to non-GitHub URLs; minimal request headers; no cookies | | Denial of service via large attachments | Many large base64 payloads | 500 KB inline limit; 3 MB total inline; 10 MB per-attachment; 50 MB total; 10 count limit; 6 MB Lambda payload limit | | Path traversal via filename | `filename: "../../etc/passwd"` | Filename sanitization regex; reject path separators, dots-prefix, null bytes; use `attachment_id` as primary path component | | Zip bomb / decompression bomb | Compressed content that expands massively | No archive types in MIME allowlist; PDF text extraction capped at 50 pages and 1 MB output | -| Polyglot files | File with valid image header + appended executable | Magic bytes validation at upload; image re-encoding strips trailing data | +| Polyglot files | File with valid image header + appended executable | Magic bytes validation at upload; only PNG/JPEG image types accepted; Bedrock screens the actual image content | | Presigned URL abuse | Leaked presigned POST policy used to upload different content | Content-Type fixed in presigned POST policy; `content-length-range` enforced by S3 (rejects > 10 MB before writing); 10-minute expiry; screening runs after upload regardless; size verified via HeadObject (defense-in-depth) | | S3 object replacement (TOCTOU) | Client uploads benign content (passes screening), then replaces object with malicious content before agent downloads | S3 versioning enabled; `VersionId` pinned at screening time and stored in `AttachmentRecord`; agent downloads with pinned `VersionId`; `noncurrentVersionExpiration: 7 days` prevents storage bloat while allowing long-running tasks to complete | | Upload slot exhaustion | Create many tasks in PENDING_UPLOADS, never confirm | 30-minute EventBridge auto-cancel with S3 cleanup; PENDING_UPLOADS does not count against concurrency; rate limiting on task creation already exists | @@ -1617,7 +1561,7 @@ This is correct even for inline uploads from authenticated users. The content of | S3 PUT/GET | ~$0.00001 | 10 PUTs + 10 GETs | | Bedrock Guardrail (image) | ~$0.01-0.05 per image | Depends on image size and guardrail config | | Bedrock Guardrail (text) | ~$0.001-0.01 per file | Same as existing text screening | -| Lambda compute (confirm-uploads) | ~$0.01-0.05 | 30-180s at 2048 MB for screening + re-encoding | +| Lambda compute (confirm-uploads) | ~$0.005-0.03 | 30-180s at 1024 MB for screening (no image re-encoding) | | Lambda compute (create-task, inline) | ~$0.001 | Additional 1-3s at 256 MB for small inline attachments | | Lambda compute (auto-cancel rule) | ~$0.0001 | 5-minute schedule, mostly no-op | | Data transfer (URL fetch) | ~$0.001 | Outbound fetch within region is free; cross-region is negligible | @@ -1644,8 +1588,8 @@ The implementation is ordered to deliver value incrementally while maintaining s 11. Add attachment validation to `validation.ts` (sync: schema, limits, magic bytes, content_type detection, filename generation; async: SSRF DNS pre-check with differentiated error messages) 12. Increase `MAX_TASK_DESCRIPTION_LENGTH` to 10,000 (standalone API change — update API_CONTRACT.md) 13. Add Bedrock screening retry logic (3 retries, exponential backoff) -14. Add GIF/WebP → PNG conversion before Bedrock screening (Bedrock only supports png|jpeg), with post-conversion size check -15. Add inline upload path to `create-task-core.ts` (base64 → magic bytes → screen with retry → EXIF strip/re-encode → S3 with versioning; sharp failure → ATTACHMENT_INVALID_CONTENT) +14. ~~Add GIF/WebP → PNG conversion~~ — Removed: only PNG/JPEG accepted (no native image dependency) +15. Add inline upload path to `create-task-core.ts` (base64 → magic bytes → dimension check → screen with retry → S3 with versioning) 16. Add SHA-256 checksum computation at upload time (stored in AttachmentRecord, required by factory) 17. Add partial failure cleanup with proper `DeleteObjects` error handling and metrics 18. Add `attachments` field to `TaskRecord` @@ -1654,13 +1598,13 @@ The implementation is ordered to deliver value incrementally while maintaining s 21. Increase create-task Lambda memory to 256 MB (from CDK default 128 MB), timeout to 15s (from CDK default 3s) 22. Update `API_CONTRACT.md` (inline limit, task_description limit, note Bedrock format constraints) -**Security included in Phase 1:** magic bytes validation, EXIF stripping, image re-encoding, GIF/WebP conversion (with post-conversion size check), sharp fail-closed, filename sanitization, partial failure cleanup, Bedrock retry, S3 versioning (TOCTOU prevention), SHA-256 integrity. +**Security included in Phase 1:** magic bytes validation, dimension checks, filename sanitization, partial failure cleanup, Bedrock retry, S3 versioning (TOCTOU prevention), SHA-256 integrity. ### Phase 2: Presigned upload + state machine (large attachments) 23. Add `PENDING_UPLOADS` to `task-status.ts` (status, transitions, `PRE_ACTIVE_STATUSES` classification) 24. Update all code paths that assume binary status classification (active vs terminal) — see [Impact on existing code](#impact-on-existing-code-that-assumes-binary-status-classification) -25. Add `confirm-uploads` Lambda (2048 MB, 180s timeout) with parallel screening (concurrency 3), internal deadline timer, per-attachment screening state, and sharp cold-start validation +25. Add `confirm-uploads` Lambda (1024 MB, 180s timeout) with parallel screening (concurrency 3), internal deadline timer, per-attachment screening state 26. Add `POST /v1/tasks/{task_id}/confirm-uploads` API endpoint with concurrent-call safety (early short-circuit, conditional DynamoDB write for both success and failure paths) 27. Move concurrency increment from create-task to confirm-uploads for presigned-upload tasks 28. Add presigned POST policy generation in create-task handler (with `content-length-range` enforcement, `expected_size_bytes` validation, S3 versioning) @@ -1696,5 +1640,5 @@ The implementation is ordered to deliver value incrementally while maintaining s 47. Add alerting on `OrphanedAttachmentCleanupFailure` and `PendingUploadExpired` 48. Add comprehensive integration tests (all upload paths, all failure modes, all channels, concurrent confirm-uploads, auto-cancel racing with confirm-uploads) 49. Add load testing for screening pipeline (sustained 10-attachment tasks at peak rate) -50. Add monitoring for `sharp` native module health (cold-start validation failures) -51. Add monitoring for format conversion size expansion (GIF/WebP → PNG rejection rate) +50. ~~Add monitoring for sharp native module health~~ — Removed: no native image dependencies +51. ~~Add monitoring for format conversion size expansion~~ — Removed: no format conversion diff --git a/docs/src/content/docs/architecture/Security.md b/docs/src/content/docs/architecture/Security.md index 0da72358..744148b9 100644 --- a/docs/src/content/docs/architecture/Security.md +++ b/docs/src/content/docs/architecture/Security.md @@ -50,7 +50,7 @@ Input screening happens at two points in the pipeline, forming a defense-in-dept - **Input validation** - Required fields, types, and size limits are enforced before any processing. Task descriptions are capped at 10,000 characters. - **Bedrock Guardrails** - A `PROMPT_ATTACK` content filter at `MEDIUM` input strength screens task descriptions for prompt injection. -- **Attachment screening** - All attachments (images, text files, URLs) pass through security screening before reaching the agent. Images are validated via magic bytes, screened through Bedrock Guardrails (image content blocks), stripped of EXIF/metadata, and re-encoded. Text files and PDFs are extracted and screened through Bedrock Guardrails text content screening. URL attachments undergo SSRF protection (DNS resolution pinning, private IP blocking, redirect validation) and content screening during hydration. See [ATTACHMENTS.md](/architecture/attachments) for the full screening pipeline. +- **Attachment screening** - All attachments (images, text files, URLs) pass through security screening before reaching the agent. Images (PNG and JPEG only) are validated via magic bytes and dimension checks, then screened through Bedrock Guardrails (image content blocks). Text files and PDFs are extracted and screened through Bedrock Guardrails text content screening. URL attachments undergo SSRF protection (DNS resolution pinning, private IP blocking, redirect validation) and content screening during hydration. See [ATTACHMENTS.md](/architecture/attachments) for the full screening pipeline. - **Fail-closed** - If the Bedrock API is unavailable, submissions are rejected (HTTP 503). Unscreened content never reaches the agent. ### Hydration-time screening diff --git a/docs/src/content/docs/using/Using-the-cli.md b/docs/src/content/docs/using/Using-the-cli.md index f1782920..0de774b4 100644 --- a/docs/src/content/docs/using/Using-the-cli.md +++ b/docs/src/content/docs/using/Using-the-cli.md @@ -101,7 +101,7 @@ Attachments let you provide non-text context to the agent — screenshots of bug | Category | Types | Extensions | |---|---|---| -| Images | PNG, JPEG, GIF, WebP | `.png`, `.jpg`, `.gif`, `.webp` | +| Images | PNG, JPEG | `.png`, `.jpg` | | Text files | Plain text, CSV, Markdown, JSON, PDF, Log | `.txt`, `.csv`, `.md`, `.json`, `.pdf`, `.log` | **Limits:** @@ -140,7 +140,7 @@ The CLI automatically routes attachments through the optimal upload path: All attachments are screened before reaching the agent: -- **Images**: Magic bytes validation, Bedrock Guardrail content screening (prompt attack detection), EXIF/metadata stripping, re-encoding to remove embedded payloads. +- **Images**: Magic bytes validation, dimension checks (max 8000px per side), Bedrock Guardrail content screening (prompt attack detection). Only PNG and JPEG are accepted. - **Text files**: Magic bytes validation, Bedrock Guardrail text content screening. PDFs have text extracted (max 50 pages) before screening. - **URLs**: HTTPS-only enforcement, DNS resolution pinning (prevents DNS rebinding/SSRF), private IP blocking, redirect validation, size and timeout limits. diff --git a/yarn.lock b/yarn.lock index f3085af5..cc406036 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2633,13 +2633,6 @@ "@emnapi/wasi-threads" "1.2.0" tslib "^2.4.0" -"@emnapi/runtime@^1.2.0": - version "1.10.0" - resolved "https://registry.yarnpkg.com/@emnapi/runtime/-/runtime-1.10.0.tgz#4b260c0d3534204e98c6110b8db1a987d26ec87c" - integrity sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA== - dependencies: - tslib "^2.4.0" - "@emnapi/runtime@^1.4.3": version "1.9.1" resolved "https://registry.yarnpkg.com/@emnapi/runtime/-/runtime-1.9.1.tgz#115ff2a0d589865be6bd8e9d701e499c473f2a8d" @@ -2940,13 +2933,6 @@ resolved "https://registry.yarnpkg.com/@img/colour/-/colour-1.1.0.tgz#b0c2c2fa661adf75effd6b4964497cd80010bb9d" integrity sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ== -"@img/sharp-darwin-arm64@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz#ef5b5a07862805f1e8145a377c8ba6e98813ca08" - integrity sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ== - optionalDependencies: - "@img/sharp-libvips-darwin-arm64" "1.0.4" - "@img/sharp-darwin-arm64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz#6e0732dcade126b6670af7aa17060b926835ea86" @@ -2954,13 +2940,6 @@ optionalDependencies: "@img/sharp-libvips-darwin-arm64" "1.2.4" -"@img/sharp-darwin-x64@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz#e03d3451cd9e664faa72948cc70a403ea4063d61" - integrity sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q== - optionalDependencies: - "@img/sharp-libvips-darwin-x64" "1.0.4" - "@img/sharp-darwin-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz#19bc1dd6eba6d5a96283498b9c9f401180ee9c7b" @@ -2968,41 +2947,21 @@ optionalDependencies: "@img/sharp-libvips-darwin-x64" "1.2.4" -"@img/sharp-libvips-darwin-arm64@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz#447c5026700c01a993c7804eb8af5f6e9868c07f" - integrity sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg== - "@img/sharp-libvips-darwin-arm64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz#2894c0cb87d42276c3889942e8e2db517a492c43" integrity sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g== -"@img/sharp-libvips-darwin-x64@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz#e0456f8f7c623f9dbfbdc77383caa72281d86062" - integrity sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ== - "@img/sharp-libvips-darwin-x64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz#e63681f4539a94af9cd17246ed8881734386f8cc" integrity sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg== -"@img/sharp-libvips-linux-arm64@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz#979b1c66c9a91f7ff2893556ef267f90ebe51704" - integrity sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA== - "@img/sharp-libvips-linux-arm64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz#b1b288b36864b3bce545ad91fa6dadcf1a4ad318" integrity sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw== -"@img/sharp-libvips-linux-arm@1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz#99f922d4e15216ec205dcb6891b721bfd2884197" - integrity sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g== - "@img/sharp-libvips-linux-arm@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz#b9260dd1ebe6f9e3bdbcbdcac9d2ac125f35852d" @@ -3018,53 +2977,26 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz#880b4678009e5a2080af192332b00b0aaf8a48de" integrity sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA== -"@img/sharp-libvips-linux-s390x@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz#f8a5eb1f374a082f72b3f45e2fb25b8118a8a5ce" - integrity sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA== - "@img/sharp-libvips-linux-s390x@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz#74f343c8e10fad821b38f75ced30488939dc59ec" integrity sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ== -"@img/sharp-libvips-linux-x64@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz#d4c4619cdd157774906e15770ee119931c7ef5e0" - integrity sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw== - "@img/sharp-libvips-linux-x64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz#df4183e8bd8410f7d61b66859a35edeab0a531ce" integrity sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw== -"@img/sharp-libvips-linuxmusl-arm64@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz#166778da0f48dd2bded1fa3033cee6b588f0d5d5" - integrity sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA== - "@img/sharp-libvips-linuxmusl-arm64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz#c8d6b48211df67137541007ee8d1b7b1f8ca8e06" integrity sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw== -"@img/sharp-libvips-linuxmusl-x64@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz#93794e4d7720b077fcad3e02982f2f1c246751ff" - integrity sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw== - "@img/sharp-libvips-linuxmusl-x64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz#be11c75bee5b080cbee31a153a8779448f919f75" integrity sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg== -"@img/sharp-linux-arm64@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz#edb0697e7a8279c9fc829a60fc35644c4839bb22" - integrity sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA== - optionalDependencies: - "@img/sharp-libvips-linux-arm64" "1.0.4" - "@img/sharp-linux-arm64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz#7aa7764ef9c001f15e610546d42fce56911790cc" @@ -3072,13 +3004,6 @@ optionalDependencies: "@img/sharp-libvips-linux-arm64" "1.2.4" -"@img/sharp-linux-arm@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz#422c1a352e7b5832842577dc51602bcd5b6f5eff" - integrity sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ== - optionalDependencies: - "@img/sharp-libvips-linux-arm" "1.0.5" - "@img/sharp-linux-arm@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz#5fb0c3695dd12522d39c3ff7a6bc816461780a0d" @@ -3100,13 +3025,6 @@ optionalDependencies: "@img/sharp-libvips-linux-riscv64" "1.2.4" -"@img/sharp-linux-s390x@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz#f5c077926b48e97e4a04d004dfaf175972059667" - integrity sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q== - optionalDependencies: - "@img/sharp-libvips-linux-s390x" "1.0.4" - "@img/sharp-linux-s390x@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz#93eac601b9f329bb27917e0e19098c722d630df7" @@ -3114,13 +3032,6 @@ optionalDependencies: "@img/sharp-libvips-linux-s390x" "1.2.4" -"@img/sharp-linux-x64@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz#d806e0afd71ae6775cc87f0da8f2d03a7c2209cb" - integrity sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA== - optionalDependencies: - "@img/sharp-libvips-linux-x64" "1.0.4" - "@img/sharp-linux-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz#55abc7cd754ffca5002b6c2b719abdfc846819a8" @@ -3128,13 +3039,6 @@ optionalDependencies: "@img/sharp-libvips-linux-x64" "1.2.4" -"@img/sharp-linuxmusl-arm64@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz#252975b915894fb315af5deea174651e208d3d6b" - integrity sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g== - optionalDependencies: - "@img/sharp-libvips-linuxmusl-arm64" "1.0.4" - "@img/sharp-linuxmusl-arm64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz#d6515ee971bb62f73001a4829b9d865a11b77086" @@ -3142,13 +3046,6 @@ optionalDependencies: "@img/sharp-libvips-linuxmusl-arm64" "1.2.4" -"@img/sharp-linuxmusl-x64@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz#3f4609ac5d8ef8ec7dadee80b560961a60fd4f48" - integrity sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw== - optionalDependencies: - "@img/sharp-libvips-linuxmusl-x64" "1.0.4" - "@img/sharp-linuxmusl-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz#d97978aec7c5212f999714f2f5b736457e12ee9f" @@ -3156,13 +3053,6 @@ optionalDependencies: "@img/sharp-libvips-linuxmusl-x64" "1.2.4" -"@img/sharp-wasm32@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz#6f44f3283069d935bb5ca5813153572f3e6f61a1" - integrity sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg== - dependencies: - "@emnapi/runtime" "^1.2.0" - "@img/sharp-wasm32@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz#2f15803aa626f8c59dd7c9d0bbc766f1ab52cfa0" @@ -3175,21 +3065,11 @@ resolved "https://registry.yarnpkg.com/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz#3706e9e3ac35fddfc1c87f94e849f1b75307ce0a" integrity sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g== -"@img/sharp-win32-ia32@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz#1a0c839a40c5351e9885628c85f2e5dfd02b52a9" - integrity sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ== - "@img/sharp-win32-ia32@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz#0b71166599b049e032f085fb9263e02f4e4788de" integrity sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg== -"@img/sharp-win32-x64@0.33.5": - version "0.33.5" - resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz#56f00962ff0c4e0eb93d34a047d29fa995e3e342" - integrity sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg== - "@img/sharp-win32-x64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz#a81ffb00e69267cd0a1d626eaedb8a8430b2b2f8" @@ -6212,27 +6092,11 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" -color-name@^1.0.0, color-name@~1.1.4: +color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -color-string@^1.9.0: - version "1.9.1" - resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4" - integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg== - dependencies: - color-name "^1.0.0" - simple-swizzle "^0.2.2" - -color@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/color/-/color-4.2.3.tgz#d781ecb5e57224ee43ea9627560107c0e0c6463a" - integrity sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A== - dependencies: - color-convert "^2.0.1" - color-string "^1.9.0" - comma-separated-tokens@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" @@ -6468,7 +6332,7 @@ destr@^2.0.5: resolved "https://registry.yarnpkg.com/destr/-/destr-2.0.5.tgz#7d112ff1b925fb8d2079fac5bdb4a90973b51fdb" integrity sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA== -detect-libc@^2.0.3, detect-libc@^2.1.2: +detect-libc@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.2.tgz#689c5dcdc1900ef5583a4cb9f6d7b473742074ad" integrity sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ== @@ -7901,11 +7765,6 @@ is-arrayish@^0.2.1: resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== -is-arrayish@^0.3.1: - version "0.3.4" - resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.4.tgz#1ee5553818511915685d33bb13d31bf854e5059d" - integrity sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA== - is-async-function@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/is-async-function/-/is-async-function-2.1.1.tgz#3e69018c8e04e73b738793d020bfe884b9fd3523" @@ -10445,11 +10304,6 @@ semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.2, semver@^7.7.1, semve resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.4.tgz#28464e36060e991fa7a11d0279d2d3f3b57a7e8a" integrity sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA== -semver@^7.6.3: - version "7.8.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.8.0.tgz#ed0661039fcbcda2ce71f01fa6adbefaa77040df" - integrity sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA== - set-function-length@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" @@ -10481,35 +10335,6 @@ set-proto@^1.0.0: es-errors "^1.3.0" es-object-atoms "^1.0.0" -sharp@^0.33.5: - version "0.33.5" - resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.33.5.tgz#13e0e4130cc309d6a9497596715240b2ec0c594e" - integrity sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw== - dependencies: - color "^4.2.3" - detect-libc "^2.0.3" - semver "^7.6.3" - optionalDependencies: - "@img/sharp-darwin-arm64" "0.33.5" - "@img/sharp-darwin-x64" "0.33.5" - "@img/sharp-libvips-darwin-arm64" "1.0.4" - "@img/sharp-libvips-darwin-x64" "1.0.4" - "@img/sharp-libvips-linux-arm" "1.0.5" - "@img/sharp-libvips-linux-arm64" "1.0.4" - "@img/sharp-libvips-linux-s390x" "1.0.4" - "@img/sharp-libvips-linux-x64" "1.0.4" - "@img/sharp-libvips-linuxmusl-arm64" "1.0.4" - "@img/sharp-libvips-linuxmusl-x64" "1.0.4" - "@img/sharp-linux-arm" "0.33.5" - "@img/sharp-linux-arm64" "0.33.5" - "@img/sharp-linux-s390x" "0.33.5" - "@img/sharp-linux-x64" "0.33.5" - "@img/sharp-linuxmusl-arm64" "0.33.5" - "@img/sharp-linuxmusl-x64" "0.33.5" - "@img/sharp-wasm32" "0.33.5" - "@img/sharp-win32-ia32" "0.33.5" - "@img/sharp-win32-x64" "0.33.5" - sharp@^0.34.0: version "0.34.5" resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.34.5.tgz#b6f148e4b8c61f1797bde11a9d1cfebbae2c57b0" @@ -10634,13 +10459,6 @@ signal-exit@^4.0.1: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== -simple-swizzle@^0.2.2: - version "0.2.4" - resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.4.tgz#a8d11a45a11600d6a1ecdff6363329e3648c3667" - integrity sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw== - dependencies: - is-arrayish "^0.3.1" - sisteransi@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" From 572160019aa2cdb70de20edd5b8d6c33eaa00498 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 00:06:16 -0500 Subject: [PATCH 12/19] chore(validation): update validation for url --- agent/tests/test_attachments.py | 10 +- .../shared/resolve-url-attachments.ts | 12 +- cdk/src/handlers/shared/validation.ts | 35 +- .../docs/architecture/Change-manifest.md | 874 ++++++++++++++++++ 4 files changed, 917 insertions(+), 14 deletions(-) create mode 100644 docs/src/content/docs/architecture/Change-manifest.md diff --git a/agent/tests/test_attachments.py b/agent/tests/test_attachments.py index c779c478..456d3aba 100644 --- a/agent/tests/test_attachments.py +++ b/agent/tests/test_attachments.py @@ -54,7 +54,7 @@ def test_rejects_extra_fields(self): filename="test.txt", local_path="/tmp/test.txt", size_bytes=100, - extra_field="bad", + extra_field="bad", # ty: ignore[unknown-argument] ) @@ -102,9 +102,7 @@ def test_checksum_mismatch_raises(self, mock_client, tmp_path): mock_s3 = MagicMock() mock_client.return_value = mock_s3 - mock_s3.get_object.return_value = { - "Body": MagicMock(read=lambda: tampered_content) - } + mock_s3.get_object.return_value = {"Body": MagicMock(read=lambda: tampered_content)} with pytest.raises(RuntimeError, match="integrity check failed"): download_attachments([config], str(tmp_path)) @@ -177,9 +175,7 @@ def test_multiple_attachments_all_verified(self, mock_client, tmp_path): {"Body": MagicMock(read=lambda: configs_and_contents[1][1])}, ] - result = download_attachments( - [c[0] for c in configs_and_contents], str(tmp_path) - ) + result = download_attachments([c[0] for c in configs_and_contents], str(tmp_path)) assert len(result) == 2 assert result[0].filename == "file1.txt" assert result[1].filename == "file2.txt" diff --git a/cdk/src/handlers/shared/resolve-url-attachments.ts b/cdk/src/handlers/shared/resolve-url-attachments.ts index 669b9531..e194e6bf 100644 --- a/cdk/src/handlers/shared/resolve-url-attachments.ts +++ b/cdk/src/handlers/shared/resolve-url-attachments.ts @@ -452,9 +452,13 @@ export async function resolveUrlAttachments( content_type: finalContentType, }); - // Determine if this is an image or file based on content type - const isImage = finalContentType.startsWith('image/'); - const resolvedContentType = att.content_type || finalContentType; + // Determine actual content type from the HTTP response (preferred over any placeholder + // set during validation). If the user explicitly declared a content_type on the URL + // attachment, use it; otherwise use what the server returned. + const resolvedContentType = (att.content_type && att.content_type !== 'application/octet-stream') + ? att.content_type + : finalContentType; + const isImage = resolvedContentType.startsWith('image/'); // Validate content-type is in the allowlist (attacker controls the response header) const attachmentType = isImage ? 'image' : 'file'; @@ -533,7 +537,7 @@ export async function resolveUrlAttachments( resolved.set(att.attachment_id, createAttachmentRecord({ attachment_id: att.attachment_id, - type: att.type, + type: isImage ? 'image' : 'file', content_type: resolvedContentType, filename: att.filename, s3_key: s3Key, diff --git a/cdk/src/handlers/shared/validation.ts b/cdk/src/handlers/shared/validation.ts index 53e6320c..c1a24584 100644 --- a/cdk/src/handlers/shared/validation.ts +++ b/cdk/src/handlers/shared/validation.ts @@ -384,6 +384,30 @@ function generateFilename(type: string, contentType: string, index: number): str return `attachment_${index}.${ext}`; } +/** Extract a safe filename from a URL path, falling back to a generated name. */ +function filenameFromUrl(url: string, index: number): string { + try { + const parsed = new URL(url); + const lastSegment = parsed.pathname.split('/').filter(Boolean).pop(); + if (lastSegment && lastSegment.includes('.') && lastSegment.length <= 255) { + // Decode percent-encoding (e.g., %20 → space) then sanitize + let decoded: string; + try { + decoded = decodeURIComponent(lastSegment); + } catch { + decoded = lastSegment; + } + const sanitized = decoded.replace(/[^a-zA-Z0-9._\-]/g, '_'); + if (isValidFilename(sanitized)) { + return sanitized; + } + } + } catch { + // URL parse failure — fall through to default + } + return `url_attachment_${index}`; +} + const MIME_TO_EXTENSION: Record = { 'image/png': 'png', 'image/jpeg': 'jpg', @@ -498,9 +522,14 @@ export function validateAttachments( } // Filename resolution - const resolvedFilename = (att.filename && typeof att.filename === 'string') - ? att.filename - : generateFilename(attType, resolvedContentType, i); + let resolvedFilename: string; + if (att.filename && typeof att.filename === 'string') { + resolvedFilename = att.filename; + } else if (attType === 'url' && att.url && typeof att.url === 'string') { + resolvedFilename = filenameFromUrl(att.url as string, i); + } else { + resolvedFilename = generateFilename(attType, resolvedContentType, i); + } if (!isValidFilename(resolvedFilename)) { return { valid: false, error: `attachments[${i}]: invalid filename` }; } diff --git a/docs/src/content/docs/architecture/Change-manifest.md b/docs/src/content/docs/architecture/Change-manifest.md new file mode 100644 index 00000000..5a3e901e --- /dev/null +++ b/docs/src/content/docs/architecture/Change-manifest.md @@ -0,0 +1,874 @@ +--- +title: Change manifest +--- + +# Change Manifest + +A structured intermediate representation between task intent and code execution. The agent produces a verifiable plan (the **Change Manifest**) before writing any code. The manifest is validated through multiple verification layers, then constrains execution so drift is detected early rather than discovered at PR review time. + +- **Use this doc for:** understanding the planning phase, manifest schema, verification pipeline, drift enforcement, and reconciliation. +- **Related docs:** [ORCHESTRATOR.md](/architecture/orchestrator) for the task lifecycle this extends, [SECURITY.md](/architecture/security) for Cedar policy enforcement, [COMPUTE.md](/architecture/compute) for the agent runtime, [REPO_ONBOARDING.md](/architecture/repo-onboarding) for per-repo configuration. + +## Motivation + +Today the agent receives a task and immediately begins writing code. The platform has no visibility into the agent's intent until after execution completes. This creates three problems: + +1. **Late failure detection** — A scope creep or wrong approach is only caught at PR review, after 10-30 minutes of compute. +2. **Opaque progress** — `bgagent watch` shows tool calls but not meaningful progress toward the goal. +3. **Coarse-grained governance** — Cedar policies govern tool-level access (can this agent use `Write`?) but not intent-level actions (should this agent modify the CI pipeline for a "fix typo" task?). + +The Change Manifest solves all three by inserting a verifiable planning phase between task receipt and code execution. + +## Design principle + +**Verify intent before permitting execution.** The manifest is a structured claim about what the agent intends to do. Verification checks whether that intent is consistent, proportionate, and authorized. Execution is then constrained to the verified intent, with drift detection enforcing adherence. + +The manifest is a guardrail, not a straitjacket. The agent can amend the manifest mid-execution when it discovers something unexpected, but amendments trigger re-verification. + +## Pipeline overview + +```mermaid +stateDiagram-v2 + [*] --> SUBMITTED + SUBMITTED --> HYDRATING : Admission passes + HYDRATING --> PLANNING : Context assembled + PLANNING --> PLAN_VERIFIED : All checks pass + PLANNING --> PLAN_REJECTED : Hard failure + PLANNING --> AWAITING_APPROVAL : Requires human decision + AWAITING_APPROVAL --> PLAN_VERIFIED : Approved + AWAITING_APPROVAL --> PLAN_REJECTED : Denied + PLAN_VERIFIED --> RUNNING : Execution begins + PLAN_REJECTED --> FAILED : Terminal + RUNNING --> FINALIZING : Session ends + FINALIZING --> COMPLETED : Reconciliation passes + FINALIZING --> FAILED : Reconciliation fails +``` + +### New states + +| State | Description | Duration | +|---|---|---| +| `PLANNING` | Agent has produced a manifest; verification is running | Seconds | +| `PLAN_VERIFIED` | Manifest passed all verification layers | Transient (immediate transition to RUNNING) | +| `PLAN_REJECTED` | Manifest failed verification; task cannot proceed | Terminal | +| `AWAITING_APPROVAL` | Manifest requires human approval before execution | Minutes to hours | + +## Change Manifest schema + +The manifest is a JSON document produced by the agent using structured output (tool_use response with a JSON schema). JSON is chosen over YAML because: + +- LLMs produce valid JSON reliably via constrained decoding / structured output modes +- JSON has no implicit type coercion, no indentation sensitivity, no anchors — fewer parsing surprises +- Schema validation is native (JSON Schema), well-tooled, and battle-tested +- The manifest is machine-produced and machine-consumed; human readability is secondary + +The agent emits the manifest as a tool call response to a `produce_change_manifest` tool, ensuring schema conformance at generation time rather than requiring post-hoc parsing and error recovery. + +```json +{ + "version": 1, + "task_id": "01J5X7...", + "task_type": "new_task", + "intent": { + "summary": "Add rate limiting to the /tasks endpoint", + "category": "feature" + }, + "scope": { + "files_to_create": [ + { + "path": "cdk/src/constructs/rate-limiter.ts", + "purpose": "L2 construct wrapping WAF rate-limit rule" + }, + { + "path": "cdk/test/constructs/rate-limiter.test.ts", + "purpose": "Unit tests for rate limiter construct" + } + ], + "files_to_modify": [ + { + "path": "cdk/src/stacks/api-stack.ts", + "reason": "Attach rate limiter to API Gateway stage", + "sections": ["TaskApi construct instantiation"] + } + ], + "files_to_delete": [] + }, + "approach": { + "strategy": "WAF-based rate limiting per Cognito user sub", + "alternatives_considered": [ + { + "description": "API Gateway throttling", + "rejected_because": "Per-stage not per-user" + } + ], + "risks": ["WAF rule count limit (current: 4/10)"], + "dependencies": [ + { + "type": "npm_package", + "name": "@aws-cdk/aws-wafv2", + "action": "add" + } + ] + }, + "knowledge_requirements": [ + { + "domain": "aws-wafv2", + "description": "WAF v2 CDK constructs: WebACL, RateBasedStatement, association to API Gateway", + "sources": ["@aws-cdk/aws-wafv2 API reference"], + "tool_needed": "awsdocs" + }, + { + "domain": "cdk-patterns", + "description": "L2 construct authoring conventions in this repo", + "sources": ["cdk/src/constructs/task-orchestrator.ts"], + "tool_needed": null + } + ], + "assertions": { + "max_files_changed": 5, + "no_changes_to": ["agent/", "cli/", ".github/"], + "must_pass": ["mise //cdk:test", "mise //cdk:synth"] + }, + "verification": null +} +``` + +### Schema constraints + +| Field | Required | Validation | +|---|---|---| +| `version` | Yes | Must be `1` | +| `task_id` | Yes | Must match the running task | +| `intent.summary` | Yes | 10-200 characters | +| `intent.category` | Yes | Enum: feature, bugfix, refactor, docs, test, chore | +| `scope.files_to_create` | Yes (may be empty) | Each entry needs `path` and `purpose` | +| `scope.files_to_modify` | Yes (may be empty) | Each entry needs `path` and `reason` | +| `scope.files_to_delete` | Yes (may be empty) | Each entry needs `path` and `reason` | +| `approach.strategy` | Yes | Non-empty string | +| `knowledge_requirements` | Yes (may be empty) | Each entry needs `domain` and `description` | +| `assertions.max_files_changed` | Yes | Integer, >= total declared files | +| `assertions.must_pass` | Yes | At least one command | + +## Verification pipeline + +The manifest passes through five verification layers in sequence. Each layer can produce one of three outcomes: **PASS**, **FAIL** (hard rejection), or **REQUIRE_APPROVAL** (pause for human decision). If any layer fails, subsequent layers are skipped and the task transitions to `PLAN_REJECTED`. + +```mermaid +flowchart TD + M[Change Manifest] --> L1[L1: Schema & Structural Validation] + L1 -->|PASS| L2[L2: Scope & Proportionality Analysis] + L2 -->|PASS| L3[L3: Cedar Intent-Level Authorization] + L3 -->|PASS| L4[L4: Semantic Consistency Checks] + L4 -->|PASS| L5[L5: Knowledge Resolution] + L5 -->|PASS| V[PLAN_VERIFIED] + + L1 -->|FAIL| R[PLAN_REJECTED] + L2 -->|FAIL| R + L3 -->|FAIL| R + L4 -->|FAIL| R + L5 -->|FAIL| R + + L2 -->|REQUIRE_APPROVAL| A[AWAITING_APPROVAL] + L3 -->|REQUIRE_APPROVAL| A + A -->|Approved| V + A -->|Denied| R +``` + +### L1: Schema and structural validation + +**Purpose:** Ensure the manifest is well-formed and internally consistent. + +**Duration:** <100ms (in-process, no I/O) + +**Checks:** + +| Check | Failure condition | Outcome | +|---|---|---| +| JSON parsing | Invalid JSON (should not happen with structured output, but fail-safe) | FAIL | +| Schema conformance | Missing required fields, wrong types | FAIL | +| Task ID match | `manifest.task_id` != running task ID | FAIL | +| Path syntax | Paths contain `..`, absolute paths, or null bytes | FAIL | +| Self-consistency | `max_files_changed` < total declared files | FAIL | +| Assertion validity | `must_pass` contains unknown or disallowed commands | FAIL | +| Category coherence | `task_type=pr_review` but `category=feature` | FAIL | +| Knowledge source validity | `tool_needed` references a tool not in the known tool registry | FAIL | + +**Implementation:** Pure function, no external dependencies. JSON Schema validation plus custom rules. Runs in the agent container before any network calls. + +### L2: Scope and proportionality analysis + +**Purpose:** Detect scope creep, disproportionate changes, and out-of-bounds modifications relative to the task description and blueprint configuration. + +**Duration:** <500ms (reads blueprint config from memory, no external calls) + +**Checks:** + +| Check | Logic | Outcome | +|---|---|---| +| Allowed paths | All declared paths match blueprint's `allowed_paths` patterns | FAIL if outside boundary | +| Protected paths | Any declared path matches blueprint's `protected_paths` | REQUIRE_APPROVAL | +| Proportionality | File count vs. task complexity heuristic (see below) | REQUIRE_APPROVAL if disproportionate | +| Scope containment | Files outside the task's primary domain (e.g., task says "fix API bug" but manifest modifies `agent/`) | REQUIRE_APPROVAL | +| Deletion guard | Any `files_to_delete` entry | REQUIRE_APPROVAL (deletions are high-risk) | +| Dependency changes | `approach.dependencies` includes additions or removals | REQUIRE_APPROVAL | + +**Proportionality heuristic:** + +The proportionality check prevents a "fix typo" task from declaring 20 file modifications. The heuristic uses a lookup based on task category and description length as a proxy for complexity: + +| Category | Baseline file budget | Multiplier for long descriptions (>500 chars) | +|---|---|---| +| docs | 3 | 2x | +| bugfix | 5 | 1.5x | +| chore | 5 | 1.5x | +| test | 8 | 2x | +| refactor | 10 | 2x | +| feature | 15 | 2x | + +If the manifest declares more files than the budget, the check returns REQUIRE_APPROVAL (not FAIL — the agent might be right, but a human should confirm). + +**Blueprint configuration:** + +```json +{ + "change_manifest": { + "allowed_paths": ["cdk/src/**", "cdk/test/**", "agent/src/**", "agent/tests/**"], + "protected_paths": [".github/**", "**/package.json", "cdk/cdk.json", "Dockerfile"], + "auto_approve_categories": ["docs", "test"] + } +} +``` + +### L3: Cedar intent-level authorization + +**Purpose:** Evaluate high-level intent actions against Cedar policies. This elevates governance from "can the agent use the Write tool?" to "should the agent modify CI configuration for this task type?" + +**Duration:** <50ms (in-process Cedar evaluation via cedarpy) + +**New Cedar actions (intent-level):** + +| Action | Triggered when | Resource | +|---|---|---| +| `Agent::Action::"create_file"` | Manifest declares `files_to_create` | `Agent::FileDomain::""` | +| `Agent::Action::"modify_file"` | Manifest declares `files_to_modify` | `Agent::FileDomain::""` | +| `Agent::Action::"delete_file"` | Manifest declares `files_to_delete` | `Agent::FileDomain::""` | +| `Agent::Action::"add_dependency"` | `approach.dependencies` includes additions | `Agent::Dependency::""` | +| `Agent::Action::"remove_dependency"` | `approach.dependencies` includes removals | `Agent::Dependency::""` | +| `Agent::Action::"modify_ci"` | Any path matches `.github/**` or CI-related patterns | `Agent::CIDomain::"ci"` | + +**Domain derivation:** File paths are mapped to domains using a simple prefix table: + +| Path prefix | Domain | +|---|---| +| `cdk/src/` | `cdk_source` | +| `cdk/test/` | `cdk_test` | +| `agent/src/` | `agent_source` | +| `agent/tests/` | `agent_test` | +| `cli/src/` | `cli_source` | +| `.github/` | `ci` | +| `docs/` | `docs` | +| Other | `other` | + +**Example Cedar policies (intent-level):** + +```cedar +// pr_review tasks cannot create or modify source files +forbid ( + principal == Agent::TaskAgent::"pr_review", + action == Agent::Action::"create_file", + resource == Agent::FileDomain::"cdk_source" +); +forbid ( + principal == Agent::TaskAgent::"pr_review", + action == Agent::Action::"modify_file", + resource == Agent::FileDomain::"cdk_source" +); + +// No task type can modify CI without approval +forbid ( + principal, + action == Agent::Action::"modify_ci", + resource +); + +// Only new_task can add dependencies +forbid ( + principal == Agent::TaskAgent::"pr_iteration", + action == Agent::Action::"add_dependency", + resource +); +``` + +**Three-outcome model:** This layer uses Phase 3's `REQUIRE_APPROVAL` outcome (see `Phase3-cedar-hitl.md`). The Cedar evaluation returns: + +- **Allow** (no applicable forbid) → PASS +- **Deny** (hard forbid, no `@advice("require_approval")`) → FAIL +- **Deny with `@advice("require_approval")` annotation** → REQUIRE_APPROVAL + +### L4: Semantic consistency checks + +**Purpose:** Verify that the manifest makes sense given the current repository state. Unlike L1-L3 (which are purely structural or policy-based), L4 inspects the actual filesystem. + +**Duration:** <2s (reads files, runs lightweight checks) + +**Checks:** + +| Check | Logic | Outcome | +|---|---|---| +| Existing file check | `files_to_create` paths must NOT already exist | FAIL (would overwrite) | +| Modify target exists | `files_to_modify` paths MUST exist | FAIL (can't modify what's not there) | +| Delete target exists | `files_to_delete` paths MUST exist | FAIL (can't delete what's not there) | +| Import graph impact | Modified files are imported by N other files; if N > threshold (default 10), flag | REQUIRE_APPROVAL | +| Test coverage | Every `files_to_create` in a source dir has a corresponding test entry | WARN (non-blocking, logged) | +| Circular scope | Manifest modifies a file that is part of the verification pipeline itself | FAIL | + +**Implementation:** Runs inside the agent container after the repo is cloned. Uses `glob` and basic AST-level import tracing (language-specific; TypeScript first via regex, extensible). + +### L5: Knowledge resolution + +**Purpose:** Detect what domain knowledge the agent will need during execution and verify that appropriate knowledge sources are available. This prevents the agent from hallucinating APIs, using deprecated patterns, or spending turns groping in the dark for documentation that could have been provided upfront. + +**Duration:** <3s (checks tool availability, fetches documentation indexes, validates sources exist) + +**Inputs:** The `knowledge_requirements` array from the manifest. Each entry declares: + +| Field | Description | +|---|---| +| `domain` | Short identifier for the knowledge area (e.g., `aws-wafv2`, `jest-mocks`, `cdk-nag`) | +| `description` | What specifically the agent needs to know (e.g., "WAF v2 RateBasedStatement configuration") | +| `sources` | Where this knowledge lives — can be documentation URLs, local file paths, or named references | +| `tool_needed` | Which documentation tool the agent requires access to (e.g., `awsdocs`, `npm-docs`, `mcp-server-name`), or `null` if the source is local | + +**Checks:** + +| Check | Logic | Outcome | +|---|---|---| +| Tool availability | Every non-null `tool_needed` must be in the agent's configured tool set | FAIL | +| Source reachability | Local file `sources` must exist in the cloned repo | FAIL | +| Documentation index match | External `sources` must resolve to known documentation packages or MCP server endpoints | WARN (non-blocking) | +| Knowledge gap detection | Cross-reference declared dependencies against knowledge requirements — a new dependency without a corresponding knowledge requirement is suspicious | WARN (non-blocking) | +| Redundant tool check | If `tool_needed` is declared but the agent already has the relevant knowledge embedded via repo examples (existing usage of the same library in-tree) | INFO (logged, may skip tool injection) | + +**Resolution actions:** + +When a knowledge requirement is validated, the verification layer produces a **tool provisioning plan** that the execution phase uses to configure the agent's environment: + +```json +{ + "knowledge_resolution": { + "resolved": [ + { + "domain": "aws-wafv2", + "tool": "awsdocs", + "action": "inject", + "config": { + "scope": "@aws-cdk/aws-wafv2", + "version": "latest" + } + }, + { + "domain": "cdk-patterns", + "tool": null, + "action": "context_inject", + "config": { + "files": ["cdk/src/constructs/task-orchestrator.ts"], + "inject_as": "reference_example" + } + } + ], + "unresolved": [], + "warnings": [] + } +} +``` + +**Tool provisioning categories:** + +| Action | Effect | +|---|---| +| `inject` | Enable an MCP server or documentation tool for this session | +| `context_inject` | Pre-load specific files or documentation snippets into the agent's context at execution start | +| `fetch_and_cache` | Download documentation to the sandbox filesystem before execution begins | +| `skip` | Knowledge is already available (in-tree examples found); no action needed | + +**How this prevents hallucination:** + +Without knowledge resolution, a typical failure mode is: + +1. Agent needs to use `@aws-cdk/aws-wafv2` +2. Agent's training data has outdated API signatures +3. Agent writes code using non-existent constructs +4. Build fails after 5 minutes +5. Agent tries 3 more variations, all wrong +6. Task fails after 15 minutes of wasted compute + +With knowledge resolution: + +1. Agent declares `knowledge_requirements: [{domain: "aws-wafv2", tool_needed: "awsdocs"}]` +2. L5 verifies `awsdocs` tool is available and the package exists +3. Execution starts with `awsdocs` MCP server configured and scoped to `@aws-cdk/aws-wafv2` +4. Agent queries actual API docs before writing code +5. First attempt uses correct construct names and properties + +**Blueprint configuration for knowledge tools:** + +```json +{ + "change_manifest": { + "knowledge_tools": { + "available": ["awsdocs", "npm-docs", "gh-docs"], + "mcp_servers": { + "awsdocs": { + "command": "npx", + "args": ["-y", "@anthropic/awsdocs-mcp-server"], + "scoped": true + }, + "npm-docs": { + "command": "npx", + "args": ["-y", "@anthropic/npm-docs-mcp-server"], + "scoped": true + } + }, + "max_tools_per_task": 3, + "auto_detect_from_dependencies": true + } + } +} +``` + +**Auto-detection:** When `auto_detect_from_dependencies` is enabled, the verification layer cross-references declared `approach.dependencies` against the tool registry. If an agent adds a dependency but does not declare a knowledge requirement for it, L5 emits a warning and optionally auto-injects the relevant documentation tool (configurable: `warn` | `auto_inject` | `ignore`). + +**Interaction with Cedar policies:** + +Tool injection from knowledge resolution is subject to Cedar authorization. A new action is evaluated: + +```cedar +// Allow documentation tools to be injected for new_task +permit ( + principal == Agent::TaskAgent::"new_task", + action == Agent::Action::"inject_knowledge_tool", + resource +); + +// pr_review can only use read-only documentation tools +permit ( + principal == Agent::TaskAgent::"pr_review", + action == Agent::Action::"inject_knowledge_tool", + resource == Agent::KnowledgeTool::"awsdocs" +); +forbid ( + principal == Agent::TaskAgent::"pr_review", + action == Agent::Action::"inject_knowledge_tool", + resource == Agent::KnowledgeTool::"npm-docs" +); +``` + +## Verification report + +After all five layers execute, a structured verification report is attached to the manifest and persisted to the TaskEvents table: + +```json +{ + "verification": { + "status": "VERIFIED", + "timestamp": "2026-05-09T14:32:01Z", + "duration_ms": 1247, + "layers": [ + { + "name": "schema_validation", + "status": "PASS", + "duration_ms": 12, + "checks_run": 8, + "checks_passed": 8 + }, + { + "name": "scope_proportionality", + "status": "PASS", + "duration_ms": 203, + "checks_run": 6, + "checks_passed": 6, + "notes": ["File budget: 5/15 (feature category)"] + }, + { + "name": "cedar_intent_authorization", + "status": "PASS", + "duration_ms": 38, + "checks_run": 4, + "checks_passed": 4 + }, + { + "name": "semantic_consistency", + "status": "PASS", + "duration_ms": 594, + "checks_run": 6, + "checks_passed": 5, + "warnings": [ + "cdk/src/constructs/rate-limiter.ts has no corresponding test in manifest (non-blocking)" + ] + }, + { + "name": "knowledge_resolution", + "status": "PASS", + "duration_ms": 400, + "checks_run": 3, + "checks_passed": 3, + "tools_injected": ["awsdocs"], + "context_files_loaded": ["cdk/src/constructs/task-orchestrator.ts"], + "notes": ["awsdocs scoped to @aws-cdk/aws-wafv2"] + } + ], + "approved_scope": { + "files_hash": "sha256:abc123...", + "assertions_hash": "sha256:def456..." + } + } +} +``` + +The `files_hash` and `assertions_hash` are used during execution to detect manifest tampering — if the agent somehow modifies the manifest without going through the amendment flow, the hashes won't match and execution is halted. + +## Execution constraint: drift detection + +Once the manifest is verified, it constrains execution. The PreToolUse hook gains a manifest-aware layer that runs after Cedar tool-level checks pass: + +### Drift detection rules + +| Agent action | Manifest check | On violation | +|---|---|---| +| `Write` to a new file | Path must be in `files_to_create` | Block + prompt agent to amend manifest | +| `Edit` an existing file | Path must be in `files_to_modify` | Block + prompt agent to amend manifest | +| `Bash` that deletes a file | Path must be in `files_to_delete` | Block | +| `Write`/`Edit` any file | Total unique files touched <= `max_files_changed` | Block | +| `Write`/`Edit` any file | Path must not match `no_changes_to` patterns | Block (hard, no amendment) | +| `Bash` that runs tests | Allowed regardless (read-only observation) | Allow always | +| `Read`/`Glob`/`Grep` | No restriction (reading is non-destructive) | Allow always | + +### Manifest amendment flow + +When the agent discovers that its plan was incomplete (e.g., a file needs modification that wasn't in the original manifest), it can amend the manifest: + +1. Agent produces an updated manifest with new entries +2. The delta (new entries only) goes through L1-L5 verification +3. If verified, the execution constraint updates with the expanded scope +4. If rejected or requires approval, the agent must find an alternative approach or the task fails + +Amendments are tracked in the verification report so reviewers can see what changed during execution: + +```json +{ + "amendments": [ + { + "timestamp": "2026-05-09T14:45:12Z", + "added": { + "files_to_modify": [ + { + "path": "cdk/src/constructs/api-gateway.ts", + "reason": "Rate limiter requires WAF association on the gateway construct" + } + ] + }, + "verification": { + "status": "VERIFIED", + "duration_ms": 312 + } + } + ] +} +``` + +**Amendment budget:** The blueprint can limit how many amendments are allowed per task (default: 3). Excessive amendments suggest the planning phase produced a poor plan, and the task should fail with a clear signal for prompt improvement. + +## Reconciliation + +After execution completes, reconciliation compares the actual diff against the manifest. This produces a structured report that becomes part of the PR description and is stored for learning. + +### Reconciliation checks + +| Check | Logic | Outcome | +|---|---|---| +| Completeness | Every `files_to_create` was created | Partial completion warning | +| Completeness | Every `files_to_modify` was modified | Partial completion warning | +| Scope adherence | No files outside manifest were touched | Violation (should not happen if drift detection works) | +| Assertion checks | All `must_pass` commands succeed | FAIL if any fails | +| Unused declarations | Manifest declared files that were never touched | Note (over-planning) | + +### Reconciliation report + +```json +{ + "reconciliation": { + "plan_adherence_score": 92, + "manifest_items": 5, + "items_completed": 4, + "items_skipped": 1, + "scope_violations": 0, + "amendments_used": 1, + "knowledge_tools_used": ["awsdocs"], + "knowledge_queries": 4, + "assertions": [ + { "command": "mise //cdk:test", "status": "PASS", "duration_ms": 12400 }, + { "command": "mise //cdk:synth", "status": "PASS", "duration_ms": 8200 } + ], + "skipped_items": [ + { + "path": "cdk/test/constructs/rate-limiter.test.ts", + "declared_as": "files_to_create", + "reason": "Agent determined existing test file covered the construct" + } + ] + } +} +``` + +The `plan_adherence_score` is computed as: `(items_completed / manifest_items) * 100`, penalized by `-10` per scope violation and `-5` per unused amendment. + +## Human approval flow + +When verification returns `REQUIRE_APPROVAL`, the task pauses and waits for a human decision. + +### Approval request + +Written to the TaskEvents table and surfaced via `bgagent status` and notifications: + +```json +{ + "event_type": "approval_required", + "metadata": { + "manifest_summary": "Add rate limiting: 3 files created, 1 modified", + "trigger_layer": "cedar_intent_authorization", + "trigger_reason": "Task modifies CI domain (.github/workflows/ci.yml)", + "scope_options": { + "narrow": "Approve this specific file only", + "medium": "Approve all .github/ modifications for this task", + "broad": "Approve CI modifications for all tasks in this repo" + } + } +} +``` + +### Approval scopes + +| Scope | Effect | Duration | +|---|---|---| +| `narrow` | Approve only the specific action in this manifest | This task only | +| `medium` | Approve the action category for this task | This task only | +| `broad` | Add a persistent Cedar permit rule to the blueprint | All future tasks | + +Broad approvals generate a Cedar policy that is appended to the blueprint's `security.cedarPolicies` array and persisted to the RepoTable. + +### Timeout + +If no approval is received within the configured timeout (default: 4 hours), the task transitions to `PLAN_REJECTED` with reason `approval_timeout`. + +## Integration with existing orchestrator + +The planning phase inserts between hydration and session start in the existing blueprint execution: + +```mermaid +flowchart LR + A[Admission] --> B[Hydration] + B --> C[Pre-flight] + C --> P[Planning + Verification] + P --> D[Start session] + D --> E[Await completion] + E --> R[Reconciliation] + R --> F[Finalize] +``` + +### Implementation boundaries + +| Component | Changes | +|---|---| +| `orchestrator.ts` | New states (`PLANNING`, `PLAN_VERIFIED`, `PLAN_REJECTED`, `AWAITING_APPROVAL`), new steps in blueprint execution | +| `agent/src/pipeline.py` | Split into two-phase execution: planning call + constrained execution call | +| `agent/src/hooks.py` | Manifest-aware drift detection in PreToolUse hook | +| `agent/src/policy.py` | New intent-level Cedar actions and domain-mapping logic | +| `agent/prompts/` | Planning prompt variant that instructs agent to produce manifest before code | +| Blueprint / RepoConfig | `change_manifest` configuration block (allowed_paths, protected_paths, amendment_budget, etc.) | +| TaskEvents table | New event types: `manifest_produced`, `manifest_verified`, `approval_required`, `approval_granted`, `manifest_amended`, `reconciliation_complete` | +| `bgagent watch` | Display manifest items as structured progress | +| `types.ts` | New task states, manifest types, reconciliation report types | + +### Opt-in rollout + +The planning phase is enabled per-repo via blueprint configuration: + +```json +{ + "change_manifest": { + "enabled": true, + "require_approval_for_protected": true, + "amendment_budget": 3, + "approval_timeout_hours": 4, + "proportionality_check": true, + "skip_for_categories": ["docs", "chore"] + } +} +``` + +When disabled (default for existing repos), the orchestrator skips directly from pre-flight to session start, preserving current behavior. + +## Two-tier verification model + +The verification pipeline operates at two tiers, cleanly separating concerns: + +### Tier 1: Generic structural verification (platform-provided) + +Layers L1-L5 as described above. These run on every task regardless of the target codebase. They answer universal questions: + +- Is the plan well-formed? (L1) +- Is the scope proportionate and within bounds? (L2) +- Is the intent authorized? (L3) +- Does the plan match filesystem reality? (L4) +- Does the agent have the knowledge it needs? (L5) + +The manifest format is JSON because Tier 1 is a **coordination artifact** — it describes intent at a level meaningful across any tech stack. A TLA+ codebase and a React app both benefit from scope checking, Cedar authorization, and knowledge resolution. + +### Tier 2: Repo-specific formal verification (blueprint-provided) + +Some codebases have their own verification tooling — TLA+ specs, Alloy models, property-based test frameworks, Stateright model checkers, strict type systems with refinement types. For these repos, the manifest should declare what formal artifacts the agent will produce, and the repo's own verification pipeline should run before execution proceeds. + +Tier 2 hooks into the pipeline as a **custom verification step** between L5 and `PLAN_VERIFIED`: + +```mermaid +flowchart TD + L5[L5: Knowledge Resolution] -->|PASS| T2{Tier 2 configured?} + T2 -->|No| V[PLAN_VERIFIED] + T2 -->|Yes| FV[Repo Formal Verification] + FV -->|PASS| V + FV -->|FAIL| R[PLAN_REJECTED] + FV -->|REQUIRE_APPROVAL| A[AWAITING_APPROVAL] +``` + +**How it works:** + +The manifest gains an optional `formal_artifacts` field when Tier 2 is configured: + +```json +{ + "formal_artifacts": [ + { + "type": "tla_plus", + "path": "specs/rate-limiter.tla", + "purpose": "Prove mutual exclusion on concurrent rate limit counter updates", + "verifier": "tlc" + }, + { + "type": "proptest", + "path": "cdk/test/constructs/rate-limiter.proptest.ts", + "purpose": "Property: rate limit never exceeds configured max under concurrent requests", + "verifier": "jest" + } + ] +} +``` + +**Tier 2 verification flow:** + +1. The agent produces the formal artifact during the planning phase (alongside the manifest) +2. The blueprint specifies which verifier to run and its expected output format +3. The platform executes the verifier in the sandbox (time-bounded, sandboxed) +4. Pass/fail is determined by exit code and structured output + +**Blueprint configuration for Tier 2:** + +```json +{ + "change_manifest": { + "formal_verification": { + "enabled": true, + "verifiers": { + "tlc": { + "command": "java -jar tla2tools.jar -deadlock", + "timeout_seconds": 60, + "required_for": ["feature", "refactor"] + }, + "jest": { + "command": "npx jest --testPathPattern=proptest", + "timeout_seconds": 30, + "required_for": ["feature", "bugfix"] + }, + "cargo_test": { + "command": "cargo test --lib -- --include-ignored formal_", + "timeout_seconds": 120, + "required_for": ["feature"] + } + }, + "skip_for_categories": ["docs", "chore", "test"] + } + } +} +``` + +**Key design decisions:** + +- **The manifest does NOT try to be the formal spec.** It's the envelope that declares "I will produce a TLA+ spec proving X." The spec itself is a separate file in the repo's native format. +- **The platform has no opinion about verification semantics.** It knows how to run a command, check the exit code, and capture output. TLC, Alloy, Z3, proptest, Stateright — all look the same from the platform's perspective: `command → exit code → pass/fail`. +- **Tier 2 is fully opt-in.** Most repos don't have formal methods tooling. The generic Tier 1 checks still provide meaningful governance without any special setup. +- **Formal artifacts become part of the PR.** The TLA+ spec or property test produced during planning is committed alongside the implementation code, providing ongoing verification value beyond the initial task. + +**Interaction with knowledge resolution (L5):** + +When Tier 2 is configured, L5 automatically adds knowledge requirements for the verification tooling. If the blueprint declares a `tlc` verifier, L5 ensures the agent has access to TLA+ documentation and examples from the repo's existing specs (via `context_inject`). This prevents the agent from producing syntactically invalid formal artifacts. + +**When Tier 2 adds value vs. when it's overhead:** + +| Scenario | Tier 2 appropriate? | Rationale | +|---|---|---| +| Concurrent state machine changes | Yes | TLA+/Stateright catches race conditions static analysis misses | +| New API endpoint with CRUD | No | Generic scope + type checking is sufficient | +| Distributed system coordination | Yes | Model checking explores interleavings humans can't reason about | +| UI component addition | No | Property tests on render are lower value than integration tests | +| Security-critical auth changes | Maybe | Depends on whether the repo has formal security properties | + +## Observability + +### New CloudWatch metrics + +| Metric | Dimensions | Purpose | +|---|---|---| +| `ManifestVerificationDuration` | Layer, Outcome | Track verification latency | +| `ManifestOutcome` | Outcome (VERIFIED/REJECTED/APPROVAL_REQUIRED) | Rejection rate | +| `DriftDetectionBlocks` | Repo, TaskType | How often agents try to exceed scope | +| `AmendmentRate` | Repo, TaskType | Planning quality signal | +| `PlanAdherenceScore` | Repo, TaskType, Category | Execution quality signal | +| `ApprovalLatency` | Scope | Time humans take to respond | +| `KnowledgeToolsInjected` | Tool, Repo | Which doc tools are actually used | +| `KnowledgeQueriesPerTask` | Tool, TaskType | How much agents use injected docs | +| `KnowledgeGapDetected` | Repo, Domain | Dependencies declared without matching knowledge requirements | + +### Learning signal + +The `(manifest, actual_diff, PR_outcome)` triple is stored for post-hoc analysis: + +- **High adherence + merged PR** → Good planning, reinforce prompt patterns +- **High adherence + rejected PR** → Plan was wrong; improve verification heuristics +- **Low adherence (many amendments) + merged PR** → Planning prompt needs improvement +- **Frequent REQUIRE_APPROVAL for same action** → Consider adding a persistent permit +- **Knowledge tool injected + zero queries** → Agent didn't need it; remove from auto-detect for this domain +- **No knowledge tool + build failure on unknown API** → Missing knowledge requirement; improve planning prompt to declare dependencies + +## Cost impact + +| Component | Additional cost per task | Notes | +|---|---|---| +| Planning phase (extra model call) | ~$0.02-0.08 | Short context, structured output | +| Verification L1-L4 (in-process) | ~$0 | CPU-only, <2s | +| Knowledge resolution L5 | ~$0-0.01 | MCP server startup + index fetch; cached after first use | +| Knowledge tool usage during execution | ~$0.01-0.05 | Per-query cost to doc servers; bounded by query budget | +| Manifest storage | Negligible | Small JSON in DynamoDB | +| Drift detection | ~$0 | In-memory check per tool call | +| Reconciliation | ~$0.01 | Runs `must_pass` commands (would run anyway) | + +**Net impact:** ~$0.04-0.15 per task. Offset by: +- Reduced wasted compute from early scope-creep detection (estimated 15-20% of tasks drift and get rejected at PR review) +- Fewer retry loops from API hallucination when knowledge tools are available (estimated 2-4 turns saved per task involving unfamiliar libraries) + +## Security considerations + +- **Manifest injection** — The manifest is agent-produced. A compromised or jailbroken agent could craft a manifest that passes verification but is misleading. Mitigation: verification checks the manifest against actual filesystem state (L4), and drift detection constrains execution to declared scope regardless of what the manifest claims. +- **Amendment abuse** — An agent could repeatedly amend to gradually expand scope. Mitigation: amendment budget, full re-verification on each amendment, all amendments logged. +- **Approval fatigue** — Too many approval requests train humans to auto-approve. Mitigation: `auto_approve_categories` for low-risk work, proportionality thresholds tuned to minimize false positives, `broad` scope option to permanently resolve recurring patterns. +- **Hash tampering** — The verification report hashes are computed server-side (in the hook, not by the agent) from the verified manifest content. The agent cannot modify the hashes without re-triggering verification. +- **Knowledge tool as exfiltration channel** — An injected MCP server could be used to send data out of the sandbox. Mitigation: knowledge tools must be from the blueprint's allowlist (`knowledge_tools.available`), Cedar gates every injection, and MCP servers run in read-only mode (no write/execute actions exposed). Network egress is already constrained by the MicroVM sandbox. +- **Poisoned documentation** — If a knowledge tool returns malicious content (prompt injection in docs), the agent could be manipulated. Mitigation: documentation responses pass through the same content screening as PR comments (Bedrock Guardrails for prompt attack detection), and the manifest's scope constraints limit what the agent can do regardless of what it's told. From e5328ca9f9adc51d9bafd6f3c0a0f7489ba42a71 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 00:52:27 -0500 Subject: [PATCH 13/19] chore(review): implement fixes from automated review --- agent/src/pipeline.py | 27 ++- cdk/src/handlers/cleanup-pending-uploads.ts | 50 ++--- cdk/src/handlers/confirm-uploads.ts | 4 + .../handlers/shared/attachment-screening.ts | 13 +- cdk/src/handlers/shared/create-task-core.ts | 23 +- cdk/src/handlers/shared/orchestrator.ts | 1 + .../shared/resolve-url-attachments.ts | 51 ++++- cdk/src/handlers/shared/types.ts | 54 ++++- cdk/src/handlers/slack-command-processor.ts | 22 +- .../handlers/cleanup-pending-uploads.test.ts | 19 +- cdk/test/handlers/confirm-uploads.test.ts | 116 ++++++++++ .../shared/attachment-screening.test.ts | 153 +++++++++++++ .../shared/resolve-url-attachments.test.ts | 203 +++++++++++++++++- scripts/check-types-sync.ts | 3 + 14 files changed, 676 insertions(+), 63 deletions(-) diff --git a/agent/src/pipeline.py b/agent/src/pipeline.py index 2845afac..c4116fc8 100644 --- a/agent/src/pipeline.py +++ b/agent/src/pipeline.py @@ -473,12 +473,27 @@ def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None: if config.attachments: from attachments import download_attachments - with task_span("task.attachment_download"): - prepared_attachments = download_attachments(config.attachments, setup.repo_dir) - progress.write_agent_milestone( - "attachments_downloaded", - f"count={len(prepared_attachments)}", - ) + try: + with task_span("task.attachment_download"): + prepared_attachments = download_attachments( + config.attachments, setup.repo_dir + ) + progress.write_agent_milestone( + "attachments_downloaded", + f"count={len(prepared_attachments)}", + ) + except RuntimeError as e: + log("ERROR", f"Attachment integrity check failed: {e}") + raise RuntimeError( + f"Attachment download/verification failed: {e}. " + "The task cannot proceed without valid attachments." + ) from e + except Exception as e: + err_type = type(e).__name__ + log("ERROR", f"Attachment download failed: {err_type}: {e}") + raise RuntimeError( + f"Failed to download task attachments from S3: {err_type}: {e}" + ) from e # Log discovered repo-level project configuration # (all files loaded by setting_sources=["project"]) diff --git a/cdk/src/handlers/cleanup-pending-uploads.ts b/cdk/src/handlers/cleanup-pending-uploads.ts index 60f649f2..adcc7477 100644 --- a/cdk/src/handlers/cleanup-pending-uploads.ts +++ b/cdk/src/handlers/cleanup-pending-uploads.ts @@ -40,7 +40,7 @@ import { UpdateItemCommand, PutItemCommand, } from '@aws-sdk/client-dynamodb'; -import { DeleteObjectsCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'; +import { DeleteObjectsCommand, ListObjectVersionsCommand, S3Client } from '@aws-sdk/client-s3'; import { ulid } from 'ulid'; import { ATTACHMENT_OBJECT_KEY_PREFIX } from '../constructs/attachments-bucket'; import { logger } from './shared/logger'; @@ -187,41 +187,43 @@ async function cancelExpiredTask(task: ExpiredTask): Promise { async function cleanupTaskAttachments(task: ExpiredTask): Promise { const prefix = `${ATTACHMENT_OBJECT_KEY_PREFIX}${task.user_id}/${task.task_id}/`; - let continuationToken: string | undefined; + let keyMarker: string | undefined; + let versionIdMarker: string | undefined; let totalDeleted = 0; do { - const listResp = await s3.send(new ListObjectsV2Command({ + const listResp = await s3.send(new ListObjectVersionsCommand({ Bucket: ATTACHMENTS_BUCKET, Prefix: prefix, - ContinuationToken: continuationToken, + KeyMarker: keyMarker, + VersionIdMarker: versionIdMarker, })); - const objects = listResp.Contents; - if (!objects || objects.length === 0) break; + // Collect all versions and delete markers for deletion + const objects = [ + ...(listResp.Versions ?? []).map(v => ({ Key: v.Key!, VersionId: v.VersionId })), + ...(listResp.DeleteMarkers ?? []).map(d => ({ Key: d.Key!, VersionId: d.VersionId })), + ].filter(obj => obj.Key !== undefined); - const keys = objects - .map(obj => obj.Key) - .filter((key): key is string => key !== undefined); + if (objects.length === 0) break; - if (keys.length > 0) { - const deleteResp = await s3.send(new DeleteObjectsCommand({ - Bucket: ATTACHMENTS_BUCKET, - Delete: { Objects: keys.map(Key => ({ Key })) }, - })); - - if (deleteResp.Errors && deleteResp.Errors.length > 0) { - logger.error('Partial S3 cleanup failure for expired pending-upload task', { - task_id: task.task_id, - failedKeys: deleteResp.Errors.map(e => e.Key), - }); - } + const deleteResp = await s3.send(new DeleteObjectsCommand({ + Bucket: ATTACHMENTS_BUCKET, + Delete: { Objects: objects.map(({ Key, VersionId }) => ({ Key, VersionId })) }, + })); - totalDeleted += keys.length - (deleteResp.Errors?.length ?? 0); + if (deleteResp.Errors && deleteResp.Errors.length > 0) { + logger.error('Partial S3 cleanup failure for expired pending-upload task', { + task_id: task.task_id, + failedKeys: deleteResp.Errors.map(e => e.Key), + }); } - continuationToken = listResp.NextContinuationToken; - } while (continuationToken); + totalDeleted += objects.length - (deleteResp.Errors?.length ?? 0); + + keyMarker = listResp.NextKeyMarker; + versionIdMarker = listResp.NextVersionIdMarker; + } while (keyMarker); if (totalDeleted > 0) { logger.info('Cleaned up S3 objects for expired pending-upload task', { diff --git a/cdk/src/handlers/confirm-uploads.ts b/cdk/src/handlers/confirm-uploads.ts index 983dc8d7..a1042bef 100644 --- a/cdk/src/handlers/confirm-uploads.ts +++ b/cdk/src/handlers/confirm-uploads.ts @@ -799,6 +799,10 @@ async function decrementConcurrency(userId: string): Promise { error: err instanceof Error ? err.message : String(err), metric_type: 'concurrency_counter_leak', }); + throw new Error( + `Concurrency counter decrement failed for user ${userId} after ${maxAttempts} attempts. ` + + 'Manual intervention may be required to reset the counter.', + ); } } } diff --git a/cdk/src/handlers/shared/attachment-screening.ts b/cdk/src/handlers/shared/attachment-screening.ts index 91aabdaf..10ee2a7c 100644 --- a/cdk/src/handlers/shared/attachment-screening.ts +++ b/cdk/src/handlers/shared/attachment-screening.ts @@ -404,7 +404,18 @@ function assertImageDimensionsWithinLimits( } else if (contentType === 'image/jpeg') { dims = readJpegDimensions(content); if (!dims) { - // Non-fatal: if we can't parse dimensions, let Bedrock handle rejection + // Fail-closed for large JPEGs where dimensions cannot be verified (> 5 MB). + // Smaller files are allowed through to Bedrock which will reject if oversized. + if (content.length > 5 * 1024 * 1024) { + throw new AttachmentScreeningError( + `Image "${filename}" is ${(content.length / (1024 * 1024)).toFixed(1)} MB and its dimensions ` + + 'could not be verified. Please use a standard JPEG encoder or convert to PNG.', + ); + } + logger.warn('Could not parse JPEG dimensions — relying on Bedrock validation', { + filename, + size_bytes: content.length, + }); return; } } else { diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index 0a27958d..c4037531 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -25,7 +25,7 @@ import { BedrockRuntimeClient, ApplyGuardrailCommand } from '@aws-sdk/client-bed import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; import { PutObjectCommand, DeleteObjectsCommand, S3Client } from '@aws-sdk/client-s3'; -import { DynamoDBDocumentClient, PutCommand, QueryCommand, GetCommand } from '@aws-sdk/lib-dynamodb'; +import { DynamoDBDocumentClient, PutCommand, QueryCommand, GetCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; import { createPresignedPost } from '@aws-sdk/s3-presigned-post'; import type { APIGatewayProxyResult } from 'aws-lambda'; import { ulid } from 'ulid'; @@ -666,12 +666,31 @@ export async function createTaskCore( taskRecord, validatedAttachments, context.userId, taskId, s3Client, ); } catch (presignErr) { - logger.error('Failed to generate presigned upload instructions — task orphaned in PENDING_UPLOADS', { + logger.error('Failed to generate presigned upload instructions — transitioning task to FAILED', { task_id: taskId, error: presignErr instanceof Error ? presignErr.message : String(presignErr), request_id: requestId, metric_type: 'presigned_post_generation_failure', }); + // Transition task to FAILED so it doesn't remain orphaned in PENDING_UPLOADS + try { + await ddb.send(new UpdateCommand({ + TableName: TABLE_NAME, + Key: { task_id: taskId }, + UpdateExpression: 'SET #s = :failed, error_message = :err, updated_at = :now', + ExpressionAttributeNames: { '#s': 'status' }, + ExpressionAttributeValues: { + ':failed': TaskStatus.FAILED, + ':err': 'Failed to generate upload instructions. Please try again.', + ':now': new Date().toISOString(), + }, + })); + } catch (cleanupErr) { + logger.error('Failed to transition orphaned task to FAILED', { + task_id: taskId, + error: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr), + }); + } return errorResponse(500, ErrorCode.INTERNAL_ERROR, 'Failed to generate upload instructions. Please try again.', requestId); } diff --git a/cdk/src/handlers/shared/orchestrator.ts b/cdk/src/handlers/shared/orchestrator.ts index b2d482a7..6e85068a 100644 --- a/cdk/src/handlers/shared/orchestrator.ts +++ b/cdk/src/handlers/shared/orchestrator.ts @@ -483,6 +483,7 @@ export async function hydrateAndTransition(task: TaskRecord, blueprintConfig?: B bucketName: ATTACHMENTS_BUCKET_NAME, screeningConfig, githubToken, + githubInstallationDomain: process.env.GITHUB_INSTALLATION_DOMAIN, }, ); } diff --git a/cdk/src/handlers/shared/resolve-url-attachments.ts b/cdk/src/handlers/shared/resolve-url-attachments.ts index e194e6bf..df03ffc0 100644 --- a/cdk/src/handlers/shared/resolve-url-attachments.ts +++ b/cdk/src/handlers/shared/resolve-url-attachments.ts @@ -67,7 +67,13 @@ const PRIVATE_IP_RANGES = [ { prefix: '169.254.', mask: null }, { prefix: '127.', mask: null }, { prefix: '0.', mask: null }, - { prefix: '100.64.', mask: null }, // CGN / Shared Address Space (RFC 6598) + { + prefix: '100.', + mask: (ip: string) => { + const second = parseInt(ip.split('.')[1], 10); + return second >= 64 && second <= 127; // 100.64.0.0/10 (RFC 6598) + }, + }, // IPv6 { prefix: '::1', mask: null }, { prefix: '::', mask: (ip: string) => ip === '::' }, // Unspecified address (could route to localhost) @@ -131,14 +137,29 @@ async function resolveAndValidate(hostname: string): Promise { try { // Try IPv4 first (more common for HTTP endpoints) addresses = await dns.resolve4(hostname); - } catch (ipv4Err) { - try { - addresses = await dns.resolve6(hostname); - } catch (ipv6Err) { - throw new AttachmentResolutionError( - `DNS resolution failed for '${hostname}'. Check that the URL is correct and the server is reachable.`, - { cause: new AggregateError([ipv4Err, ipv6Err], `Both IPv4 and IPv6 resolution failed for '${hostname}'`) }, - ); + } catch (ipv4Err: any) { + // Only fall through to IPv6 for NODATA/NXDOMAIN — system errors should propagate + const dnsNoRecordCodes = ['ENODATA', 'ENOTFOUND', 'NODATA']; + if (!dnsNoRecordCodes.includes(ipv4Err?.code)) { + // System-level DNS failure (ENOMEM, ESERVFAIL, etc.) — do not mask + try { + addresses = await dns.resolve6(hostname); + } catch (ipv6Err) { + throw new AttachmentResolutionError( + `DNS resolution failed for '${hostname}': ${ipv4Err?.code ?? ipv4Err?.message ?? 'unknown error'}`, + { cause: new AggregateError([ipv4Err, ipv6Err], `Both IPv4 and IPv6 resolution failed for '${hostname}'`) }, + ); + } + } else { + // No IPv4 records — try IPv6 + try { + addresses = await dns.resolve6(hostname); + } catch (ipv6Err) { + throw new AttachmentResolutionError( + `DNS resolution failed for '${hostname}'. Check that the URL is correct and the server is reachable.`, + { cause: new AggregateError([ipv4Err, ipv6Err], `Both IPv4 and IPv6 resolution failed for '${hostname}'`) }, + ); + } } } @@ -207,7 +228,17 @@ async function pinnedHttpsRequest( }, (res) => { const chunks: Buffer[] = []; - res.on('data', (chunk: Buffer) => chunks.push(chunk)); + let totalBytes = 0; + res.on('data', (chunk: Buffer) => { + totalBytes += chunk.length; + if (totalBytes > MAX_FETCH_SIZE_BYTES) { + res.destroy(); + agent.destroy(); + reject(new Error(`Response exceeds ${MAX_FETCH_SIZE_BYTES} byte size limit`)); + return; + } + chunks.push(chunk); + }); res.on('end', () => { const body = Buffer.concat(chunks); const responseHeaders = new Headers(); diff --git a/cdk/src/handlers/shared/types.ts b/cdk/src/handlers/shared/types.ts index 36d5121f..f5f00254 100644 --- a/cdk/src/handlers/shared/types.ts +++ b/cdk/src/handlers/shared/types.ts @@ -418,10 +418,47 @@ export type ScreeningResult = | { readonly status: 'blocked'; readonly screened_at: string; readonly categories: [string, ...string[]] }; // --------------------------------------------------------------------------- -// Attachment record (persisted metadata in TaskRecord) +// Attachment record (persisted metadata in TaskRecord) — discriminated union +// keyed on screening.status ensures that passed records always have storage fields. // --------------------------------------------------------------------------- -export interface AttachmentRecord { +interface BaseAttachmentRecord { + readonly attachment_id: string; + readonly type: AttachmentType; + readonly content_type: string; + readonly filename: string; + readonly source_url?: string; + readonly token_estimate?: number; +} + +export interface PendingAttachmentRecord extends BaseAttachmentRecord { + readonly screening: { readonly status: 'pending' }; + readonly s3_key?: string; + readonly s3_version_id?: string; + readonly size_bytes?: number; + readonly checksum_sha256?: string; +} + +export interface PassedAttachmentRecord extends BaseAttachmentRecord { + readonly screening: { readonly status: 'passed'; readonly screened_at: string }; + readonly s3_key: string; + readonly s3_version_id: string; + readonly size_bytes: number; + readonly checksum_sha256: string; +} + +export interface BlockedAttachmentRecord extends BaseAttachmentRecord { + readonly screening: { readonly status: 'blocked'; readonly screened_at: string; readonly categories: [string, ...string[]] }; + readonly s3_key?: string; + readonly s3_version_id?: string; + readonly size_bytes?: number; + readonly checksum_sha256?: string; +} + +export type AttachmentRecord = PendingAttachmentRecord | PassedAttachmentRecord | BlockedAttachmentRecord; + +/** Parameters for creating an AttachmentRecord — accepts the union of all fields. */ +export type CreateAttachmentRecordParams = { readonly attachment_id: string; readonly type: AttachmentType; readonly content_type: string; @@ -433,22 +470,23 @@ export interface AttachmentRecord { readonly source_url?: string; readonly checksum_sha256?: string; readonly token_estimate?: number; -} - -/** Parameters for creating an AttachmentRecord with cross-field invariant validation. */ -export type CreateAttachmentRecordParams = AttachmentRecord; +}; /** * Factory function enforcing cross-field invariants on AttachmentRecord construction. - * Validates that required fields are present based on screening status and type. + * Returns the appropriate discriminated union variant based on screening status. */ export function createAttachmentRecord(params: CreateAttachmentRecordParams): AttachmentRecord { if (params.screening.status === 'passed') { if (!params.s3_key || !params.s3_version_id || !params.checksum_sha256 || !params.size_bytes) { throw new Error('Passed screening requires s3_key, s3_version_id, checksum_sha256, and size_bytes'); } + return params as PassedAttachmentRecord; + } + if (params.screening.status === 'blocked') { + return params as BlockedAttachmentRecord; } - return params; + return params as PendingAttachmentRecord; } // --------------------------------------------------------------------------- diff --git a/cdk/src/handlers/slack-command-processor.ts b/cdk/src/handlers/slack-command-processor.ts index f9a72ec2..3832ffa0 100644 --- a/cdk/src/handlers/slack-command-processor.ts +++ b/cdk/src/handlers/slack-command-processor.ts @@ -375,8 +375,8 @@ const SLACK_FILE_MAX_SIZE_BYTES = 10 * 1024 * 1024; /** Max number of file attachments per Slack message. */ const SLACK_FILE_MAX_COUNT = 10; -/** MIME types supported for attachments (must match validation.ts). */ -const SUPPORTED_IMAGE_MIMES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']); +/** MIME types supported for attachments (must match validation.ts — PNG/JPEG only). */ +const SUPPORTED_IMAGE_MIMES = new Set(['image/png', 'image/jpeg']); const SUPPORTED_FILE_MIMES = new Set([ 'text/plain', 'text/csv', 'text/markdown', 'application/json', 'application/pdf', 'text/x-log', @@ -428,6 +428,13 @@ async function extractSlackFileAttachments( continue; } + // Validate the download URL points to a legitimate Slack domain before + // sending the bot token — prevents SSRF and token exfiltration via crafted events. + if (!isSlackFileUrl(file.url_private_download)) { + errors.push(`\`${file.name}\` (invalid download URL — not a Slack domain)`); + continue; + } + // Download the file from Slack CDN using the bot token try { const response = await fetch(file.url_private_download, { @@ -476,6 +483,17 @@ async function extractSlackFileAttachments( return attachments; } +/** Validate that a URL points to a legitimate Slack file domain. */ +function isSlackFileUrl(url: string): boolean { + try { + const parsed = new URL(url); + return parsed.protocol === 'https:' + && (parsed.hostname === 'files.slack.com' || parsed.hostname.endsWith('.slack.com')); + } catch { + return false; + } +} + // ─── Helpers ────────────────────────────────────────────────────────────────── async function lookupPlatformUser(teamId: string, userId: string): Promise { diff --git a/cdk/test/handlers/cleanup-pending-uploads.test.ts b/cdk/test/handlers/cleanup-pending-uploads.test.ts index 815a7763..5ca077bf 100644 --- a/cdk/test/handlers/cleanup-pending-uploads.test.ts +++ b/cdk/test/handlers/cleanup-pending-uploads.test.ts @@ -30,7 +30,7 @@ jest.mock('@aws-sdk/client-dynamodb', () => ({ jest.mock('@aws-sdk/client-s3', () => ({ S3Client: jest.fn(() => ({ send: mockS3Send })), - ListObjectsV2Command: jest.fn((input: any) => ({ input, _type: 'ListObjectsV2' })), + ListObjectVersionsCommand: jest.fn((input: any) => ({ input, _type: 'ListObjectVersions' })), DeleteObjectsCommand: jest.fn((input: any) => ({ input, _type: 'DeleteObjects' })), })); @@ -132,24 +132,25 @@ describe('cleanup-pending-uploads handler', () => { // write event mockDdbSend.mockResolvedValueOnce({}); - // S3 list returns objects + // S3 ListObjectVersions returns versions mockS3Send.mockResolvedValueOnce({ - Contents: [ - { Key: 'attachments/user-123/TASK001/ATT001/image.png' }, - { Key: 'attachments/user-123/TASK001/ATT002/doc.pdf' }, + Versions: [ + { Key: 'attachments/user-123/TASK001/ATT001/image.png', VersionId: 'v1' }, + { Key: 'attachments/user-123/TASK001/ATT002/doc.pdf', VersionId: 'v2' }, ], + DeleteMarkers: [], }); // S3 delete succeeds mockS3Send.mockResolvedValueOnce({ Deleted: [{}, {}] }); await handler(); - // Verify S3 delete was called with the right keys + // Verify S3 delete was called with VersionIds (versioned cleanup) const deleteCall = mockS3Send.mock.calls[1][0]; expect(deleteCall.input.Bucket).toBe('test-attachments-bucket'); expect(deleteCall.input.Delete.Objects).toEqual([ - { Key: 'attachments/user-123/TASK001/ATT001/image.png' }, - { Key: 'attachments/user-123/TASK001/ATT002/doc.pdf' }, + { Key: 'attachments/user-123/TASK001/ATT001/image.png', VersionId: 'v1' }, + { Key: 'attachments/user-123/TASK001/ATT002/doc.pdf', VersionId: 'v2' }, ]); }); @@ -185,7 +186,7 @@ describe('cleanup-pending-uploads handler', () => { // First task: cancel succeeds mockDdbSend.mockResolvedValueOnce({}); mockDdbSend.mockResolvedValueOnce({}); // event - mockS3Send.mockResolvedValueOnce({ Contents: [] }); // no S3 objects + mockS3Send.mockResolvedValueOnce({ Versions: [], DeleteMarkers: [] }); // no S3 objects // Second task: cancel fails with infra error const infraErr = new Error('Timeout'); diff --git a/cdk/test/handlers/confirm-uploads.test.ts b/cdk/test/handlers/confirm-uploads.test.ts index b38b6f33..caa28af2 100644 --- a/cdk/test/handlers/confirm-uploads.test.ts +++ b/cdk/test/handlers/confirm-uploads.test.ts @@ -196,4 +196,120 @@ describe('confirm-uploads handler', () => { const body = JSON.parse(result.body); expect(body.error.code).toBe('ATTACHMENT_UPLOAD_MISSING'); }); + + test('happy path: HeadObject → screen → transition to SUBMITTED → invoke orchestrator', async () => { + const { screenImage, screenTextFile } = jest.requireMock('../../src/handlers/shared/attachment-screening'); + + const pngContent = Buffer.alloc(1024); + pngContent.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); // PNG magic + const textContent = Buffer.alloc(512); + textContent.write('hello world'); + + // Screening passes + screenImage.mockResolvedValue({ + content: pngContent, + contentType: 'image/png', + checksum: 'abc123', + screening: { status: 'passed' }, + }); + screenTextFile.mockResolvedValue({ + content: textContent, + contentType: 'text/plain', + checksum: 'def456', + screening: { status: 'passed' }, + }); + + // Use mockImplementation to route DDB calls correctly + let ddbCallCount = 0; + ddbSend.mockImplementation(() => { + ddbCallCount++; + switch (ddbCallCount) { + case 1: return Promise.resolve({ Item: PENDING_TASK }); // GetCommand (task) + case 2: return Promise.resolve({ Item: { active_count: 1 } }); // GetCommand (concurrency pre-check) + case 3: return Promise.resolve({}); // UpdateCommand (checkConcurrency) + case 4: return Promise.resolve({}); // UpdateCommand (status transition) + case 5: return Promise.resolve({}); // PutCommand (event) + default: return Promise.resolve({}); + } + }); + + // Use mockImplementation for S3 calls to handle interleaving + let s3CallCount = 0; + s3Send.mockImplementation((cmd: any) => { + s3CallCount++; + if (cmd._type === 'S3Head') { + // HeadObject — return valid metadata + const isAtt1 = cmd.input.Key?.includes('att-1'); + return Promise.resolve({ + VersionId: isAtt1 ? 'v1' : 'v2', + ContentLength: isAtt1 ? 1024 : 512, + }); + } + if (cmd._type === 'S3Get') { + // GetObject — return content for screening + const isAtt1 = cmd.input.Key?.includes('att-1'); + const content = isAtt1 ? pngContent : textContent; + return Promise.resolve({ + Body: { transformToByteArray: () => content }, + }); + } + if (cmd._type === 'S3Put') { + return Promise.resolve({ VersionId: 'v-screened' }); + } + return Promise.resolve({}); + }); + + lambdaSend.mockResolvedValueOnce({}); + + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.data.status).toBe('SUBMITTED'); + expect(lambdaSend).toHaveBeenCalled(); + }); + + test('returns 429 when concurrency pre-check fails', async () => { + ddbSend.mockResolvedValueOnce({ Item: PENDING_TASK }); + s3Send + .mockResolvedValueOnce({ VersionId: 'v1', ContentLength: 1024 }) + .mockResolvedValueOnce({ VersionId: 'v2', ContentLength: 512 }); + + // Concurrency pre-check shows user is at limit + ddbSend.mockResolvedValueOnce({ Item: { active_count: 3 } }); + + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(429); + const body = JSON.parse(result.body); + expect(body.error.code).toBe('RATE_LIMIT_EXCEEDED'); + }); + + test('returns 400 ATTACHMENT_BLOCKED when screening rejects content', async () => { + const { screenImage, AttachmentScreeningError } = jest.requireMock('../../src/handlers/shared/attachment-screening'); + + ddbSend.mockResolvedValueOnce({ Item: PENDING_TASK }); + s3Send + .mockResolvedValueOnce({ VersionId: 'v1', ContentLength: 1024 }) + .mockResolvedValueOnce({ VersionId: 'v2', ContentLength: 512 }); + + // Pre-check passes + ddbSend.mockResolvedValueOnce({ Item: { active_count: 0 } }); + + // GetObject for first attachment + const pngContent = Buffer.alloc(1024); + s3Send.mockResolvedValueOnce({ Body: { transformToByteArray: () => pngContent } }); + + // Screening blocks the image + screenImage.mockRejectedValueOnce(new AttachmentScreeningError('Inappropriate content detected')); + + // DDB updates for failure (conditional write + event) + ddbSend.mockResolvedValue({}); + // S3 cleanup (delete objects) + s3Send.mockResolvedValue({}); + + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error.code).toBe('ATTACHMENT_BLOCKED'); + }); }); diff --git a/cdk/test/handlers/shared/attachment-screening.test.ts b/cdk/test/handlers/shared/attachment-screening.test.ts index fb9f310a..2f340567 100644 --- a/cdk/test/handlers/shared/attachment-screening.test.ts +++ b/cdk/test/handlers/shared/attachment-screening.test.ts @@ -26,6 +26,7 @@ import { readJpegDimensions, readPngDimensions, screenImage, + screenTextFile, } from '../../../src/handlers/shared/attachment-screening'; const ARCHITECTURE_PNG = path.join( @@ -244,4 +245,156 @@ describe('screenImage', () => { expect(result.screening.categories).toContain('SEXUAL'); } }); + + test('retries on 429 status and eventually succeeds', async () => { + const png = minimalPng(); + const send = jest.fn() + .mockRejectedValueOnce({ $metadata: { httpStatusCode: 429 }, message: 'throttled' }) + .mockRejectedValueOnce({ $metadata: { httpStatusCode: 429 }, message: 'throttled' }) + .mockResolvedValueOnce({ action: 'NONE', outputs: [], assessments: [] }); + + const retryConfig = { + bedrockClient: { send } as unknown as BedrockRuntimeClient, + guardrailId: 'g', + guardrailVersion: '1', + }; + + const result = await screenImage(png, 'image/png', 'retry.png', retryConfig); + expect(result.screening.status).toBe('passed'); + expect(send).toHaveBeenCalledTimes(3); + }); + + test('throws after exhausting retries on persistent 500', async () => { + const png = minimalPng(); + const error = { $metadata: { httpStatusCode: 500 }, message: 'internal error' }; + const send = jest.fn().mockRejectedValue(error); + + const retryConfig = { + bedrockClient: { send } as unknown as BedrockRuntimeClient, + guardrailId: 'g', + guardrailVersion: '1', + }; + + await expect(screenImage(png, 'image/png', 'fail.png', retryConfig)).rejects.toBeDefined(); + // 1 initial + 3 retries = 4 attempts + expect(send).toHaveBeenCalledTimes(4); + }); + + test('does not retry non-retryable errors (e.g., 400)', async () => { + const png = minimalPng(); + const error = { $metadata: { httpStatusCode: 400 }, message: 'bad request' }; + const send = jest.fn().mockRejectedValue(error); + + const retryConfig = { + bedrockClient: { send } as unknown as BedrockRuntimeClient, + guardrailId: 'g', + guardrailVersion: '1', + }; + + await expect(screenImage(png, 'image/png', 'bad.png', retryConfig)).rejects.toBeDefined(); + // Should not retry — only 1 attempt + expect(send).toHaveBeenCalledTimes(1); + }); + + test('rejects large JPEG with unparseable dimensions (fail-closed)', async () => { + // 6 MB JPEG-like buffer with valid signature but no SOF marker + const largeJpeg = Buffer.alloc(6 * 1024 * 1024); + largeJpeg[0] = 0xff; + largeJpeg[1] = 0xd8; + largeJpeg[2] = 0xff; + + const send = jest.fn(); + const retryConfig = { + bedrockClient: { send } as unknown as BedrockRuntimeClient, + guardrailId: 'g', + guardrailVersion: '1', + }; + + await expect( + screenImage(largeJpeg, 'image/jpeg', 'obfuscated.jpg', retryConfig), + ).rejects.toThrow('dimensions could not be verified'); + expect(send).not.toHaveBeenCalled(); + }); +}); + +describe('screenTextFile', () => { + test('screens plain text content', async () => { + const config = { + bedrockClient: mockBedrockPass(), + guardrailId: 'test-guardrail', + guardrailVersion: '1', + }; + + const content = Buffer.from('Hello, this is a test file with some content.'); + const result = await screenTextFile(content, 'text/plain', 'test.txt', config); + + expect(result.screening.status).toBe('passed'); + expect(result.content).toBe(content); + expect(result.checksum).toMatch(/^[0-9a-f]{64}$/); + }); + + test('screens CSV content', async () => { + const config = { + bedrockClient: mockBedrockPass(), + guardrailId: 'test-guardrail', + guardrailVersion: '1', + }; + + const content = Buffer.from('name,age\nAlice,30\nBob,25'); + const result = await screenTextFile(content, 'text/csv', 'data.csv', config); + expect(result.screening.status).toBe('passed'); + }); + + test('returns blocked status for text that triggers guardrail', async () => { + const config = { + bedrockClient: mockBedrockBlock(), + guardrailId: 'test-guardrail', + guardrailVersion: '1', + }; + + const content = Buffer.from('This content triggers the guardrail'); + const result = await screenTextFile(content, 'text/plain', 'bad.txt', config); + expect(result.screening.status).toBe('blocked'); + if (result.screening.status === 'blocked') { + expect(result.screening.categories.length).toBeGreaterThan(0); + } + }); + + test('throws for PDF with no extractable text', async () => { + // Mock pdf-parse to return empty text + jest.mock('pdf-parse', () => ({ + __esModule: true, + default: jest.fn().mockResolvedValue({ text: '' }), + }), { virtual: true }); + + const config = { + bedrockClient: mockBedrockPass(), + guardrailId: 'test-guardrail', + guardrailVersion: '1', + }; + + // A minimal PDF-like buffer (pdf-parse is mocked so content doesn't matter) + const content = Buffer.from('%PDF-1.4 empty'); + + await expect( + screenTextFile(content, 'application/pdf', 'empty.pdf', config), + ).rejects.toThrow(/no extractable text/); + }); + + test('retries on transient Bedrock errors for text screening', async () => { + const send = jest.fn() + .mockRejectedValueOnce({ $metadata: { httpStatusCode: 503 }, message: 'service unavailable' }) + .mockResolvedValueOnce({ action: 'NONE', outputs: [], assessments: [] }); + + const config = { + bedrockClient: { send } as unknown as BedrockRuntimeClient, + guardrailId: 'g', + guardrailVersion: '1', + }; + + const content = Buffer.from('retry test'); + const result = await screenTextFile(content, 'text/plain', 'retry.txt', config); + expect(result.screening.status).toBe('passed'); + expect(send).toHaveBeenCalledTimes(2); + }); }); diff --git a/cdk/test/handlers/shared/resolve-url-attachments.test.ts b/cdk/test/handlers/shared/resolve-url-attachments.test.ts index 8990c51c..8a80374d 100644 --- a/cdk/test/handlers/shared/resolve-url-attachments.test.ts +++ b/cdk/test/handlers/shared/resolve-url-attachments.test.ts @@ -17,7 +17,8 @@ * SOFTWARE. */ -import { isPrivateIp } from '../../../src/handlers/shared/resolve-url-attachments'; +import { isPrivateIp, resolveUrlAttachments } from '../../../src/handlers/shared/resolve-url-attachments'; +import { createAttachmentRecord } from '../../../src/handlers/shared/types'; describe('isPrivateIp', () => { describe('IPv4 private ranges', () => { @@ -61,6 +62,17 @@ describe('isPrivateIp', () => { expect(isPrivateIp('100.64.255.255')).toBeDefined(); }); + test('blocks full RFC 6598 range (100.64.0.0/10: 100.64-127.x.x)', () => { + expect(isPrivateIp('100.65.0.1')).toBeDefined(); + expect(isPrivateIp('100.100.0.1')).toBeDefined(); + expect(isPrivateIp('100.127.255.254')).toBeDefined(); + }); + + test('allows 100.128.x.x (above RFC 6598 range)', () => { + expect(isPrivateIp('100.128.0.1')).toBeUndefined(); + expect(isPrivateIp('100.200.0.1')).toBeUndefined(); + }); + test('allows public IPv4 addresses', () => { expect(isPrivateIp('8.8.8.8')).toBeUndefined(); expect(isPrivateIp('1.1.1.1')).toBeUndefined(); @@ -112,3 +124,192 @@ describe('isPrivateIp', () => { }); }); }); + +// --------------------------------------------------------------------------- +// resolveUrlAttachments integration tests +// --------------------------------------------------------------------------- + +// Mock DNS, https, and S3 +jest.mock('dns', () => ({ + promises: { + resolve4: jest.fn(), + resolve6: jest.fn(), + }, +})); + +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn().mockImplementation(() => ({ send: jest.fn() })), + PutObjectCommand: jest.fn(), +})); + +jest.mock('../../../src/handlers/shared/attachment-screening', () => ({ + screenImage: jest.fn(), + screenTextFile: jest.fn(), + AttachmentScreeningError: class AttachmentScreeningError extends Error { + constructor(message: string) { super(message); this.name = 'AttachmentScreeningError'; } + }, +})); + +jest.mock('../../../src/handlers/shared/image-tokens', () => ({ + estimateImageTokensFromBuffer: jest.fn().mockReturnValue(100), +})); + +jest.mock('../../../src/handlers/shared/validation', () => ({ + isAllowedMimeType: jest.fn().mockReturnValue(true), + validateMagicBytes: jest.fn().mockReturnValue(true), +})); + +jest.mock('../../../src/handlers/shared/logger', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +describe('resolveUrlAttachments', () => { + const dns = jest.requireMock('dns').promises; + const { screenImage } = jest.requireMock('../../../src/handlers/shared/attachment-screening'); + const { isAllowedMimeType, validateMagicBytes } = jest.requireMock('../../../src/handlers/shared/validation'); + + const PNG_MAGIC = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + + const mockS3Client = { send: jest.fn().mockResolvedValue({ VersionId: 'v1' }) }; + const mockScreeningConfig = { + guardrailId: 'g-123', + guardrailVersion: '1', + bedrockClient: {}, + }; + + beforeEach(() => { + jest.clearAllMocks(); + dns.resolve4.mockResolvedValue(['1.2.3.4']); + screenImage.mockResolvedValue({ + content: PNG_MAGIC, + contentType: 'image/png', + checksum: 'abc123', + screening: { status: 'passed' }, + }); + }); + + test('skips attachments that are not pending URL type', async () => { + const passedAttachment = createAttachmentRecord({ + attachment_id: 'att-1', + type: 'image', + content_type: 'image/png', + filename: 'test.png', + s3_key: 'attachments/u/t/a/test.png', + s3_version_id: 'v1', + size_bytes: 100, + checksum_sha256: 'abc', + screening: { status: 'passed', screened_at: '2024-01-01T00:00:00Z' }, + }); + + const result = await resolveUrlAttachments( + [passedAttachment], + 'task-1', + 'user-1', + { s3Client: mockS3Client as any, bucketName: 'bucket', screeningConfig: mockScreeningConfig as any }, + ); + + expect(result).toEqual([passedAttachment]); + expect(dns.resolve4).not.toHaveBeenCalled(); + }); + + test('rejects HTTP (non-HTTPS) URLs', async () => { + const urlAttachment = createAttachmentRecord({ + attachment_id: 'att-1', + type: 'url', + content_type: 'application/octet-stream', + filename: 'file.txt', + screening: { status: 'pending' }, + source_url: 'http://example.com/file.txt', + }); + + await expect(resolveUrlAttachments( + [urlAttachment], + 'task-1', + 'user-1', + { s3Client: mockS3Client as any, bucketName: 'bucket', screeningConfig: mockScreeningConfig as any }, + )).rejects.toThrow('URL attachment must use HTTPS'); + }); + + test('rejects URLs resolving to private IPs', async () => { + dns.resolve4.mockResolvedValue(['10.0.0.1']); + + const urlAttachment = createAttachmentRecord({ + attachment_id: 'att-1', + type: 'url', + content_type: 'application/octet-stream', + filename: 'file.txt', + screening: { status: 'pending' }, + source_url: 'https://evil.example.com/file.txt', + }); + + await expect(resolveUrlAttachments( + [urlAttachment], + 'task-1', + 'user-1', + { s3Client: mockS3Client as any, bucketName: 'bucket', screeningConfig: mockScreeningConfig as any }, + )).rejects.toThrow('private'); + }); + + test('rejects URLs when DNS returns no addresses', async () => { + dns.resolve4.mockResolvedValue([]); + + const urlAttachment = createAttachmentRecord({ + attachment_id: 'att-1', + type: 'url', + content_type: 'application/octet-stream', + filename: 'file.txt', + screening: { status: 'pending' }, + source_url: 'https://no-records.example.com/file.txt', + }); + + await expect(resolveUrlAttachments( + [urlAttachment], + 'task-1', + 'user-1', + { s3Client: mockS3Client as any, bucketName: 'bucket', screeningConfig: mockScreeningConfig as any }, + )).rejects.toThrow('DNS resolution returned no addresses'); + }); + + test('rejects URLs with unsupported content type', async () => { + isAllowedMimeType.mockReturnValue(false); + + // We need to mock the actual HTTP request here, but since pinnedHttpsRequest + // uses native https module which is hard to mock cleanly, we verify the + // validation-level protection via isAllowedMimeType + const urlAttachment = createAttachmentRecord({ + attachment_id: 'att-1', + type: 'url', + content_type: 'application/octet-stream', + filename: 'file.exe', + screening: { status: 'pending' }, + source_url: 'https://example.com/file.exe', + }); + + // DNS resolves to a private IP so we don't need to actually fetch + dns.resolve4.mockResolvedValue(['169.254.169.254']); + + await expect(resolveUrlAttachments( + [urlAttachment], + 'task-1', + 'user-1', + { s3Client: mockS3Client as any, bucketName: 'bucket', screeningConfig: mockScreeningConfig as any }, + )).rejects.toThrow(/private/); + }); + + test('rejects when source_url is missing', async () => { + const urlAttachment = createAttachmentRecord({ + attachment_id: 'att-1', + type: 'url', + content_type: 'application/octet-stream', + filename: 'file.txt', + screening: { status: 'pending' }, + }); + + await expect(resolveUrlAttachments( + [urlAttachment], + 'task-1', + 'user-1', + { s3Client: mockS3Client as any, bucketName: 'bucket', screeningConfig: mockScreeningConfig as any }, + )).rejects.toThrow('no source_url'); + }); +}); diff --git a/scripts/check-types-sync.ts b/scripts/check-types-sync.ts index eba1a748..190f9294 100644 --- a/scripts/check-types-sync.ts +++ b/scripts/check-types-sync.ts @@ -100,6 +100,9 @@ const CDK_ONLY_ALLOWLIST = new Set([ 'ValidatedAttachment', 'ScreeningResult', 'AttachmentRecord', + 'PendingAttachmentRecord', + 'PassedAttachmentRecord', + 'BlockedAttachmentRecord', 'CreateAttachmentRecordParams', 'AgentAttachmentPayload', ]); From 4f410af6a8913d733b864dbfeaf8b4009d75bef6 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 00:56:56 -0500 Subject: [PATCH 14/19] chore(review): update doc --- docs/design/ATTACHMENTS.md | 30 ++++++++----------- .../content/docs/architecture/Attachments.md | 30 ++++++++----------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/docs/design/ATTACHMENTS.md b/docs/design/ATTACHMENTS.md index a2f9d924..6d48cc6e 100644 --- a/docs/design/ATTACHMENTS.md +++ b/docs/design/ATTACHMENTS.md @@ -423,6 +423,8 @@ flowchart TD MB -->|Invalid| R[REJECTED: not a valid image] MB -->|Valid| D[Dimension check: parse PNG IHDR / JPEG SOF] D -->|> 8000px| OV[REJECTED: oversized] + D -->|Unparseable + > 5 MB| FC[REJECTED: fail-closed, dimensions unverifiable] + D -->|Unparseable + <= 5 MB| G[Bedrock Guardrail: rely on Bedrock validation] D -->|OK| G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] G -->|INTERVENED| B[BLOCKED: content policy violation] G -->|NONE| P[PASSED: original bytes stored as-is] @@ -433,7 +435,7 @@ flowchart TD **Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the screening pipeline. -**Dimension checks:** Image dimensions are read from PNG IHDR chunks and JPEG SOF markers using pure buffer parsing (no native dependencies). Images exceeding 8000px on either side are rejected before the Bedrock call. +**Dimension checks:** Image dimensions are read from PNG IHDR chunks and JPEG SOF markers using pure buffer parsing (no native dependencies). Images exceeding 8000px on either side are rejected before the Bedrock call. For PNGs, a missing IHDR chunk is a hard failure (the file is corrupt or incomplete). For JPEGs, if the SOF marker cannot be found: files > 5 MB are rejected (fail-closed — an unparseable large JPEG is too risky to forward without dimension verification); smaller files are allowed through with a logged warning, relying on Bedrock's own validation to reject oversized images. **Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks with `png` and `jpeg` formats. Raw image bytes are passed directly — no re-encoding or format conversion needed. @@ -728,21 +730,13 @@ async function resolveAttachments(attachments, ...) { for (const att of attachments) { if (att.type === 'image') { - // getImageDimensions parses PNG IHDR / JPEG SOF markers from the buffer. - // If dimensions cannot be determined (corrupt image, unsupported format variant), - // throw AttachmentResolutionError — never default to (0,0) or skip the estimate. - let width: number, height: number; - try { - ({ width, height } = await getImageDimensions(att)); - } catch (err) { - throw new AttachmentResolutionError( - `Cannot determine dimensions for image "${att.filename}". ` + - `The image may be corrupt or in an unsupported format variant. ` + - `Re-export the image and try again.`, - { cause: err }, - ); - } - const tokenCost = estimateImageTokens(width, height); + // estimateImageTokensFromBuffer parses PNG IHDR / JPEG SOF markers. + // Returns undefined when dimensions cannot be determined (unusual JPEG + // encoder, corrupt tail). This is non-fatal — use MAX_IMAGE_TOKENS as a + // conservative fallback so budget enforcement still works (overestimates + // rather than underestimates). + const tokenCost = estimateImageTokensFromBuffer(att.content, att.content_type) + ?? MAX_IMAGE_TOKENS; att.token_estimate = tokenCost; attachmentTokenBudget += tokenCost; } @@ -763,7 +757,7 @@ async function resolveAttachments(attachments, ...) { } ``` -**Policy:** If image attachments consume more than `USER_PROMPT_TOKEN_BUDGET - MIN_TEXT_TOKEN_BUDGET` tokens (i.e., they would leave fewer than 20K tokens for text context), the task fails with a clear error. The user can reduce image count or downscale images before resubmitting. +**Policy:** If image attachments consume more than `USER_PROMPT_TOKEN_BUDGET - MIN_TEXT_TOKEN_BUDGET` tokens (i.e., they would leave fewer than 20K tokens for text context), the task fails with a clear error. The user can reduce image count or downscale images before resubmitting. When dimensions are unparseable, `MAX_IMAGE_TOKENS` (1568) is used as a conservative budget estimate — this may slightly overcount, but ensures the budget check never underestimates token cost due to parsing limitations. **Token budget vs. payload size:** The token budget above measures **vision tokens** (based on pixel dimensions). This is separate from the **API payload size**, which is affected by base64 encoding overhead (~33% expansion). Image attachments are sent as multimodal content blocks with base64-encoded data, so a 10 MB image becomes ~13.3 MB in the API request. The Anthropic API has its own request size limits (separate from our Lambda payload limits). The `MAX_ATTACHMENT_SIZE_BYTES` (10 MB) is chosen to ensure that even after base64 expansion, individual images stay within the Anthropic API's per-image limits. For multiple large images, the total base64-encoded payload is bounded by the 50 MB total task limit (which produces ~67 MB base64), but in practice the vision token budget is the binding constraint — 10 full-resolution images would consume ~18,820 vision tokens (well within the 100K budget) but produce a very large API payload. The agent should stream images from local files rather than holding all base64 data in memory simultaneously. @@ -1615,7 +1609,7 @@ The implementation is ordered to deliver value incrementally while maintaining s 34. Add `AttachmentConfig` and `PreparedAttachment` Pydantic models to agent `models.py` (with validators, `s3_version_id` required, `checksum_sha256` required as lowercase hex) 35. Add attachment download from S3 with pinned `VersionId` (via IAM role) and mandatory SHA-256 integrity verification 36. Add multimodal content blocks for image attachments in agent prompt -37. Add token budget accounting with resize-aware formula matching Anthropic docs (1568px cap, 28px tile padding, 1568 token cap, 1.2x safety margin), with explicit error path for `getImageDimensions` failures +37. Add token budget accounting with resize-aware formula matching Anthropic docs (1568px cap, 28px tile padding, 1568 token cap, 1.2x safety margin), with conservative `MAX_IMAGE_TOKENS` fallback when dimensions are unparseable (non-fatal — avoids rejecting valid images with unusual JPEG encoders) 38. Add `AttachmentBudgetExceededError` (extends `AttachmentError` base class — already caught by hydration re-throw list from Phase 1 step 10) 39. Add agent capability check in orchestrator (fail task if agent doesn't support attachments) 40. Add parity test: `AgentAttachmentPayload` fields match `AttachmentConfig` fields (including `s3_version_id` and `checksum_sha256`) diff --git a/docs/src/content/docs/architecture/Attachments.md b/docs/src/content/docs/architecture/Attachments.md index b5a792eb..1a1f7e68 100644 --- a/docs/src/content/docs/architecture/Attachments.md +++ b/docs/src/content/docs/architecture/Attachments.md @@ -427,6 +427,8 @@ flowchart TD MB -->|Invalid| R[REJECTED: not a valid image] MB -->|Valid| D[Dimension check: parse PNG IHDR / JPEG SOF] D -->|> 8000px| OV[REJECTED: oversized] + D -->|Unparseable + > 5 MB| FC[REJECTED: fail-closed, dimensions unverifiable] + D -->|Unparseable + <= 5 MB| G[Bedrock Guardrail: rely on Bedrock validation] D -->|OK| G[Bedrock Guardrail: ApplyGuardrail with image content block, retries] G -->|INTERVENED| B[BLOCKED: content policy violation] G -->|NONE| P[PASSED: original bytes stored as-is] @@ -437,7 +439,7 @@ flowchart TD **Magic bytes validation:** Verify the first bytes against known image signatures before any further processing. A file claiming to be `image/png` must start with `\x89PNG\r\n\x1a\n`. This prevents polyglot files (e.g., an image header followed by executable code) from reaching the screening pipeline. -**Dimension checks:** Image dimensions are read from PNG IHDR chunks and JPEG SOF markers using pure buffer parsing (no native dependencies). Images exceeding 8000px on either side are rejected before the Bedrock call. +**Dimension checks:** Image dimensions are read from PNG IHDR chunks and JPEG SOF markers using pure buffer parsing (no native dependencies). Images exceeding 8000px on either side are rejected before the Bedrock call. For PNGs, a missing IHDR chunk is a hard failure (the file is corrupt or incomplete). For JPEGs, if the SOF marker cannot be found: files > 5 MB are rejected (fail-closed — an unparseable large JPEG is too risky to forward without dimension verification); smaller files are allowed through with a logged warning, relying on Bedrock's own validation to reject oversized images. **Bedrock image screening:** The `ApplyGuardrailCommand` supports `image` content blocks with `png` and `jpeg` formats. Raw image bytes are passed directly — no re-encoding or format conversion needed. @@ -732,21 +734,13 @@ async function resolveAttachments(attachments, ...) { for (const att of attachments) { if (att.type === 'image') { - // getImageDimensions parses PNG IHDR / JPEG SOF markers from the buffer. - // If dimensions cannot be determined (corrupt image, unsupported format variant), - // throw AttachmentResolutionError — never default to (0,0) or skip the estimate. - let width: number, height: number; - try { - ({ width, height } = await getImageDimensions(att)); - } catch (err) { - throw new AttachmentResolutionError( - `Cannot determine dimensions for image "${att.filename}". ` + - `The image may be corrupt or in an unsupported format variant. ` + - `Re-export the image and try again.`, - { cause: err }, - ); - } - const tokenCost = estimateImageTokens(width, height); + // estimateImageTokensFromBuffer parses PNG IHDR / JPEG SOF markers. + // Returns undefined when dimensions cannot be determined (unusual JPEG + // encoder, corrupt tail). This is non-fatal — use MAX_IMAGE_TOKENS as a + // conservative fallback so budget enforcement still works (overestimates + // rather than underestimates). + const tokenCost = estimateImageTokensFromBuffer(att.content, att.content_type) + ?? MAX_IMAGE_TOKENS; att.token_estimate = tokenCost; attachmentTokenBudget += tokenCost; } @@ -767,7 +761,7 @@ async function resolveAttachments(attachments, ...) { } ``` -**Policy:** If image attachments consume more than `USER_PROMPT_TOKEN_BUDGET - MIN_TEXT_TOKEN_BUDGET` tokens (i.e., they would leave fewer than 20K tokens for text context), the task fails with a clear error. The user can reduce image count or downscale images before resubmitting. +**Policy:** If image attachments consume more than `USER_PROMPT_TOKEN_BUDGET - MIN_TEXT_TOKEN_BUDGET` tokens (i.e., they would leave fewer than 20K tokens for text context), the task fails with a clear error. The user can reduce image count or downscale images before resubmitting. When dimensions are unparseable, `MAX_IMAGE_TOKENS` (1568) is used as a conservative budget estimate — this may slightly overcount, but ensures the budget check never underestimates token cost due to parsing limitations. **Token budget vs. payload size:** The token budget above measures **vision tokens** (based on pixel dimensions). This is separate from the **API payload size**, which is affected by base64 encoding overhead (~33% expansion). Image attachments are sent as multimodal content blocks with base64-encoded data, so a 10 MB image becomes ~13.3 MB in the API request. The Anthropic API has its own request size limits (separate from our Lambda payload limits). The `MAX_ATTACHMENT_SIZE_BYTES` (10 MB) is chosen to ensure that even after base64 expansion, individual images stay within the Anthropic API's per-image limits. For multiple large images, the total base64-encoded payload is bounded by the 50 MB total task limit (which produces ~67 MB base64), but in practice the vision token budget is the binding constraint — 10 full-resolution images would consume ~18,820 vision tokens (well within the 100K budget) but produce a very large API payload. The agent should stream images from local files rather than holding all base64 data in memory simultaneously. @@ -1619,7 +1613,7 @@ The implementation is ordered to deliver value incrementally while maintaining s 34. Add `AttachmentConfig` and `PreparedAttachment` Pydantic models to agent `models.py` (with validators, `s3_version_id` required, `checksum_sha256` required as lowercase hex) 35. Add attachment download from S3 with pinned `VersionId` (via IAM role) and mandatory SHA-256 integrity verification 36. Add multimodal content blocks for image attachments in agent prompt -37. Add token budget accounting with resize-aware formula matching Anthropic docs (1568px cap, 28px tile padding, 1568 token cap, 1.2x safety margin), with explicit error path for `getImageDimensions` failures +37. Add token budget accounting with resize-aware formula matching Anthropic docs (1568px cap, 28px tile padding, 1568 token cap, 1.2x safety margin), with conservative `MAX_IMAGE_TOKENS` fallback when dimensions are unparseable (non-fatal — avoids rejecting valid images with unusual JPEG encoders) 38. Add `AttachmentBudgetExceededError` (extends `AttachmentError` base class — already caught by hydration re-throw list from Phase 1 step 10) 39. Add agent capability check in orchestrator (fail task if agent doesn't support attachments) 40. Add parity test: `AgentAttachmentPayload` fields match `AttachmentConfig` fields (including `s3_version_id` and `checksum_sha256`) From ea8ad44372467d1023ed8999e8a619b562bddc15 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 01:04:05 -0500 Subject: [PATCH 15/19] chore(review): update package and variable --- cdk/package.json | 2 +- cdk/src/handlers/confirm-uploads.ts | 14 +- yarn.lock | 354 +++++++++++++++++++++++++++- 3 files changed, 348 insertions(+), 22 deletions(-) diff --git a/cdk/package.json b/cdk/package.json index 4a450795..48668607 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -25,7 +25,7 @@ "@aws-sdk/client-s3": "^3.1021.0", "@aws-sdk/client-secrets-manager": "^3.1021.0", "@aws-sdk/lib-dynamodb": "^3.1021.0", - "@aws-sdk/s3-presigned-post": "3.1040.0", + "@aws-sdk/s3-presigned-post": "^3.1021.0", "@aws-sdk/s3-request-presigner": "^3.1021.0", "@aws/durable-execution-sdk-js": "^1.1.0", "@cedar-policy/cedar-wasm": "4.10.0", diff --git a/cdk/src/handlers/confirm-uploads.ts b/cdk/src/handlers/confirm-uploads.ts index a1042bef..777eae05 100644 --- a/cdk/src/handlers/confirm-uploads.ts +++ b/cdk/src/handlers/confirm-uploads.ts @@ -170,13 +170,11 @@ export async function handler(event: APIGatewayProxyEvent, context: Context): Pr const screenedAttachments: AttachmentRecord[] = []; - // Internal deadline timer (design §ConfirmUploadsFunction): abort screening - // before the Lambda times out so we can return a graceful 503 + Retry-After - // instead of an opaque timeout error. On retry, already-screened attachments - // (status === 'passed' in DDB) are skipped, so retries make forward progress. - const deadlineMs = context.getRemainingTimeInMillis() - DEADLINE_MARGIN_MS; - - // Process in batches of SCREENING_CONCURRENCY + // Process in batches of SCREENING_CONCURRENCY. + // Deadline check: abort screening before the Lambda times out so we can + // return a graceful 503 + Retry-After instead of an opaque timeout error. + // On retry, already-screened attachments (status === 'passed' in DDB) are + // skipped, so retries make forward progress. for (let i = 0; i < pendingAttachments.length; i += SCREENING_CONCURRENCY) { // Deadline check before starting a new batch if (context.getRemainingTimeInMillis() <= DEADLINE_MARGIN_MS) { @@ -186,7 +184,7 @@ export async function handler(event: APIGatewayProxyEvent, context: Context): Pr task_id: taskId, screened, remaining, - deadline_ms: deadlineMs, + remaining_ms: context.getRemainingTimeInMillis(), request_id: requestId, metric_type: 'confirm_uploads_deadline_exceeded', }); diff --git a/yarn.lock b/yarn.lock index cc406036..fb211a56 100644 --- a/yarn.lock +++ b/yarn.lock @@ -673,7 +673,31 @@ "@smithy/util-waiter" "^4.2.14" tslib "^2.6.2" -"@aws-sdk/client-s3@3.1040.0", "@aws-sdk/client-s3@^3.1021.0": +"@aws-sdk/client-s3@3.1053.0": + version "3.1053.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.1053.0.tgz#13568ec27b695d01c56c11c3c11afb6d3d96496a" + integrity sha512-/oGxoB6p1Nqs935Blt+v1o+anSCEf2n3RjIrcLz84i4cn2Gr+Z7JpDdUkG5+74r5ctqEPG7k/phTGbJ9fNKnHg== + dependencies: + "@aws-crypto/sha1-browser" "5.2.0" + "@aws-crypto/sha256-browser" "5.2.0" + "@aws-crypto/sha256-js" "5.2.0" + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/credential-provider-node" "^3.972.44" + "@aws-sdk/middleware-bucket-endpoint" "^3.972.15" + "@aws-sdk/middleware-expect-continue" "^3.972.13" + "@aws-sdk/middleware-flexible-checksums" "^3.974.21" + "@aws-sdk/middleware-location-constraint" "^3.972.11" + "@aws-sdk/middleware-sdk-s3" "^3.972.42" + "@aws-sdk/middleware-ssec" "^3.972.11" + "@aws-sdk/signature-v4-multi-region" "^3.996.28" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/fetch-http-handler" "^5.4.3" + "@smithy/node-http-handler" "^4.7.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + +"@aws-sdk/client-s3@^3.1021.0": version "3.1040.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.1040.0.tgz#96fa3975b815e6cd6d0855a7bd4a72adf7dc1016" integrity sha512-Ldfby1xDrlZwNY2NxP9pwdVrf8sqHbGBKP1UkoG/oWcePGlGhjY8iVwy8hRy9f1EQfHVFWIFunwHaPQxhYTnWQ== @@ -874,6 +898,20 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/core@^3.974.13": + version "3.974.13" + resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.974.13.tgz#a785d4a726590f679671d18b36c69e3fc9b6cab5" + integrity sha512-+Y5/4tHki0uYgyx8eun146DegRVQBpdKGK5RbV0FTKJPpaKTchvqVxrrRFK6Wk0JksO4iAZKw3eqxGEIwtO98w== + dependencies: + "@aws-sdk/types" "^3.973.9" + "@aws-sdk/xml-builder" "^3.972.25" + "@aws/lambda-invoke-store" "^0.2.2" + "@smithy/core" "^3.24.3" + "@smithy/signature-v4" "^5.4.2" + "@smithy/types" "^4.14.2" + bowser "^2.11.0" + tslib "^2.6.2" + "@aws-sdk/core@^3.974.7": version "3.974.7" resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.974.7.tgz#1b78801c86f54947971ead2d4b9913a2b5b7d860" @@ -902,6 +940,14 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/crc64-nvme@^3.972.9": + version "3.972.9" + resolved "https://registry.yarnpkg.com/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.9.tgz#4ea4d574d473e25e59973fcbab101ca1b64fab91" + integrity sha512-P+QGozmXn2mZZI7sDgk+aUm+RTI61MPSFB+Ir2vjEjEbEsE4e7hYtzrDvAUxZy9ko81h53e11+F/GYlvwDkaOQ== + dependencies: + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-env@^3.972.24": version "3.972.24" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.24.tgz#bc33a34f15704d02552aa8b3994d17008b991f86" @@ -946,6 +992,17 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-env@^3.972.39": + version "3.972.39" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.39.tgz#538cc859f2ac0e15b141b9e246613a752849ae8c" + integrity sha512-29wX9zpAvEt1vcj0psha+y6ygBHy2V/S72mp6e7q0KARLWXq+pwE/lR6qGkwknQvruh52lXvlqZIga8Hdxkucw== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-http@^3.972.26": version "3.972.26" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.26.tgz#6524c3681dbb62d3c4de82262631ab94b800f00e" @@ -1007,6 +1064,19 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-http@^3.972.41": + version "3.972.41" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.41.tgz#07037e7346881cb8bb8ec1fe9f8ed0104072b63a" + integrity sha512-IA3CQTjtJkb6u1H4mE4936c8OPBMa9Jggtwe8U2Mqw/vvb/tZ5Ebd0mcZcX0uKWQhOyYo/+qNIwkV5Xh+FeJJA== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/fetch-http-handler" "^5.4.3" + "@smithy/node-http-handler" "^4.7.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-ini@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.28.tgz#6bc0d684c245914dca7a1a4dd3c2d84212833320" @@ -1086,6 +1156,25 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-ini@^3.972.43": + version "3.972.43" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.43.tgz#cb9779beebd45bd242c12ea48a820047c77e1b05" + integrity sha512-4mzII+3mZEVXXE1xzrLQrCJL7/r62A63bA6SVzZoNL5rqCJghpf+xgGltVrIBBs0n+mOZBKrQl2tRREtvZ5l6A== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/credential-provider-env" "^3.972.39" + "@aws-sdk/credential-provider-http" "^3.972.41" + "@aws-sdk/credential-provider-login" "^3.972.43" + "@aws-sdk/credential-provider-process" "^3.972.39" + "@aws-sdk/credential-provider-sso" "^3.972.43" + "@aws-sdk/credential-provider-web-identity" "^3.972.43" + "@aws-sdk/nested-clients" "^3.997.11" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/credential-provider-imds" "^4.3.2" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-login@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.28.tgz#b2d47d4d43690d2d824edc94ce955d86dd3877f1" @@ -1140,6 +1229,18 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-login@^3.972.43": + version "3.972.43" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.43.tgz#2d6dd4a7d082b0c54be9c5c5269161c14f7ae717" + integrity sha512-HG7kQCwXtbv3oBV61Ins0oNX8KKyvrMqqRkb6ZiAfQHbMuHaiNaEb2KnpKLPkNpqImSBK82UkVE/kaY6IfWikA== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/nested-clients" "^3.997.11" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-node@^3.972.29": version "3.972.29" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.29.tgz#4bcc991fcbf245f75494a119b3446a678a51e019" @@ -1211,6 +1312,23 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-node@^3.972.44": + version "3.972.44" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.44.tgz#af009a773d2e20214edfcc98894d3d0779fbc1c3" + integrity sha512-sDaBIT0yrNNIPfvlsiTCmANm07zKju+ipWODjEXgZlsjMeIJR3LVp7RDyAOzUoAsTbDfYKDWp+i5WrFiQP6rmQ== + dependencies: + "@aws-sdk/credential-provider-env" "^3.972.39" + "@aws-sdk/credential-provider-http" "^3.972.41" + "@aws-sdk/credential-provider-ini" "^3.972.43" + "@aws-sdk/credential-provider-process" "^3.972.39" + "@aws-sdk/credential-provider-sso" "^3.972.43" + "@aws-sdk/credential-provider-web-identity" "^3.972.43" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/credential-provider-imds" "^4.3.2" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-process@^3.972.24": version "3.972.24" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.24.tgz#940c76a2db0aece23879dcf75ac5b6ee8f8fa135" @@ -1258,6 +1376,17 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-process@^3.972.39": + version "3.972.39" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.39.tgz#236f8822180b297e0e98771ee69aea428280a4a7" + integrity sha512-2k/amBifLd75eXNwgvPw/2lKYSQ3NhvHQgkVKVjfUq13/eJ3JRtHmznuFenn74OK3sSfp4SMy1YB2w+UVXoKqA== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-sso@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.28.tgz#bf150bfb7e708d58f35bb2b5786b902df19fd92d" @@ -1313,6 +1442,19 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-sso@^3.972.43": + version "3.972.43" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.43.tgz#24546d197ce74a29c89c37b3860952ee28f90c9b" + integrity sha512-LPc3+Y4vhH1T4x6CMqwCM6hk5+SRf/Lwmgm8INm95wxTtIRHcMwQUVkDzWu4Iw/RSncxYM2BC01OrYbxOPZvyg== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/nested-clients" "^3.997.11" + "@aws-sdk/token-providers" "3.1052.0" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/credential-provider-web-identity@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.28.tgz#27fc2a0fe0d2ff1460171d2a6912898c2235a7df" @@ -1364,6 +1506,18 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-web-identity@^3.972.43": + version "3.972.43" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.43.tgz#1d4e99a4cba32c63bab21184f8d5d418c0744224" + integrity sha512-wQtL34lUD/09VXjwAUo2T+I3aEXRDxMB3DKmTJL/Zj0Gi6sLDTrVhae1XVt01yzkquOWajI/sZW72JGDZ1ciTw== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/nested-clients" "^3.997.11" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/dynamodb-codec@^3.972.27": version "3.972.27" resolved "https://registry.yarnpkg.com/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.972.27.tgz#3d29a2f00bbc145260419878a5f3640af81d36b3" @@ -1431,6 +1585,17 @@ "@smithy/util-config-provider" "^4.2.2" tslib "^2.6.2" +"@aws-sdk/middleware-bucket-endpoint@^3.972.15": + version "3.972.15" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.15.tgz#f1752b8107289df1313b647bf42e8e5f78f44192" + integrity sha512-O2HDANa+MrvbxpaRVQDiH3T13uAa9AkMjKyZmDygwauAmmvqZ5B0iRmKW+fuVGW6NPXuyXurFgIx69lSvmAWGA== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/middleware-endpoint-discovery@^3.972.9": version "3.972.9" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.9.tgz#664f9074b0017255680c200bd9b8b23a864c0ad5" @@ -1463,6 +1628,16 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/middleware-expect-continue@^3.972.13": + version "3.972.13" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.13.tgz#d6eac0372151e7aa978985ceb67311ab77b03939" + integrity sha512-sHiqIFg8o2ipT7t40B89Vj0ubSUtY6OSt/+Ee/OXhHch5K4+81zP2+QX8Lkc/nJ2QSmCySxOke7TEbmX69fe2g== + dependencies: + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/middleware-flexible-checksums@^3.974.15": version "3.974.15" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.15.tgz#11e688424dd08fae175d08597dd2a7edeaa4773a" @@ -1483,6 +1658,21 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@aws-sdk/middleware-flexible-checksums@^3.974.21": + version "3.974.21" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.21.tgz#efa1acea9921691f8fe80160ebaa6514b2e0839c" + integrity sha512-alAu9heyiBK/OmRNXVxq8mmPTgeW2AQ6EYjRsI38kPZa1MZvt2Jh+BlGq7/GG9OVXOaEgD7DlGj/Lzfy5OmuEg== + dependencies: + "@aws-crypto/crc32" "5.2.0" + "@aws-crypto/crc32c" "5.2.0" + "@aws-crypto/util" "5.2.0" + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/crc64-nvme" "^3.972.9" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/middleware-host-header@^3.972.10": version "3.972.10" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.10.tgz#e63b91959ce46948d789582351b2a44c4876e924" @@ -1532,6 +1722,15 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/middleware-location-constraint@^3.972.11": + version "3.972.11" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.11.tgz#272507843738acd4a5644842911a2016f7dfb0e1" + integrity sha512-hkfspNUP4criAH6ton6BGKgnm5dZx+7bUOy1YqlTfejDeUPAM23D81q/IX+hdlS3KUsfwGz5ADTqZWKBEUpf4A== + dependencies: + "@aws-sdk/types" "^3.973.9" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/middleware-logger@^3.972.10": version "3.972.10" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-logger/-/middleware-logger-3.972.10.tgz#d92b3374dcaddd523930bdff441207946343c270" @@ -1623,6 +1822,19 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@aws-sdk/middleware-sdk-s3@^3.972.42": + version "3.972.42" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.42.tgz#f4217eea10d2de43b482f6f2d0d9895be061e571" + integrity sha512-/xNqNGXv9LaxZd25L9VV4pnSOw9OdDNO4rAHamM+h3KQBSITljIH9vk3dveGga1I2j36lQd0rdG3gjNEXvtNew== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/signature-v4-multi-region" "^3.996.28" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/signature-v4" "^5.4.2" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/middleware-ssec@^3.972.10": version "3.972.10" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.10.tgz#46b5c030c0116f51110e18042ad3cf863ab5c81c" @@ -1632,6 +1844,15 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/middleware-ssec@^3.972.11": + version "3.972.11" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.11.tgz#b5d5ddde7d54239137949f63b3d5dee6331628ea" + integrity sha512-7PQvGNhtveKlvVqNahqWx5yrwxP7ecwAoB1dYBf8eKwfo2tzzCbNnW+q2nO3N066ktQaB4iBQbDRWtizm+amoQ== + dependencies: + "@aws-sdk/types" "^3.973.9" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/middleware-user-agent@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.28.tgz#7f81d96d2fed0334ff601af62d77e14f67fb9d22" @@ -1792,6 +2013,22 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@aws-sdk/nested-clients@^3.997.11": + version "3.997.11" + resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.11.tgz#ed97d5dadc5ee15a31834e8af218e502d986d632" + integrity sha512-nWXXJ1r/r8N2Gw1pWolRgED38/A9A8DHR2ETWIv220zh4PZHcybbR4hUVWWktmNXTRHzDJwRluapHn0rZxuoqA== + dependencies: + "@aws-crypto/sha256-browser" "5.2.0" + "@aws-crypto/sha256-js" "5.2.0" + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/signature-v4-multi-region" "^3.996.28" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/fetch-http-handler" "^5.4.3" + "@smithy/node-http-handler" "^4.7.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/nested-clients@^3.997.5": version "3.997.5" resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.5.tgz#0b66825b14b1a06b43b71e95354f22cb6b4926df" @@ -1904,19 +2141,17 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/s3-presigned-post@3.1040.0": - version "3.1040.0" - resolved "https://registry.yarnpkg.com/@aws-sdk/s3-presigned-post/-/s3-presigned-post-3.1040.0.tgz#3b03fb5ffa5b5476780ffd04337cdb938af20445" - integrity sha512-Ycnktl5kAqgM7ottZIujR+/MsmmZ/iiaKobQXjtZhy/QO9nwnChxHJuOruuuhI5+f/NMbhb0G8aeUexK/D6iQA== +"@aws-sdk/s3-presigned-post@^3.1021.0": + version "3.1053.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/s3-presigned-post/-/s3-presigned-post-3.1053.0.tgz#81e976fe671b67fb86f48c9b5cafa421010207b1" + integrity sha512-7BAOxPfFd9HZLtamgffpqSqACpQIuEwiTWAHbr1j3Ddr0mwWgiuk5uWAAbZU1PduPlpuwtJKcT7Zc1aGuRkUyA== dependencies: - "@aws-sdk/client-s3" "3.1040.0" - "@aws-sdk/types" "^3.973.8" - "@aws-sdk/util-format-url" "^3.972.10" - "@smithy/middleware-endpoint" "^4.4.32" - "@smithy/signature-v4" "^5.3.14" - "@smithy/types" "^4.14.1" - "@smithy/util-hex-encoding" "^4.2.2" - "@smithy/util-utf8" "^4.2.2" + "@aws-sdk/client-s3" "3.1053.0" + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/signature-v4" "^5.4.2" + "@smithy/types" "^4.14.2" tslib "^2.6.2" "@aws-sdk/s3-request-presigner@^3.1021.0": @@ -1956,6 +2191,17 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/signature-v4-multi-region@^3.996.28": + version "3.996.28" + resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.28.tgz#79c12506d5545953c06fe75956b38050d57902f2" + integrity sha512-qs9z5LqXO/CZC2Lg9SGKpoLU8Rhi+m2pFKZqfO9pytX1clc0katqtsDNupJxFy0xT9wsZSPzM2v1y+/H/zfp5Q== + dependencies: + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/signature-v4" "^5.4.2" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/token-providers@3.1021.0": version "3.1021.0" resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1021.0.tgz#90905a8def49f90e54a73849e25ad4bcc4dbea2a" @@ -2007,6 +2253,18 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/token-providers@3.1052.0": + version "3.1052.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1052.0.tgz#0793c2f58351bf91937e8f83abf39d11937ec8f2" + integrity sha512-QqZNB3so7UIDxZtroc85TQaLVxdZRFm0eWM1CSR2N+b06as9TOrilvrlTZuj3guYlxMs6yLOgGxnklJ5qMYtTw== + dependencies: + "@aws-sdk/core" "^3.974.13" + "@aws-sdk/nested-clients" "^3.997.11" + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.3" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/types@^3.222.0", "@aws-sdk/types@^3.973.6": version "3.973.6" resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.973.6.tgz#1964a7c01b5cb18befa445998ad1d02f86c5432d" @@ -2031,6 +2289,14 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@aws-sdk/types@^3.973.9": + version "3.973.9" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.973.9.tgz#7d1c08cc6e82ec2ac2f2da102a7dd55806592f7f" + integrity sha512-kuBfgQVdcz5Bmapc4A13YbpVw/pXkesfhetcFYwbntqas8sF41OHyd4o28+/TG2ZQdHBsv90Lsu5y6oitvYCdg== + dependencies: + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/util-arn-parser@^3.972.3": version "3.972.3" resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.3.tgz#ed989862bbb172ce16d9e1cd5790e5fe367219c2" @@ -2240,6 +2506,16 @@ fast-xml-parser "5.7.3" tslib "^2.6.2" +"@aws-sdk/xml-builder@^3.972.25": + version "3.972.25" + resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.972.25.tgz#252ed0afef165a247c2dcc5d72e54b8f9e45f2e2" + integrity sha512-GH+Kjz4nPKWKHnsiQpnhP1MJdTGIcK4rAka6tzakgjjUkVgNsmPeEbbRAf09SzS1hjGu6duGHCBsxYke0BhHjQ== + dependencies: + "@nodable/entities" "2.1.0" + "@smithy/types" "^4.14.2" + fast-xml-parser "5.7.3" + tslib "^2.6.2" + "@aws/durable-execution-sdk-js@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@aws/durable-execution-sdk-js/-/durable-execution-sdk-js-1.1.0.tgz#c32a4a358cc5940414accc13cd9825766299898d" @@ -3842,6 +4118,15 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@smithy/core@^3.24.3", "@smithy/core@^3.24.4": + version "3.24.4" + resolved "https://registry.yarnpkg.com/@smithy/core/-/core-3.24.4.tgz#aded2ba46962b5cceaaa75f646433ac4813c2e17" + integrity sha512-3UNRKEyQyAgVgM0LGlerCLm+ChZWZ1GPfde+jBEW6bm6bSBGU1p0EbblaUV3unbhwvidjLA5Zs3sOs7mnZwvAw== + dependencies: + "@aws-crypto/crc32" "5.2.0" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@smithy/credential-provider-imds@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.12.tgz#fa2e52116cac7eaf5625e0bfd399a4927b598f66" @@ -3884,6 +4169,15 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@smithy/credential-provider-imds@^4.3.2": + version "4.3.4" + resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.3.4.tgz#0ab80322b380902d404682ad8adbbcf021c657c3" + integrity sha512-vKW0MEFRU4Y3MkVZUkpJm+g9qyPGLCXhc0YLggUdSdBB4g7IaSSsCE75P9rBXyWHrXY1UYSQUl8/DwsTR7QciA== + dependencies: + "@smithy/core" "^3.24.4" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@smithy/eventstream-codec@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-4.2.12.tgz#8cd62d08709344fb8b35fd17870fdf1435de61a3" @@ -4016,6 +4310,15 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@smithy/fetch-http-handler@^5.4.3": + version "5.4.4" + resolved "https://registry.yarnpkg.com/@smithy/fetch-http-handler/-/fetch-http-handler-5.4.4.tgz#df28cfdbdbd192cef9508347b488d8874d0166dd" + integrity sha512-qM7AUKI4G6d7lNgaZD3lA1tWSolh5r6gcixfTZAPstVURfjIbvreVTPz+994M0yC3HbX4YYhDRgr31Xy3XwWOQ== + dependencies: + "@smithy/core" "^3.24.4" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@smithy/hash-blob-browser@^4.2.15": version "4.2.15" resolved "https://registry.yarnpkg.com/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.15.tgz#1323f9717cad352b3e18065b738387bb9684f993" @@ -4351,6 +4654,15 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@smithy/node-http-handler@^4.7.3": + version "4.7.4" + resolved "https://registry.yarnpkg.com/@smithy/node-http-handler/-/node-http-handler-4.7.4.tgz#dfa9634130841cbb0a780c8b4a3ea7ec1c904f0c" + integrity sha512-HIeF+1vrDGzPkkv39Hj2vlHSXHY3p958jd/8ZnePIY6+ZOsQX8coyEUKO5yQu4r0bQIVsbpotVIrXXwyycMStQ== + dependencies: + "@smithy/core" "^3.24.4" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@smithy/property-provider@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/property-provider/-/property-provider-4.2.12.tgz#e9f8e5ce125413973b16e39c87cf4acd41324e21" @@ -4546,6 +4858,15 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" +"@smithy/signature-v4@^5.4.2": + version "5.4.4" + resolved "https://registry.yarnpkg.com/@smithy/signature-v4/-/signature-v4-5.4.4.tgz#8302828623453d84e41210dda99f18753bb3da7e" + integrity sha512-e5UtkMvsatzBfbeBZjEOt0k0Z3BEsjTFL/n6fdO5vtBLe67tdy0dX7xw2DU7uZ3acwoHyeCqpU2Fzb7pxwHb6Q== + dependencies: + "@smithy/core" "^3.24.4" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@smithy/smithy-client@^4.12.13": version "4.12.13" resolved "https://registry.yarnpkg.com/@smithy/smithy-client/-/smithy-client-4.12.13.tgz#dec184a1d2d5027370ae1582bddbdbc068c97da5" @@ -4606,6 +4927,13 @@ dependencies: tslib "^2.6.2" +"@smithy/types@^4.14.2": + version "4.14.2" + resolved "https://registry.yarnpkg.com/@smithy/types/-/types-4.14.2.tgz#6034ff1e0e52bfb7d744ac371b651a8bf21f30f1" + integrity sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw== + dependencies: + tslib "^2.6.2" + "@smithy/url-parser@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/url-parser/-/url-parser-4.2.12.tgz#e940557bf0b8e9a25538a421970f64bd827f456f" From 2e34e2d959e0b8e453e6e1ed8067bb7717257a87 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 01:10:40 -0500 Subject: [PATCH 16/19] chore(deps): update deps --- cdk/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdk/package.json b/cdk/package.json index 48668607..b41df6e3 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -61,7 +61,8 @@ "typescript": "^5.9.3" }, "resolutions": { - "eslint-plugin-import/minimatch": "^3.1.2" + "eslint-plugin-import/minimatch": "^3.1.2", + "@aws-sdk/s3-presigned-post/@aws-sdk/client-s3": "^3.1021.0" }, "optionalDependencies": {}, "engines": { From 0aae3a6fa2ed66cb47be9597f5bd81d00869d34d Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 01:15:51 -0500 Subject: [PATCH 17/19] chore(review): update package and variable --- cdk/package.json | 3 +-- cdk/src/handlers/shared/create-task-core.ts | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cdk/package.json b/cdk/package.json index b41df6e3..48668607 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -61,8 +61,7 @@ "typescript": "^5.9.3" }, "resolutions": { - "eslint-plugin-import/minimatch": "^3.1.2", - "@aws-sdk/s3-presigned-post/@aws-sdk/client-s3": "^3.1021.0" + "eslint-plugin-import/minimatch": "^3.1.2" }, "optionalDependencies": {}, "engines": { diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index c4037531..cba7c019 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -739,7 +739,10 @@ async function generateUploadInstructions( if (!record) continue; const s3Key = `${ATTACHMENT_OBJECT_KEY_PREFIX}${userId}/${taskId}/${record.attachment_id}/${att.filename}`; - const { url, fields } = await createPresignedPost(client, { + // Type assertion: @aws-sdk/s3-presigned-post bundles a nested @aws-sdk/client-s3 + // with divergent @smithy/types declarations. The runtime is compatible; only the + // type declarations conflict. Cast to `any` at the boundary. + const { url, fields } = await createPresignedPost(client as any, { Bucket: ATTACHMENTS_BUCKET!, Key: s3Key, Conditions: [ From a9d4c6f939d98d86d1a6c0bf507b776d01d577b4 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 13:24:58 -0500 Subject: [PATCH 18/19] fix(pr): review comments --- cdk/src/handlers/cleanup-pending-uploads.ts | 14 +- cdk/src/handlers/confirm-uploads.ts | 32 ++-- .../handlers/cleanup-pending-uploads.test.ts | 57 ++++++ cdk/test/handlers/confirm-uploads.test.ts | 169 ++++++++++++++++++ 4 files changed, 250 insertions(+), 22 deletions(-) diff --git a/cdk/src/handlers/cleanup-pending-uploads.ts b/cdk/src/handlers/cleanup-pending-uploads.ts index adcc7477..d5a4c055 100644 --- a/cdk/src/handlers/cleanup-pending-uploads.ts +++ b/cdk/src/handlers/cleanup-pending-uploads.ts @@ -190,8 +190,9 @@ async function cleanupTaskAttachments(task: ExpiredTask): Promise { let keyMarker: string | undefined; let versionIdMarker: string | undefined; let totalDeleted = 0; + let isTruncated = true; - do { + while (isTruncated) { const listResp = await s3.send(new ListObjectVersionsCommand({ Bucket: ATTACHMENTS_BUCKET, Prefix: prefix, @@ -199,13 +200,17 @@ async function cleanupTaskAttachments(task: ExpiredTask): Promise { VersionIdMarker: versionIdMarker, })); + isTruncated = listResp.IsTruncated ?? false; + keyMarker = listResp.NextKeyMarker; + versionIdMarker = listResp.NextVersionIdMarker; + // Collect all versions and delete markers for deletion const objects = [ ...(listResp.Versions ?? []).map(v => ({ Key: v.Key!, VersionId: v.VersionId })), ...(listResp.DeleteMarkers ?? []).map(d => ({ Key: d.Key!, VersionId: d.VersionId })), ].filter(obj => obj.Key !== undefined); - if (objects.length === 0) break; + if (objects.length === 0) continue; const deleteResp = await s3.send(new DeleteObjectsCommand({ Bucket: ATTACHMENTS_BUCKET, @@ -220,10 +225,7 @@ async function cleanupTaskAttachments(task: ExpiredTask): Promise { } totalDeleted += objects.length - (deleteResp.Errors?.length ?? 0); - - keyMarker = listResp.NextKeyMarker; - versionIdMarker = listResp.NextVersionIdMarker; - } while (keyMarker); + } if (totalDeleted > 0) { logger.info('Cleaned up S3 objects for expired pending-upload task', { diff --git a/cdk/src/handlers/confirm-uploads.ts b/cdk/src/handlers/confirm-uploads.ts index 777eae05..2457a4d3 100644 --- a/cdk/src/handlers/confirm-uploads.ts +++ b/cdk/src/handlers/confirm-uploads.ts @@ -23,7 +23,7 @@ import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; -import { DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { DynamoDBDocumentClient, GetCommand, PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; import type { APIGatewayProxyEvent, APIGatewayProxyResult, Context } from 'aws-lambda'; import { ulid } from 'ulid'; @@ -230,8 +230,10 @@ export async function handler(event: APIGatewayProxyEvent, context: Context): Pr request_id: requestId, }); // Fail the entire task - await failTaskOnScreening(task, taskId, att.filename, err.message, requestId); - await cleanupAllAttachments(task, taskId); + const transitioned = await failTaskOnScreening(task, taskId, att.filename, err.message, requestId); + if (transitioned) { + await cleanupAllAttachments(task, taskId); + } return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, `Attachment '${att.filename}' was rejected: ${err.message}`, requestId); } @@ -260,8 +262,10 @@ export async function handler(event: APIGatewayProxyEvent, context: Context): Pr const categories = blockedAtt.screening.status === 'blocked' ? blockedAtt.screening.categories.join(', ') : 'content_policy_violation'; - await failTaskOnScreening(task, taskId, blockedAtt.filename, categories, requestId); - await cleanupAllAttachments(task, taskId); + const transitioned = await failTaskOnScreening(task, taskId, blockedAtt.filename, categories, requestId); + if (transitioned) { + await cleanupAllAttachments(task, taskId); + } return errorResponse(400, ErrorCode.ATTACHMENT_BLOCKED, `Attachment '${blockedAtt.filename}' was blocked by content policy (${categories}).`, requestId); } @@ -342,13 +346,7 @@ async function screenSingleAttachment( }); } - // Screening passed — re-upload cleaned content (EXIF-stripped for images) - const putResult = await s3Client.send(new PutObjectCommand({ - Bucket: ATTACHMENTS_BUCKET, - Key: s3Key, - Body: screenResult.content, - ContentType: att.content_type, - })); + // Screening passed — reuse existing S3 object (no transformation needed) // Estimate token cost for images (using shared utility) let tokenEstimate: number | undefined; @@ -362,8 +360,8 @@ async function screenSingleAttachment( content_type: att.content_type, filename: att.filename, s3_key: s3Key, - s3_version_id: putResult.VersionId ?? 'unversioned', - size_bytes: screenResult.content.length, + s3_version_id: versionId ?? 'unversioned', + size_bytes: sizeBytes, screening: { status: 'passed', screened_at: new Date().toISOString() }, checksum_sha256: screenResult.checksum, ...(tokenEstimate !== undefined && { token_estimate: tokenEstimate }), @@ -537,7 +535,7 @@ async function failTaskOnScreening( filename: string, reason: string, requestId: string, -): Promise { +): Promise { const now = new Date().toISOString(); try { await ddb.send(new UpdateCommand({ @@ -566,7 +564,7 @@ async function failTaskOnScreening( task_id: taskId, request_id: requestId, }); - return; + return false; } throw err; } @@ -592,6 +590,8 @@ async function failTaskOnScreening( request_id: requestId, }); } + + return true; } // --------------------------------------------------------------------------- diff --git a/cdk/test/handlers/cleanup-pending-uploads.test.ts b/cdk/test/handlers/cleanup-pending-uploads.test.ts index 5ca077bf..3dd9fb99 100644 --- a/cdk/test/handlers/cleanup-pending-uploads.test.ts +++ b/cdk/test/handlers/cleanup-pending-uploads.test.ts @@ -173,6 +173,63 @@ describe('cleanup-pending-uploads handler', () => { await expect(handler()).rejects.toThrow('All 1 expired PENDING_UPLOADS task(s) failed to process'); }); + test('continues pagination when S3 returns empty page with IsTruncated=true', async () => { + const thirtyFiveMinAgo = new Date(Date.now() - 35 * 60 * 1000).toISOString(); + + mockDdbSend.mockResolvedValueOnce({ + Items: [{ + task_id: { S: 'TASK001' }, + user_id: { S: 'user-123' }, + created_at: { S: thirtyFiveMinAgo }, + }], + }); + + // cancelExpiredTask succeeds + mockDdbSend.mockResolvedValueOnce({}); + // write event + mockDdbSend.mockResolvedValueOnce({}); + + // Page 1: empty but IsTruncated=true (S3 scanned past prefix boundary) + mockS3Send.mockResolvedValueOnce({ + Versions: [], + DeleteMarkers: [], + IsTruncated: true, + NextKeyMarker: 'attachments/user-123/TASK001/ATT001/image.png', + NextVersionIdMarker: 'v1', + }); + // Page 2: has objects, not truncated + mockS3Send.mockResolvedValueOnce({ + Versions: [ + { Key: 'attachments/user-123/TASK001/ATT001/image.png', VersionId: 'v1' }, + ], + DeleteMarkers: [], + IsTruncated: false, + }); + // DeleteObjects succeeds + mockS3Send.mockResolvedValueOnce({ Deleted: [{}] }); + + await handler(); + + // Verify both ListObjectVersions calls were made (pagination continued past empty page) + const listCalls = mockS3Send.mock.calls.filter( + (call: any[]) => call[0]?._type === 'ListObjectVersions', + ); + expect(listCalls).toHaveLength(2); + + // Verify second list call used the marker from first response + expect(listCalls[1][0].input.KeyMarker).toBe('attachments/user-123/TASK001/ATT001/image.png'); + expect(listCalls[1][0].input.VersionIdMarker).toBe('v1'); + + // Verify delete was called with the objects from page 2 + const deleteCalls = mockS3Send.mock.calls.filter( + (call: any[]) => call[0]?._type === 'DeleteObjects', + ); + expect(deleteCalls).toHaveLength(1); + expect(deleteCalls[0][0].input.Delete.Objects).toEqual([ + { Key: 'attachments/user-123/TASK001/ATT001/image.png', VersionId: 'v1' }, + ]); + }); + test('does not throw on partial success (some cancelled, some errored)', async () => { const thirtyFiveMinAgo = new Date(Date.now() - 35 * 60 * 1000).toISOString(); diff --git a/cdk/test/handlers/confirm-uploads.test.ts b/cdk/test/handlers/confirm-uploads.test.ts index caa28af2..7ece5931 100644 --- a/cdk/test/handlers/confirm-uploads.test.ts +++ b/cdk/test/handlers/confirm-uploads.test.ts @@ -312,4 +312,173 @@ describe('confirm-uploads handler', () => { const body = JSON.parse(result.body); expect(body.error.code).toBe('ATTACHMENT_BLOCKED'); }); + + test('skips S3 cleanup when failTaskOnScreening loses the race (ConditionalCheckFailedException)', async () => { + const { screenImage, AttachmentScreeningError } = jest.requireMock('../../src/handlers/shared/attachment-screening'); + + ddbSend.mockResolvedValueOnce({ Item: PENDING_TASK }); + s3Send + .mockResolvedValueOnce({ VersionId: 'v1', ContentLength: 1024 }) + .mockResolvedValueOnce({ VersionId: 'v2', ContentLength: 512 }); + + // Pre-check passes + ddbSend.mockResolvedValueOnce({ Item: { active_count: 0 } }); + + // GetObject for first attachment + const pngContent = Buffer.alloc(1024); + s3Send.mockResolvedValueOnce({ Body: { transformToByteArray: () => pngContent } }); + + // Screening blocks the image + screenImage.mockRejectedValueOnce(new AttachmentScreeningError('Inappropriate content detected')); + + // failTaskOnScreening conditional write fails — another caller already transitioned + const condErr = new Error('The conditional request failed'); + condErr.name = 'ConditionalCheckFailedException'; + ddbSend.mockRejectedValueOnce(condErr); + + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(400); + + // S3 DeleteObjectsCommand should NOT have been called (only Head + Get calls) + const s3DeleteCalls = s3Send.mock.calls.filter( + (call: any[]) => call[0]?._type === 'S3Delete', + ); + expect(s3DeleteCalls).toHaveLength(0); + }); + + test('does not re-upload content to S3 after screening passes (no redundant PUT)', async () => { + const { screenImage, screenTextFile } = jest.requireMock('../../src/handlers/shared/attachment-screening'); + + const pngContent = Buffer.alloc(1024); + pngContent.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const textContent = Buffer.alloc(512); + textContent.write('hello world'); + + screenImage.mockResolvedValue({ + content: pngContent, + contentType: 'image/png', + checksum: 'abc123', + screening: { status: 'passed' }, + }); + screenTextFile.mockResolvedValue({ + content: textContent, + contentType: 'text/plain', + checksum: 'def456', + screening: { status: 'passed' }, + }); + + let ddbCallCount = 0; + ddbSend.mockImplementation(() => { + ddbCallCount++; + switch (ddbCallCount) { + case 1: return Promise.resolve({ Item: PENDING_TASK }); + case 2: return Promise.resolve({ Item: { active_count: 1 } }); + case 3: return Promise.resolve({}); + case 4: return Promise.resolve({}); + case 5: return Promise.resolve({}); + default: return Promise.resolve({}); + } + }); + + s3Send.mockImplementation((cmd: any) => { + if (cmd._type === 'S3Head') { + const isAtt1 = cmd.input.Key?.includes('att-1'); + return Promise.resolve({ + VersionId: isAtt1 ? 'v1' : 'v2', + ContentLength: isAtt1 ? 1024 : 512, + }); + } + if (cmd._type === 'S3Get') { + const isAtt1 = cmd.input.Key?.includes('att-1'); + return Promise.resolve({ + Body: { transformToByteArray: () => (isAtt1 ? pngContent : textContent) }, + }); + } + if (cmd._type === 'S3Put') { + return Promise.resolve({ VersionId: 'v-screened' }); + } + return Promise.resolve({}); + }); + + lambdaSend.mockResolvedValueOnce({}); + + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(200); + + // Verify NO S3 PutObject calls were made + const s3PutCalls = s3Send.mock.calls.filter( + (call: any[]) => call[0]?._type === 'S3Put', + ); + expect(s3PutCalls).toHaveLength(0); + }); + + test('uses original versionId and size from HeadObject in attachment record after screening', async () => { + const { screenImage, screenTextFile } = jest.requireMock('../../src/handlers/shared/attachment-screening'); + + const pngContent = Buffer.alloc(1024); + pngContent.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const textContent = Buffer.alloc(512); + textContent.write('hello world'); + + screenImage.mockResolvedValue({ + content: pngContent, + contentType: 'image/png', + checksum: 'abc123', + screening: { status: 'passed' }, + }); + screenTextFile.mockResolvedValue({ + content: textContent, + contentType: 'text/plain', + checksum: 'def456', + screening: { status: 'passed' }, + }); + + let ddbCallCount = 0; + ddbSend.mockImplementation(() => { + ddbCallCount++; + switch (ddbCallCount) { + case 1: return Promise.resolve({ Item: PENDING_TASK }); + case 2: return Promise.resolve({ Item: { active_count: 0 } }); + case 3: return Promise.resolve({}); + case 4: return Promise.resolve({}); + case 5: return Promise.resolve({}); + default: return Promise.resolve({}); + } + }); + + s3Send.mockImplementation((cmd: any) => { + if (cmd._type === 'S3Head') { + const isAtt1 = cmd.input.Key?.includes('att-1'); + return Promise.resolve({ + VersionId: isAtt1 ? 'original-v1' : 'original-v2', + ContentLength: isAtt1 ? 1024 : 512, + }); + } + if (cmd._type === 'S3Get') { + const isAtt1 = cmd.input.Key?.includes('att-1'); + return Promise.resolve({ + Body: { transformToByteArray: () => (isAtt1 ? pngContent : textContent) }, + }); + } + return Promise.resolve({}); + }); + + lambdaSend.mockResolvedValueOnce({}); + + const result = await handler(makeEvent('task-1'), makeContext(180_000)); + expect(result.statusCode).toBe(200); + + // Check the DDB UpdateCommand (transition to SUBMITTED) includes original versionIds + const updateCall = ddbSend.mock.calls.find( + (call: any[]) => call[0]?.input?.UpdateExpression?.includes('attachments'), + ); + expect(updateCall).toBeDefined(); + const attachments = updateCall![0].input.ExpressionAttributeValues[':atts']; + const att1 = attachments.find((a: any) => a.attachment_id === 'att-1'); + const att2 = attachments.find((a: any) => a.attachment_id === 'att-2'); + expect(att1.s3_version_id).toBe('original-v1'); + expect(att1.size_bytes).toBe(1024); + expect(att2.s3_version_id).toBe('original-v2'); + expect(att2.size_bytes).toBe(512); + }); }); From 537e367e9b0a22b92057aa8218552bdbee82c2e7 Mon Sep 17 00:00:00 2001 From: bgagent Date: Tue, 26 May 2026 14:00:23 -0500 Subject: [PATCH 19/19] fix(pr): reviewer comment --- agent/src/attachments.py | 30 ++++++++++++++++++------------ agent/tests/test_attachments.py | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/agent/src/attachments.py b/agent/src/attachments.py index b998cdf8..51174920 100644 --- a/agent/src/attachments.py +++ b/agent/src/attachments.py @@ -61,19 +61,25 @@ def download_attachments( s3_client = boto3.client("s3") prepared: list[PreparedAttachment] = [] - for att in attachments: - local_path = _download_single(att, attachments_dir, s3_client) - prepared.append( - PreparedAttachment( - attachment_id=att.attachment_id, - type=att.type, - content_type=att.content_type, - filename=att.filename, - local_path=str(local_path), - size_bytes=att.size_bytes, - token_estimate=att.token_estimate, + try: + for att in attachments: + local_path = _download_single(att, attachments_dir, s3_client) + prepared.append( + PreparedAttachment( + attachment_id=att.attachment_id, + type=att.type, + content_type=att.content_type, + filename=att.filename, + local_path=str(local_path), + size_bytes=att.size_bytes, + token_estimate=att.token_estimate, + ) ) - ) + except Exception: + import shutil + + shutil.rmtree(attachments_dir, ignore_errors=True) + raise log("TASK", f"Downloaded {len(prepared)} attachment(s) to {attachments_dir}") return prepared diff --git a/agent/tests/test_attachments.py b/agent/tests/test_attachments.py index 456d3aba..975f0b9d 100644 --- a/agent/tests/test_attachments.py +++ b/agent/tests/test_attachments.py @@ -179,3 +179,25 @@ def test_multiple_attachments_all_verified(self, mock_client, tmp_path): assert len(result) == 2 assert result[0].filename == "file1.txt" assert result[1].filename == "file2.txt" + + @patch("boto3.client") + def test_partial_failure_cleans_up_attachments_dir(self, mock_client, tmp_path): + """When download fails mid-loop, already-downloaded files are removed.""" + config_ok, content_ok = _make_config(b"good content", "good.txt", "ATT001") + config_bad, _ = _make_config(b"bad content", "bad.txt", "ATT002") + + mock_s3 = MagicMock() + mock_client.return_value = mock_s3 + # First attachment succeeds, second returns tampered content (checksum mismatch) + mock_s3.get_object.side_effect = [ + {"Body": MagicMock(read=lambda: content_ok)}, + {"Body": MagicMock(read=lambda: b"tampered")}, + ] + + attachments_dir = tmp_path / ATTACHMENTS_DIR + + with pytest.raises(RuntimeError, match="integrity check failed"): + download_attachments([config_ok, config_bad], str(tmp_path)) + + # The entire .attachments directory should be cleaned up + assert not attachments_dir.exists()