Skip to content

fix(cloud-agent-next): destroy sandbox on prep 500 #3081

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eshurakov merged 1 commit into from
May 7, 2026
Merged

fix(cloud-agent-next): destroy sandbox on prep 500 #3081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 60 additions & 44 deletions services/cloud-agent-next/src/execution/orchestrator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import { withDORetry } from '../utils/do-retry.js';
import { normalizeAgentMode } from '../schema.js';
import { buildImagePromptParts, downloadImagePromptParts } from './image-prompt-parts.js';
import { withTimeout } from '@kilocode/worker-utils';
import { withSandboxInternalServerErrorRecovery } from '../sandbox-recovery.js';

/** Maximum time allowed for workspace preparation (resume, init, fast path). */
const PREPARE_WORKSPACE_TIMEOUT_MS = 10 * 60 * 1000;
Expand Down Expand Up @@ -137,56 +138,71 @@ export class ExecutionOrchestrator {
);
}

// 2. Workspace preparation (may throw WORKSPACE_SETUP_FAILED)
const prepared = await this.prepareWorkspace(sandbox, plan, options?.onProgress);
const prepareExecution = async () => {
// 2. Workspace preparation (may throw WORKSPACE_SETUP_FAILED)
const prepared = await this.prepareWorkspace(sandbox, plan, options?.onProgress);

// 3. Update git remote token if needed (resume path with token overrides)
if (!workspace.shouldPrepare) {
const resumeContext = workspace.resumeContext;
if (resumeContext.githubToken || resumeContext.gitToken) {
await this.updateTokenOverrides(prepared, workspace);
// 3. Update git remote token if needed (resume path with token overrides)
if (!workspace.shouldPrepare) {
const resumeContext = workspace.resumeContext;
if (resumeContext.githubToken || resumeContext.gitToken) {
await this.updateTokenOverrides(prepared, workspace);
}
}
}

// 4. Ensure wrapper is running (starts kilo server in-process)
let wrapperClient: WrapperClient;
let kiloSessionId: string;
try {
const result = await WrapperClient.ensureWrapper(sandbox, prepared.session, {
agentSessionId: sessionId,
userId,
workspacePath: prepared.context.workspacePath,
sessionId: wrapper.kiloSessionId,
// 4. Ensure wrapper is running (starts kilo server in-process)
let wrapperClient: WrapperClient;
let kiloSessionId: string;
try {
const result = await WrapperClient.ensureWrapper(sandbox, prepared.session, {
agentSessionId: sessionId,
userId,
Copy link
Copy Markdown
Contributor

@alex-alecu alex-alecu May 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could sandbox 500s from image attachment downloads be swallowed here? downloadImagesToSandbox catches per-file session.exec failures and this rethrows WORKSPACE_SETUP_FAILED without the original cause, so the recovery wrapper may not destroy an unhealthy sandbox.

workspacePath: prepared.context.workspacePath,
sessionId: wrapper.kiloSessionId,
});
wrapperClient = result.client;
kiloSessionId = result.sessionId;
} catch (error) {
throw ExecutionError.wrapperStartFailed(
`Failed to start wrapper: ${error instanceof Error ? error.message : String(error)}`,
error
);
}

// 5. Record activity for idle timeout tracking
try {
await withDORetry(
() => this.deps.getSessionStub(userId, sessionId),
stub => stub.recordKiloServerActivity(),
'recordKiloServerActivity'
);
} catch {
// Non-fatal - log but continue
logger.warn('Failed to record kilo server activity');
}

// 6. Download images from R2 to sandbox if provided
const fileParts = await downloadImagePromptParts({
env: this.deps.env,
session: prepared.session,
userId: plan.userId,
images: plan.images,
createdOnPlatform: this.getCreatedOnPlatform(plan),
});
wrapperClient = result.client;
kiloSessionId = result.sessionId;
} catch (error) {
throw ExecutionError.wrapperStartFailed(
`Failed to start wrapper: ${error instanceof Error ? error.message : String(error)}`,
error
);
}

// 5. Record activity for idle timeout tracking
try {
await withDORetry(
() => this.deps.getSessionStub(userId, sessionId),
stub => stub.recordKiloServerActivity(),
'recordKiloServerActivity'
);
} catch {
// Non-fatal - log but continue
logger.warn('Failed to record kilo server activity');
}
return { prepared, wrapperClient, kiloSessionId, fileParts };
};

// 6. Download images from R2 to sandbox if provided
const fileParts = await downloadImagePromptParts({
env: this.deps.env,
session: prepared.session,
userId: plan.userId,
images: plan.images,
createdOnPlatform: this.getCreatedOnPlatform(plan),
});
const { prepared, wrapperClient, kiloSessionId, fileParts } =
await withSandboxInternalServerErrorRecovery(
{
sandbox,
sandboxId,
sessionId,
phase: 'executionWorkspacePreparation',
},
prepareExecution
);

// 7. Send prompt with execution binding (async - returns messageId immediately)
const ingestUrl = this.deps.getIngestUrl(sessionId, userId);
Expand Down
47 changes: 46 additions & 1 deletion services/cloud-agent-next/src/persistence/async-preparation.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { Env } from '../types.js';
import type { PreparationInput } from './schemas.js';

const { ensureWrapperMock } = vi.hoisted(() => ({
ensureWrapperMock: vi.fn(),
}));

const fakeSession = {
exec: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }),
};

const fakeSandbox = {
writeFile: vi.fn().mockResolvedValue(undefined),
destroy: vi.fn().mockResolvedValue(undefined),
};

vi.mock('@cloudflare/sandbox', () => ({
Expand Down Expand Up @@ -45,16 +50,23 @@ vi.mock('../session-service.js', () => ({

vi.mock('../kilo/wrapper-client.js', () => ({
WrapperClient: {
ensureWrapper: vi.fn().mockResolvedValue({ sessionId: 'ses_wrapper_123' }),
ensureWrapper: ensureWrapperMock,
},
}));

import { cloneGitHubRepo, cloneGitRepo } from '../workspace.js';
import { executePreparationSteps } from './async-preparation.js';

const wrapperResult = {
client: {},
sessionId: 'ses_wrapper_123',
};

describe('executePreparationSteps', () => {
beforeEach(() => {
vi.clearAllMocks();
fakeSandbox.destroy.mockResolvedValue(undefined);
ensureWrapperMock.mockResolvedValue(wrapperResult);
});

it('skips managed GitLab token resolution when caller already resolved it', async () => {
Expand Down Expand Up @@ -182,4 +194,37 @@ describe('executePreparationSteps', () => {
undefined
);
});

it('destroys the sandbox when preparation hits a sandbox 500', async () => {
const env = {
Sandbox: {} as Env['Sandbox'],
SandboxSmall: {} as Env['SandboxSmall'],
GIT_TOKEN_SERVICE: {
getTokenForRepo: vi.fn(),
},
PER_SESSION_SANDBOX_ORG_IDS: '',
GITHUB_APP_SLUG: 'kilo-connect',
GITHUB_APP_BOT_USER_ID: '12345',
} as unknown as Env;
const emitProgress = vi.fn();
const input = {
sessionId: 'agent_test',
userId: 'test-user',
orgId: 'test-org',
authToken: 'kilo-token',
githubRepo: 'acme/repo',
githubToken: 'github-token',
prompt: 'Fix bug',
mode: 'code',
model: 'kilo/test-model',
autoInitiate: false,
} satisfies PreparationInput;
const error = new Error('HTTP error! status: 500');
Object.assign(error, { name: 'SandboxError' });
ensureWrapperMock.mockRejectedValueOnce(error);

await expect(executePreparationSteps(input, env, emitProgress)).rejects.toBe(error);

expect(fakeSandbox.destroy).toHaveBeenCalledOnce();
});
});
Loading