Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 8 additions & 28 deletions src/agents/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ registerBackend(new ClaudeCodeBackend());
* 2. Project-level default backend
* 3. Cascade-level default backend
* 4. Fallback: 'llmist'
*
* All backends — including llmist — go through the shared adapter
* (executeWithBackend), which handles repo setup, lifecycle, progress
* monitoring, run tracking, and log finalization in one place.
*/
export async function runAgent(
agentType: string,
Expand All @@ -48,34 +52,10 @@ export async function runAgent(

logger.info('Running agent via backend', { agentType, backend: backendName });

// For the llmist backend, delegate directly (it wraps existing executors)
// For other backends, use the shared adapter which handles lifecycle
if (backendName === 'llmist') {
// The llmist backend needs the full AgentBackendInput, but since it
// delegates to the existing executors which handle their own lifecycle,
// we pass a minimal input and let it reconstruct what it needs.
return backend.execute({
agentType,
project: input.project,
config: input.config,
repoDir: '',
systemPrompt: '',
taskPrompt: '',
cliToolsDir: '',
availableTools: [],
contextInjections: [],
maxIterations: 0,
model: '',
progressReporter: {
onIteration: async () => {},
onToolCall: () => {},
onText: () => {},
},
logWriter: () => {},
agentInput: input,
});
}

// All backends (including llmist) use the shared adapter which handles:
// - Repo setup, CWD change/restore, env var loading
// - Run record creation, log finalization
// - Progress monitor, watchdog
return executeWithBackend(backend, agentType, input);
}

Expand Down
1 change: 1 addition & 0 deletions src/backends/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,7 @@ export async function executeWithBackend(
onText: () => {},
},
runId,
llmistLogPath: fileLogger.llmistLogPath,
};

monitor?.start();
Expand Down
189 changes: 146 additions & 43 deletions src/backends/llmist/index.ts
Original file line number Diff line number Diff line change
@@ -1,37 +1,36 @@
import { executeAgent } from '../../agents/base.js';
import { executeRespondToCIAgent } from '../../agents/respond-to-ci.js';
import { executeRespondToPRCommentAgent } from '../../agents/respond-to-pr-comment.js';
import { executeRespondToReviewAgent } from '../../agents/respond-to-review.js';
import { executeReviewAgent } from '../../agents/review.js';
import type { AgentInput, AgentResult, CascadeConfig, ProjectConfig } from '../../types/index.js';
import type { AgentBackend, AgentBackendInput, AgentBackendResult } from '../types.js';
import os from 'node:os';

/**
* Mapping from agent type to its specialized executor function.
* Agents not listed here fall through to the base `executeAgent()`.
*/
const specializedExecutors: Record<
string,
(input: AgentInput & { project: ProjectConfig; config: CascadeConfig }) => Promise<AgentResult>
> = {
'respond-to-review': (input) =>
executeRespondToReviewAgent(input as Parameters<typeof executeRespondToReviewAgent>[0]),
'respond-to-ci': (input) =>
executeRespondToCIAgent(input as Parameters<typeof executeRespondToCIAgent>[0]),
'respond-to-pr-comment': (input) =>
executeRespondToPRCommentAgent(input as Parameters<typeof executeRespondToPRCommentAgent>[0]),
review: (input) => executeReviewAgent(input as Parameters<typeof executeReviewAgent>[0]),
};
import { LLMist, type ModelSpec, createLogger } from 'llmist';

import { type BuilderType, createConfiguredBuilder } from '../../agents/shared/builderFactory.js';
import { injectSyntheticCall } from '../../agents/shared/syntheticCalls.js';
import { runAgentLoop } from '../../agents/utils/agentLoop.js';
import type { AccumulatedLlmCall } from '../../agents/utils/hooks.js';
import { getLogLevel } from '../../agents/utils/index.js';
import { createAgentLogger } from '../../agents/utils/logging.js';
import { createTrackingContext } from '../../agents/utils/tracking.js';
import { CUSTOM_MODELS } from '../../config/customModels.js';
import { createLLMCallLogger } from '../../utils/llmLogging.js';
import { extractPRUrl } from '../../utils/prUrl.js';
import { getAgentProfile } from '../agent-profiles.js';
import type { AgentBackend, AgentBackendInput, AgentBackendResult } from '../types.js';

/**
* llmist backend - wraps the existing llmist-based agent execution.
* llmist backend — executes agents using the llmist SDK.
*
* This is the "Option A" approach: the llmist backend delegates to the existing
* executeAgent()/executeGitHubAgent() functions as-is. The shared adapter from
* adapter.ts handles lifecycle only for non-llmist backends.
* Receives a fully pre-resolved AgentBackendInput from the shared adapter
* (adapter.ts → executeWithBackend → buildBackendInput), which provides:
* - systemPrompt, taskPrompt, model, maxIterations
* - contextInjections (pre-fetched PR/work-item/directory data)
* - repoDir (already set up by the outer executeAgentPipeline)
* - logWriter (shared file logger from the outer pipeline)
*
* In a follow-up, the llmist code can be refactored to also use the shared adapter,
* but that's not needed for this PR.
* Llmist-specific features preserved:
* - AccumulatedLlmCall metrics (via createObserverHooks inside createConfiguredBuilder)
* - Loop detection and hard-stop (via createObserverHooks + runAgentLoop)
* - Iteration hints / trailing messages (via createConfiguredBuilder)
* - Context compaction (via createConfiguredBuilder)
* - Synthetic gadget call injection from ContextInjection[]
*/
export class LlmistBackend implements AgentBackend {
readonly name = 'llmist';
Expand All @@ -41,25 +40,129 @@ export class LlmistBackend implements AgentBackend {
}

async execute(input: AgentBackendInput): Promise<AgentBackendResult> {
const fullInput: AgentInput & { project: ProjectConfig; config: CascadeConfig } = {
...input.agentInput,
project: input.project,
config: input.config,
};
const {
agentType,
systemPrompt,
taskPrompt,
model,
maxIterations,
contextInjections,
budgetUsd,
repoDir,
logWriter,
runId,
agentInput,
llmistLogPath,
progressReporter,
} = input;

const profile = getAgentProfile(agentType);

// Create LLMist client with custom model definitions
const client = new LLMist({ customModels: CUSTOM_MODELS as ModelSpec[] });

// Create per-execution llmist logger and tracking state
const llmistLogger = createLogger({ minLevel: getLogLevel() });
const trackingContext = createTrackingContext();
const llmCallAccumulator: AccumulatedLlmCall[] = [];

const executor = specializedExecutors[input.agentType];
const result = executor
? await executor(fullInput)
: await executeAgent(input.agentType, fullInput);
// Create a LLM call logger for raw request/response file logging.
// Lives in the system tmp dir, independent from the outer fileLogger
// (which handles cascade.log / llmist.log).
const llmCallLogger = createLLMCallLogger(os.tmpdir(), `llmist-${agentType}-${Date.now()}`);

// Point llmist SDK at the workspace directory llmist log path (provided by the outer
// pipeline's fileLogger). This ensures the structured llmist log is included in run
// records and log bundles (read from fileLogger.llmistLogPath during finalization).
if (llmistLogPath) {
process.env.LLMIST_LOG_FILE = llmistLogPath;
process.env.LLMIST_LOG_TEE = 'true';
}

// Get gadget instances from the agent profile (single source of truth for tool sets)
const gadgets = profile.getLlmistGadgets(agentType);

// Build the configured agent builder with all llmist-specific features:
// rate limiting, retry, compaction, iteration hints, observer hooks
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[SHOULD_FIX] Missing progressMonitor parameter. The adapter creates a ProgressMonitor and passes it as input.progressReporter, but this backend never passes it to createConfiguredBuilder. As a result, createObserverHooks won't call progressMonitor.onIteration() (hooks.ts:112-114), and the progress accumulator will have zero state — progress updates to Trello/GitHub will be empty.

The old code path (lifecycle.ts:267) passed the progress monitor through to the builder. Fix:

progressMonitor: input.progressReporter as ProgressMonitor,

(or pass it through a more type-safe way)

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed! Now passing input.progressReporter to createConfiguredBuilder as the progressMonitor parameter. This ensures createObserverHooks can call onIteration(), onToolCall(), and onText() to populate the progress accumulator, enabling progress updates to Trello/GitHub.

let builder: BuilderType = createConfiguredBuilder({
client,
agentType,
model,
systemPrompt,
maxIterations,
llmistLogger,
trackingContext,
logWriter,
llmCallLogger,
repoDir,
gadgets: gadgets as Parameters<typeof createConfiguredBuilder>[0]['gadgets'],
remainingBudgetUsd: budgetUsd,
llmCallAccumulator,
runId,
baseBranch: input.project.baseBranch,
projectId: input.project.id,
cardId: agentInput.cardId,
// Pass the progress monitor from the adapter so createObserverHooks can call
// onIteration/onToolCall/onText — enables progress updates to Trello/GitHub
progressMonitor: progressReporter as Parameters<
typeof createConfiguredBuilder
>[0]['progressMonitor'],
// Implementation agent uses sequential execution to ensure file operations
// are properly ordered (e.g., FileSearchAndReplace then ReadFile on same file)
postConfigure:
agentType === 'implementation' ? (b) => b.withGadgetExecutionMode('sequential') : undefined,
});

// Convert ContextInjection[] from the unified adapter into synthetic gadget calls.
// This is the llmist-native way to inject pre-fetched context: each injection
// appears in the conversation as if the agent called the gadget itself.
for (let idx = 0; idx < contextInjections.length; idx++) {
const injection = contextInjections[idx];
const invocationId = `gc_${injection.toolName.toLowerCase()}_${idx}`;
builder = injectSyntheticCall(
builder,
trackingContext,
injection.toolName,
injection.params,
injection.result,
invocationId,
);
}

// Create agent logger that writes to the shared logWriter from the outer pipeline
const log = createAgentLogger({ write: logWriter } as Parameters<typeof createAgentLogger>[0]);

log.info('Starting llmist agent', {
model,
maxIterations,
promptLength: taskPrompt.length,
contextInjections: contextInjections.length,
runId,
});

// Run the agent event loop (includes loop detection, session notices, etc.)
const agent = builder.ask(taskPrompt);
const result = await runAgentLoop(
agent,
log,
trackingContext,
agentInput.interactive === true,
agentInput.autoAccept === true,
);

log.info('Agent completed', {
iterations: result.iterations,
gadgetCalls: result.gadgetCalls,
cost: result.cost,
loopTerminated: result.loopTerminated ?? false,
});

return {
success: result.success,
success: !result.loopTerminated,
output: result.output,
prUrl: result.prUrl,
error: result.error,
prUrl: extractPRUrl(result.output) ?? undefined,
error: result.loopTerminated ? 'Agent terminated due to persistent loop' : undefined,
cost: result.cost,
logBuffer: result.logBuffer,
runId: result.runId,
};
}
}
2 changes: 2 additions & 0 deletions src/backends/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ export interface AgentBackendInput {
enableStopHooks?: boolean;
/** Whether to block git push in hooks (defaults to true) */
blockGitPush?: boolean;
/** Path where the llmist SDK should write its structured log (workspace dir, not temp) */
llmistLogPath?: string;
}

export type LogWriter = (level: string, message: string, context?: Record<string, unknown>) => void;
Expand Down
26 changes: 13 additions & 13 deletions tests/unit/agents/registry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ describe('runAgent', () => {
const backend = makeMockBackend('llmist');
mockResolveBackendName.mockReturnValue('llmist');
mockGetBackend.mockReturnValue(backend);
mockExecuteWithBackend.mockResolvedValue({ success: true, output: 'Done' });

await runAgent('implementation', makeInput());

Expand Down Expand Up @@ -120,26 +121,25 @@ describe('runAgent', () => {
expect(result.error).toContain('does not support agent type "implementation"');
});

it('for llmist: calls backend.execute with minimal input + agentInput', async () => {
it('for llmist: calls executeWithBackend (unified adapter path)', async () => {
const backend = makeMockBackend('llmist');
mockResolveBackendName.mockReturnValue('llmist');
mockGetBackend.mockReturnValue(backend);
mockExecuteWithBackend.mockResolvedValue({
success: true,
output: 'Done via adapter',
});

await runAgent('implementation', makeInput());
const input = makeInput();
const result = await runAgent('implementation', input);

expect(backend.execute).toHaveBeenCalledWith(
expect.objectContaining({
agentType: 'implementation',
repoDir: '',
systemPrompt: '',
availableTools: [],
contextInjections: [],
}),
);
expect(mockExecuteWithBackend).not.toHaveBeenCalled();
// llmist now goes through executeWithBackend like all other backends
expect(mockExecuteWithBackend).toHaveBeenCalledWith(backend, 'implementation', input);
expect(backend.execute).not.toHaveBeenCalled();
expect(result.output).toBe('Done via adapter');
});

it('for non-llmist: calls executeWithBackend with full lifecycle', async () => {
it('for claude-code: calls executeWithBackend with full lifecycle', async () => {
const backend = makeMockBackend('claude-code');
mockResolveBackendName.mockReturnValue('claude-code');
mockGetBackend.mockReturnValue(backend);
Expand Down
Loading