Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 28 additions & 19 deletions src/backends/progressModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,34 +129,43 @@ export async function callProgressModel(
}

/**
* Make the actual single-shot LLM call to generate a progress summary.
* Make the LLM call to generate a progress summary.
* Retries once on empty output — Gemini Flash Lite intermittently returns
* streaming responses with zero text content.
*/
async function callProgressModelOnce(
model: string,
context: ProgressContext,
customModels: ModelSpec[],
): Promise<string> {
const client = new LLMist({ customModels });

const builder = new AgentBuilder(client)
.withModel(model)
.withTemperature(0)
.withSystem(PROGRESS_SYSTEM_PROMPT)
.withMaxIterations(1);

const agent = builder.ask(formatProgressUserPrompt(context));
for (let attempt = 0; attempt < 2; attempt++) {
const client = new LLMist({ customModels });

const builder = new AgentBuilder(client)
.withModel(model)
.withTemperature(0)
.withSystem(PROGRESS_SYSTEM_PROMPT)
.withMaxIterations(1);

const agent = builder.ask(formatProgressUserPrompt(context));

const outputLines: string[] = [];
for await (const event of agent.run()) {
if (event.type === 'text' && event.content) {
outputLines.push(event.content);
}
}

const outputLines: string[] = [];
for await (const event of agent.run()) {
if (event.type === 'text' && event.content) {
outputLines.push(event.content);
const output = outputLines.join('\n').trim();
if (output) {
return output;
}
}

const output = outputLines.join('\n').trim();
if (!output) {
throw new Error('Progress model returned empty output');
// First attempt returned empty — retry after brief delay
if (attempt === 0) {
await new Promise((resolve) => setTimeout(resolve, 1_000));
}
}

return output;
throw new Error('Progress model returned empty output');
}
88 changes: 77 additions & 11 deletions tests/unit/backends/progressModel.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,40 +101,106 @@ describe('callProgressModel', () => {
expect(result).toBe('Valid text output.');
});

it('throws when LLM returns empty output', async () => {
it('throws when LLM returns empty output on both attempts', async () => {
async function* fakeRun() {
yield { type: 'text', content: '' };
}

// Both attempts return empty
for (let i = 0; i < 2; i++) {
MockAgentBuilder.mockImplementationOnce(() => ({
withModel: vi.fn().mockReturnThis(),
withTemperature: vi.fn().mockReturnThis(),
withSystem: vi.fn().mockReturnThis(),
withMaxIterations: vi.fn().mockReturnThis(),
ask: vi.fn().mockReturnValue({ run: fakeRun }),
}));
}

await expect(callProgressModel('test-model', makeContext(), [])).rejects.toThrow(
'Progress model returned empty output',
);
});

it('throws when LLM returns no events on both attempts', async () => {
async function* fakeRun() {
// yields nothing
}

// Both attempts yield nothing
for (let i = 0; i < 2; i++) {
MockAgentBuilder.mockImplementationOnce(() => ({
withModel: vi.fn().mockReturnThis(),
withTemperature: vi.fn().mockReturnThis(),
withSystem: vi.fn().mockReturnThis(),
withMaxIterations: vi.fn().mockReturnThis(),
ask: vi.fn().mockReturnValue({ run: fakeRun }),
}));
}

await expect(callProgressModel('test-model', makeContext(), [])).rejects.toThrow(
'Progress model returned empty output',
);
});

it('retries once on empty output and succeeds', async () => {
async function* emptyRun() {
yield { type: 'text', content: '' };
}
async function* successRun() {
yield { type: 'text', content: 'Recovered progress update.' };
}

// First attempt: empty output
MockAgentBuilder.mockImplementationOnce(() => ({
withModel: vi.fn().mockReturnThis(),
withTemperature: vi.fn().mockReturnThis(),
withSystem: vi.fn().mockReturnThis(),
withMaxIterations: vi.fn().mockReturnThis(),
ask: vi.fn().mockReturnValue({ run: fakeRun }),
ask: vi.fn().mockReturnValue({ run: emptyRun }),
}));
// Second attempt: success
MockAgentBuilder.mockImplementationOnce(() => ({
withModel: vi.fn().mockReturnThis(),
withTemperature: vi.fn().mockReturnThis(),
withSystem: vi.fn().mockReturnThis(),
withMaxIterations: vi.fn().mockReturnThis(),
ask: vi.fn().mockReturnValue({ run: successRun }),
}));

await expect(callProgressModel('test-model', makeContext(), [])).rejects.toThrow(
'Progress model returned empty output',
);
const result = await callProgressModel('test-model', makeContext(), []);
expect(result).toBe('Recovered progress update.');
expect(MockAgentBuilder).toHaveBeenCalledTimes(2);
});

it('throws when LLM returns no events', async () => {
async function* fakeRun() {
it('retries once on no events and succeeds', async () => {
async function* noEventsRun() {
// yields nothing
}
async function* successRun() {
yield { type: 'text', content: 'Recovered after no events.' };
}

// First attempt: no events
MockAgentBuilder.mockImplementationOnce(() => ({
withModel: vi.fn().mockReturnThis(),
withTemperature: vi.fn().mockReturnThis(),
withSystem: vi.fn().mockReturnThis(),
withMaxIterations: vi.fn().mockReturnThis(),
ask: vi.fn().mockReturnValue({ run: fakeRun }),
ask: vi.fn().mockReturnValue({ run: noEventsRun }),
}));
// Second attempt: success
MockAgentBuilder.mockImplementationOnce(() => ({
withModel: vi.fn().mockReturnThis(),
withTemperature: vi.fn().mockReturnThis(),
withSystem: vi.fn().mockReturnThis(),
withMaxIterations: vi.fn().mockReturnThis(),
ask: vi.fn().mockReturnValue({ run: successRun }),
}));

await expect(callProgressModel('test-model', makeContext(), [])).rejects.toThrow(
'Progress model returned empty output',
);
const result = await callProgressModel('test-model', makeContext(), []);
expect(result).toBe('Recovered after no events.');
expect(MockAgentBuilder).toHaveBeenCalledTimes(2);
});

it('throws when LLM call times out (races against a slow call)', async () => {
Expand Down