feat(runtime): recover a bounded number of output-ceiling truncations

When a turn stops at the output-token ceiling (output_truncated), the run no
longer dies. The truncated turn is still dropped whole - no tool is executed
and the invalid assistant message is never appended - but the model is given a
standalone instruction to split its proposals and the run continues. Recoveries
are capped per run (2); past that the original diagnostic still fails the run.
The connection is not cooled down for a truncation, since it responded fine and
only the model overran its budget, so the recovery turn can reach it. Both the
failed attempt and the recovering turn are accounted; neither is unknown.
Fixture regression covers recover-then-continue and the exhausted cap.
This commit is contained in:
Agent Workbench Dev
2026-11-05 05:56:10 +08:00
parent dfb6029f33
commit 0fefa65eeb
3 changed files with 80 additions and 0 deletions
+5
View File
@@ -424,6 +424,11 @@ export class ModelHub {
// A user stop is not evidence that the provider is unhealthy. Keep
// the attempt, release its lease, and leave the connection eligible.
this.health.set(this.identity(member), { ...health, state: 'interrupted', lastInterruptedAt: stamp() });
} else if (diagnostic?.reason === 'output_truncated') {
// The connection responded correctly; the model only overran its output
// budget. That is not a health failure, so do not cool the member down -
// the bounded recovery turn must be able to reach the same connection.
this.health.set(this.identity(member), { ...health, state: 'responded', lastSuccessAt: stamp(), cooldownUntil: 0, failures: 0 });
} else {
const blocked = [401, 403].includes(attempt.httpStatus) || ['quota', 'model_identity'].includes(diagnostic?.kind);
const failedAt = Date.now();
+26
View File
@@ -60,6 +60,11 @@ import { WORKER_TOOLS, COLLABORATION_CLOSED, workerForSession, assertWorkerTool,
const workerAuthority = Symbol('internal-worker-start');
// How many times one run may recover from an output-token-ceiling truncation by
// asking the model to split its proposals. A model that keeps overflowing past this
// still fails with the original diagnostic rather than looping on the budget.
const OUTPUT_TRUNCATION_RECOVERY_LIMIT = 2;
const labels = { diagnose_environment:'诊断工程环境',plan_environment:'准备依赖计划',prepare_environment:'执行依赖准备',inspect_session: '核对工作台运行记录', inspect_project: '识别工程与Git基线', index_project: '建立工程符号索引', query_code: '定位工程符号与导入', propose_patch: '准备精确补丁', plan_checks: '准备工程检查计划', run_checks: '执行工程检查', plan_delivery: '准备Git交付预览', delivery_report: '生成工程证据报告', read_artifact: '读取执行日志工件', list_files: '查看项目文件', read_file: '读取文件', search_files: '搜索项目', propose_file: '准备文件改动', run_command: '运行终端命令' };
labels.prepare_change_set = '准备多文件变更集';
@@ -472,6 +477,27 @@ export class Runtime {
if(budget instanceof BudgetExceeded&&error.diagnostic)budget.contextBudget={modelFailure:structuredClone(error.diagnostic)};
throw budget;
}
// Bounded recovery for a response stopped at the output-token ceiling. The
// truncated turn is never executed or appended to history; instead the model
// is told it was cut off and asked to split its proposals, then the run
// continues. The failed attempt and its usage are already accounted above.
// Recoveries are capped per run so a model that keeps overflowing still stops.
if (error.diagnostic?.reason === 'output_truncated' && !signal.aborted) {
run.record.outputTruncationRecoveries = (run.record.outputTruncationRecoveries ?? 0) + 1;
if (run.record.outputTruncationRecoveries <= OUTPUT_TRUNCATION_RECOVERY_LIMIT) {
this.store.finishStep(session, stepRecord);
const note = '上一次响应在达到单次输出上限前被截断(finish_reason=length),其中的工具调用整体未执行(即使某个工具参数已完整也未应用,避免落地被截断的半成品)。请把这次改动拆小:每次响应只提交一个文件的 propose_file/propose_patch,或分多轮 prepare_change_set,不要在一个响应里堆叠多个大参数工具。原目标和已读依据都保留。';
// The truncated assistant turn was never appended, so the conversation
// stays valid. Feed the instruction to the model as a standalone turn
// in the provider history, and mirror it to the visible chat log.
session.providerMessages.push({ role: 'user', content: note });
this.store.event(session, { kind: 'context', runId: run.id, label: '模型输出被上限截断,已请求拆分后继续', phase: 'completed', contextPlanId: planned.record.id,
output: '第 ' + run.record.outputTruncationRecoveries + '/' + OUTPUT_TRUNCATION_RECOVERY_LIMIT + ' 次恢复:上一响应 finish_reason=length,未执行任何工具,原响应与用量已保存。' });
this.store.message(session, 'system', note);
continue;
}
error.diagnostic.outputTruncationRecoveries = run.record.outputTruncationRecoveries - 1;
}
throw error;
}
signal.throwIfAborted();
+49
View File
@@ -238,3 +238,52 @@ test('stream parser rejects truncation, finish length, malformed JSON, duplicate
await assert.rejects(readModelStream(new Response(body, { headers: { 'content-type': 'text/event-stream' } }), { protocol }, undefined));
});
});
// A turn truncated at the output ceiling (one complete propose_file followed by a
// second call cut off at finish_reason=length) is dropped whole; the run does not
// fail but injects a split-the-work message and continues. On the next turn the model
// obeys and the run completes. This is the AB sample shape, recovered end to end.
const truncatedTurn = chat({ tool_calls: [
{ index: 0, id: 'file-a', type: 'function', function: { name: 'propose_file', arguments: '{"path":"src/a.mjs","content":"export const a=1;"}' } },
{ index: 1, id: 'file-b', type: 'function', function: { name: 'propose_file', arguments: '{"path":"src/b.mjs","content":"export const b=' } },
] }) + chat({}, 'length') + sse({ choices: [], usage: { prompt_tokens: 9958, completion_tokens: 2048 } }) + sse('[DONE]');
test('an output-ceiling truncation is recovered: the run injects a split-the-work note and continues to completion', async t => {
const f = fixture(t);
const server = await streamingServer(t, (res, body) => {
const sawTruncationNote = body.messages.some(message => message.role === 'user' && /finish_reason=length/.test(message.content ?? ''));
res.end(sawTruncationNote ? chat({ content: '已拆分,本次不再堆叠工具。' }) + chatEnd() : truncatedTurn);
});
const { provider } = configure(f, server), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll());
runtime.start(f.session.id, '写入两个文件'); await until(() => !runtime.isActive(f.session.id), 15000);
// The run recovered rather than failing on the truncation: the model saw the note
// and produced a clean follow-up turn. (It then settles per the file contract.)
assert.notEqual(f.session.status, 'failed', JSON.stringify(f.session.events.slice(-4)));
assert.equal(server.requests.length, 2);
// The second request carried the split-the-work instruction to the model.
assert.ok(server.requests[1].body.messages.some(message => message.role === 'user' && /finish_reason=length/.test(message.content ?? '')));
// The truncated turn executed no tools and was never appended as an assistant turn.
assert.equal(f.store.core.tool.size, 0);
assert.equal(f.session.providerMessages.filter(message => message.tool_calls?.length).length, 0);
assert.equal(f.session.messages.filter(message => message.role === 'system' && /finish_reason=length/.test(message.content)).length, 1);
// Both the failed attempt and the recovering turn are accounted; neither is unknown.
const run = f.store.core.run.get(f.session.runId);
assert.equal(run.usage.requests, 2); assert.equal(run.usage.unknownRequests, 0);
assert.equal(run.outputTruncationRecoveries, 1);
});
test('repeated output-ceiling truncations stop after the recovery limit with the output-truncated diagnostic and zero tools', async t => {
const f = fixture(t);
const server = await streamingServer(t, res => res.end(truncatedTurn));
const { provider } = configure(f, server), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll());
runtime.start(f.session.id, '写入两个文件'); await until(() => !runtime.isActive(f.session.id), 15000);
assert.equal(f.session.status, 'failed');
assert.equal(f.store.core.tool.size, 0);
// Initial attempt plus the capped recoveries, each a real accounted request.
assert.equal(server.requests.length, 3);
const run = f.store.core.run.get(f.session.runId);
assert.equal(run.usage.requests, 3); assert.equal(run.usage.unknownRequests, 0);
assert.equal(run.outputTruncationRecoveries, 3);
assert.equal(f.session.failure.reason, 'output_truncated');
assert.match(f.session.failure.message, /output token 上限|finish_reason=length/);
});