Files
Agent-Workbench/tests/read-budget.test.mjs

215 lines
18 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { randomUUID, createHash } from 'node:crypto';
import { artifactPage, recordArtifactRead, clearReadStreak, readMessageBudget, recordFileRead, beginReadTurn, finishReadTurn } from '../src/core/context/read-budget.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Store } from '../src/server/store.mjs';
import { fixture, compatibleFixture, call, completion, until, assertToolPairs } from './helpers.mjs';
const sha = text => createHash('sha256').update(text, 'utf8').digest('hex');
const bytes = (page, id) => Buffer.byteLength(JSON.stringify({ role: 'tool', tool_call_id: id, content: JSON.stringify(page) }), 'utf8');
test('a mixed turn with new bytes and three repeated modules advances, while repeat-only and independent no-op guards still pause', () => {
const task = { activeRunId: 'owned-run' };
const read = (file, start, end, total = 2584) => recordFileRead(task, {path:file,range_start_line:1,range_end_line:38,
sha256:'a'.repeat(64),offset:start,next_offset:end,total_characters:total,content:'x'.repeat(end-start)}, file + start, task.activeRunId);
read('test/expense.test.mjs', 0, 945);
for (const file of ['amount', 'summary', 'report']) read(file, 0, 51, 51);
beginReadTurn(task, 'owned-run', 'mixed'); read('test/expense.test.mjs', 945, 1413);
for (const file of ['amount', 'summary', 'report']) read(file, 0, 51, 51);
assert.equal(task.readProgress.pause, true);
finishReadTurn(task, 'owned-run', 'mixed'); assert.equal(task.readProgress.pause, false);
assert.equal(task.readProgress.readTurn.newCharacters, 468); assert.equal(task.readProgress.readTurn.repeatedReads, 3);
beginReadTurn(task, 'owned-run', 'repeat');
for (const file of ['amount', 'summary', 'report']) read(file, 0, 51, 51);
finishReadTurn(task, 'owned-run', 'repeat'); assert.equal(task.readProgress.pause, true);
for (const guard of ['checkPause', 'noopPause']) {
clearReadStreak(task); beginReadTurn(task, 'owned-run', guard);
read(guard + '.mjs', 0, 10); task.readProgress[guard] = true; task.readProgress.pause = true;
finishReadTurn(task, 'owned-run', guard); assert.equal(task.readProgress.pause, true);
}
clearReadStreak(task); beginReadTurn(task, 'owned-run', 'same-state');
task.readProgress.sessionInspection = {consecutive:3}; read('same-state.mjs', 0, 10);
finishReadTurn(task, 'owned-run', 'same-state'); assert.equal(task.readProgress.pause, true);
clearReadStreak(task); beginReadTurn(task, 'owned-run', 'empty'); read('empty.mjs', 0, 0, 0);
task.readProgress.pause = true; finishReadTurn(task, 'owned-run', 'empty'); assert.equal(task.readProgress.pause, true);
});
test('runtime sends the next real HTTP request after a W-shaped mixed-progress turn instead of stopping on its trailing repeats', async t => {
const f = fixture(t); const fs = await import('node:fs'), path = await import('node:path');
fs.writeFileSync(path.join(f.projectRoot, 'evidence.txt'), 'x'.repeat(2584));
for (const name of ['amount', 'summary', 'report']) fs.writeFileSync(path.join(f.projectRoot, name + '.mjs'), 'export const value = 0;\n');
let requests = 0;
const modules = index => ['amount', 'summary', 'report'].map(name => call(name + index, 'read_file', {path:name + '.mjs'}));
const model = await compatibleFixture(t, (route, body) => {
requests++;
if (requests === 1) return {body:completion([call('first', 'read_file', {path:'evidence.txt',limit:945}), ...modules(1)])};
if (requests === 2) return {body:completion([call('next', 'read_file', {path:'evidence.txt',offset:945,limit:468}), ...modules(2)])};
const state = f.store.core.task.get(f.session.taskId).readProgress;
assert.equal(state.readTurn.newCharacters, 468); assert.equal(state.readTurn.repeatedReads, 3); assert.equal(state.pause, false);
const next = body.messages.filter(message => message.role === 'tool').map(message => JSON.parse(message.content)).find(result => result.path === 'evidence.txt' && result.offset === 945);
assert.ok(next); assert.equal(next.content, 'x'.repeat(468)); assertToolPairs(body.messages);
return {body:completion([], '已核对指定的新范围与原模块,结束只读核对。')};
});
f.store.saveConfig({mode:'compatible',model:'mixed-progress-fixture',baseUrl:model.baseUrl,maxSteps:8,contextLimit:16000,maxOutputTokens:2048});
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, '只读核对指定文件范围,不修改文件,不运行工程检查。');
await runtime.active.get(f.session.id).promise;
assert.equal(requests, 3, JSON.stringify(f.session.failure)); assert.equal(f.session.status, 'completed',
JSON.stringify({failure:f.session.failure,stopReason:f.store.core.run.get(f.session.runId).stopReason,messages:f.session.messages.slice(-2)}));
assertToolPairs(f.session.providerMessages);
});
test('continuation reads advance through historical pages without a false pause while repeated current-Run pages still pause', () => {
const task = { activeRunId: randomUUID() }, text = 'Persisted failure log\n'.repeat(300),
artifact = { id: randomUUID(), kind: 'check-log', sha256: sha(text) };
recordArtifactRead(task, artifact, artifactPage(artifact, text, { limit: text.length }, { maximumMessageBytes: 30000 }), 'previous-run');
const original = structuredClone(task.readProgress.reads);
task.activeRunId = randomUUID(); clearReadStreak(task);
for (const offset of [0, 669, 1662]) {
const page = artifactPage(artifact, text, { offset, limit: offset === 0 ? 669 : offset === 669 ? 993 : 2500 }, { maximumMessageBytes: 30000 });
recordArtifactRead(task, artifact, page, 'forward-' + offset);
assert.equal(task.readProgress.repeated, 0); assert.equal(task.readProgress.pause, false);
}
assert.deepEqual(task.readProgress.reads, original, 'Historical coverage must remain intact.');
assert.equal(task.readProgress.readWindow.runId, task.activeRunId);
for (const limit of [500, 400, 300]) recordArtifactRead(task, artifact, artifactPage(artifact, text, { offset: 669, limit }), 'repeat-' + limit);
assert.equal(task.readProgress.repeated, 3); assert.equal(task.readProgress.pause, true);
assert.deepEqual(task.readProgress.reads, original);
});
test('a new tool page replaces old latest-feedback space and still obeys its escaped protocol budget', () => {
const message = { role: 'assistant', content: 'Continue reading the actual log.', tool_calls: [call('next', 'read_artifact', { artifact_id: randomUUID() })] };
const base = { budget: { inputBudget: 12000 }, nextReadFixedInputTokens: 6000, workingRetentionTokens: 1000 };
const a = readMessageBudget({ ...base, mandatoryInputTokens: 11500 }, message),
b = readMessageBudget({ ...base, mandatoryInputTokens: 8000 }, message);
assert.equal(a, b, 'Previous feedback is replaced, not reserved twice.');
const text = 'Actual test failure "detail"\n'.repeat(1500), artifact = { id: randomUUID(), kind: 'check-log', sha256: sha(text) };
const page = artifactPage(artifact, text, { limit: 12000 }, { maximumMessageBytes: a, callId: 'next' });
assert.ok(page.next_offset > 2000); assert.equal(page.content, text.slice(0, page.next_offset));
assert.ok(bytes(page, 'next') <= a);
assert.equal(readMessageBudget({ budget: base.budget, mandatoryInputTokens: base.nextReadFixedInputTokens, workingRetentionTokens: 1000 }, message), a,
'Legacy plans keep their original mandatory-budget interpretation.');
});
test('budgeted artifact pages count escaped protocol bytes and reconstruct original multilingual JSON without losing characters', () => {
const text = JSON.stringify({ source: '原文🙂\r\n"\\'.repeat(4000) }), artifact = { id: randomUUID(), sha256: sha(text), kind: 'tool-result' };
let offset = 0, reconstructed = '', pages = 0;
while (offset < text.length) {
const page = artifactPage(artifact, text, { offset, limit: 16000 }, { maximumMessageBytes: 7000, callId: 'literal-read' });
assert.ok(bytes(page, 'literal-read') <= 7000); assert.ok(page.next_offset > offset); assert.equal(page.offset, offset);
assert.equal(page.content, text.slice(offset, page.next_offset)); assert.equal(page.truncated, page.next_offset < text.length);
assert.ok(!/[\uD800-\uDBFF]$/.test(page.content)); reconstructed += page.content; offset = page.next_offset; pages++;
}
assert.ok(pages > 5); assert.equal(reconstructed, text); assert.equal(sha(reconstructed), artifact.sha256);
const unicode = artifactPage({ ...artifact, sha256: sha('🙂雪') }, '🙂雪', { offset: 1, limit: 1 }, { maximumMessageBytes: 1200 });
assert.equal(unicode.offset, 0); assert.equal(unicode.content, '🙂'); assert.equal(unicode.next_offset, 2);
});
test('known artifact coverage detects semantic repeats despite changing limits, and advancing pages or direct evidence restore progress', () => {
const task = {}, text = 'Immutable engineering evidence\n'.repeat(30), artifact = { id: randomUUID(), kind: 'check-log', sha256: sha(text) };
const page = artifactPage(artifact, text, { offset: 0, limit: 200 }, { maximumMessageBytes: 2000 });
recordArtifactRead(task, artifact, page, 'first');
for (const limit of [100, 150, 180]) recordArtifactRead(task, artifact,
artifactPage(artifact, text, { offset: 0, limit }, { maximumMessageBytes: 2000 }), 'repeat-' + limit);
assert.equal(task.readProgress.pause, true); assert.equal(task.readProgress.repeated, 3);
recordArtifactRead(task, artifact, artifactPage(artifact, text, { offset: 200, limit: 200 }, { maximumMessageBytes: 2000 }), 'next');
assert.equal(task.readProgress.pause, false); assert.deepEqual(task.readProgress.reads[0].ranges, [[0, 400]]);
for (let index = 0; index < 6; index++) recordArtifactRead(task, { ...artifact, id: randomUUID(), kind: 'context-history-index' }, page, 'index-' + index);
assert.equal(task.readProgress.pause, true); clearReadStreak(task);
assert.equal(task.readProgress.pause, false); assert.equal(task.readProgress.reads[0].identity, artifact.id + ':' + artifact.sha256);
});
test('actual large artifact reads are present in the next bounded HTTP request and next_offset progresses without redoing tools', async t => {
const f = fixture(t), original = JSON.stringify({ log: '真实证据🙂\n"\\'.repeat(2200) });
const artifact = f.store.artifactStore.write(f.session, null, original, { kind: 'check-log' });
f.store.core.artifact.set(artifact.id, artifact); f.store.save(f.session);
let requests = 0, observedPages = [];
const model = await compatibleFixture(t, (route, body) => {
if (!route.endsWith('/chat/completions')) return { body: {} };
requests++;
const received = body.messages.filter(value => value.role === 'tool').map(value => JSON.parse(value.content)).filter(value => value.artifact_id === artifact.id);
if (requests > 1) {
const page = received.at(-1); assert.ok(page, 'Newest actual result must be sent, not archived before the model sees it');
assert.ok(page.budget_limited); assert.ok(page.next_offset > page.offset); assert.equal(page.content, original.slice(page.offset, page.next_offset));
observedPages.push(page);
if (requests === 4) return { body: completion([], '已根据三段实际结果完成读取验收。') };
return { body: completion([call('page-' + requests, 'read_artifact', { artifact_id: artifact.id, offset: page.next_offset, limit: 16000 })]) };
}
return { body: completion([call('page-first', 'read_artifact', { artifact_id: artifact.id, limit: 16000 })]) };
});
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'page-budget-fixture', maxSteps: 8 });
const runtime = new Runtime(f.store, new Provider()); runtime.start(f.session.id, '读取保存的原工件,分段取得真实依据,不执行命令或修改文件。');
await until(() => !runtime.isActive(f.session.id), 20000);
assert.equal(f.session.status, 'completed', JSON.stringify({ failure: f.session.failure, requests, pages: observedPages.map(value => ({ offset: value.offset, next: value.next_offset, budget: value.maximum_message_bytes })) })); assert.equal(requests, 4); assert.equal(observedPages.length, 3);
assert.equal(observedPages[1].offset, observedPages[0].next_offset); assert.equal(observedPages[2].offset, observedPages[1].next_offset);
assert.equal(f.store.artifactStore.read(artifact), original); assertToolPairs(f.session.providerMessages);
const plans = [...f.store.core.context_plan.values()];
for (const page of observedPages) assert.ok(plans.some(plan => plan.status === 'completed' && plan.retainedGroups.length > 0));
});
test('a real repeated artifact loop pauses the same Task after recorded results, survives reload and only resumes explicitly', async t => {
const f = fixture(t), artifact = f.store.artifactStore.write(f.session, null, 'Known complete evidence', { kind: 'check-log' });
f.store.core.artifact.set(artifact.id, artifact); f.store.save(f.session);
let requests = 0, resume = false;
const model = await compatibleFixture(t, () => {
requests++;
return { body: resume ? completion([], '使用已保存内容完成核对。')
: completion([call('repeat-read-' + requests, 'read_artifact', { artifact_id: artifact.id, limit: 16000 })]) };
});
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'repeat-loop-fixture', maxSteps: 12 });
const runtime = new Runtime(f.store, new Provider()); runtime.start(f.session.id, '使用不可变证据完成当前任务。');
await until(() => !runtime.isActive(f.session.id), 20000);
const taskId = f.session.taskId;
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'no_progress');
assert.equal(requests, 4); assert.equal([...f.store.core.tool.values()].filter(value => value.state === 'completed').length, 4);
assertToolPairs(f.session.providerMessages); f.store.close();
const reopened = new Store(f.data); f.onCleanup(() => reopened.close());
const saved = reopened.session(f.session.id); assert.equal(saved.taskId, taskId); assert.equal(requests, 4);
assert.equal(reopened.core.task.get(taskId).readProgress.pause, true); assertToolPairs(saved.providerMessages);
resume = true; const resumed = new Runtime(reopened, new Provider()); resumed.start(saved.id, '', true);
await until(() => !resumed.isActive(saved.id), 20000);
assert.equal(saved.status, 'completed'); assert.equal(saved.taskId, taskId); assert.equal(requests, 5);
assert.equal([...reopened.core.tool.values()].length, 4); assert.equal(reopened.core.task.get(taskId).readProgress.pause, false);
});
test('actual HTTP continuation after reload sends distinct historically covered pages without false no_progress or replay', async t => {
const f = fixture(t), original = '实际失败日志 original test failure\n'.repeat(100);
const artifact = f.store.artifactStore.write(f.session, null, original, { kind: 'check-log' });
f.store.core.artifact.set(artifact.id, artifact); f.store.save(f.session);
let requests = 0; const observed = [];
const model = await compatibleFixture(t, (route, body) => {
if (!route.endsWith('/chat/completions')) return { body: {} };
requests++;
if (requests >= 3) {
const actual = body.messages.filter(value => value.role === 'tool').map(value => JSON.parse(value.content))
.filter(value => value.artifact_id === artifact.id).at(-1);
assert.ok(actual, 'The previous actual page must reach the resumed model.');
assert.equal(actual.content, original.slice(actual.offset, actual.next_offset)); observed.push(actual);
}
const next = requests === 1 ? { offset: 0, limit: 16000 } : requests === 2 ? { offset: 0, limit: 669 }
: requests === 3 ? { offset: 669, limit: 993 } : requests === 4 ? { offset: 1662, limit: 800 } : null;
return { body: next ? completion([call('historical-page-' + requests, 'read_artifact', { artifact_id: artifact.id, ...next })])
: completion([], '三段原失败依据均已核对。') };
});
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'continuation-pages-fixture', maxSteps: 1 });
const first = new Runtime(f.store, new Provider()); first.start(f.session.id, '读取实际保存的失败依据,然后按原任务继续核对。');
await until(() => !first.isActive(f.session.id), 20000);
assert.equal(f.session.status, 'paused'); assert.equal(requests, 1);
assert.equal(f.store.core.tool.size, 1); const savedResult = f.store.savedToolResult(f.session.id, 'historical-page-1');
assert.equal(savedResult.content, original, 'The historical Run actually returned the full source.');
const taskId = f.session.taskId, oldRunId = f.session.runId, coverage = structuredClone(f.store.core.task.get(taskId).readProgress.reads);
f.store.close(); const reopened = new Store(f.data); f.onCleanup(() => reopened.close());
reopened.saveConfig({ maxSteps: 8 }); const session = reopened.session(f.session.id);
const resumed = new Runtime(reopened, new Provider()); resumed.start(session.id, '', true);
await until(() => !resumed.isActive(session.id), 20000);
assert.equal(session.status, 'completed', JSON.stringify({ failure: session.failure, status: session.status, requests }));
assert.equal(session.taskId, taskId); assert.equal(reopened.core.run.get(session.runId).resumeOf, oldRunId);
assert.equal(requests, 5); assert.equal(reopened.core.tool.size, 4);
assert.deepEqual(observed.map(value => value.offset), [0, 669, 1662]);
const progress = reopened.core.task.get(taskId).readProgress;
assert.equal(progress.pause, false); assert.equal(progress.repeated, 0); assert.deepEqual(progress.reads, coverage);
assert.equal(progress.readWindow.runId, session.runId); assertToolPairs(session.providerMessages);
});