Files
Agent-Workbench/tests/request-plan.test.mjs

247 lines
20 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { createHash, randomUUID } from 'node:crypto';
import { spawn } from 'node:child_process';
import { RequestPlanner, historyGroups, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
import { requestMetrics } from '../src/shared/contracts/model-request.mjs';
import { Store } from '../src/server/store.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider, toolDefinitions } from '../src/server/provider.mjs';
import { fixture, call, assertToolPairs, compatibleFixture, completion, until, workspace } from './helpers.mjs';
const sha = text => createHash('sha256').update(text).digest('hex');
function setup(t, overrides = {}) {
const f = fixture(t), config = { ...f.store.config, ...overrides };
const run = f.store.beginRun(f.session, 'Keep the original engineering goal and public API.', false, config);
f.session.providerMessages.push({ role: 'user', content: 'Keep the original engineering goal and public API.' });
return { ...f, config, run, planner: new RequestPlanner(f.store), prepare(system = 'Pinned fixture goal, requirements and project identity.', tools = []) {
return this.planner.prepare(this.session, this.run, this.config, { system, tools });
} };
}
function append(f, index, content, args = { path: 'README.md' }, name = 'read_file') {
const id = 'history-' + index;
f.session.providerMessages.push({ role: 'assistant', content: 'Read evidence ' + index, tool_calls: [call(id, name, args)] },
{ role: 'tool', tool_call_id: id, content: JSON.stringify({ status: 'completed', path: 'README.md', content }) });
}
function archived(f, plan) {
return JSON.parse(f.store.artifactStore.read(f.store.owned('artifact', plan.record.historyIndexArtifactId, f.session.id)));
}
function assertBounded(plan) {
const metrics = requestMetrics(plan.body);
assert.equal(plan.record.requestBytes, Buffer.byteLength(JSON.stringify(plan.body), 'utf8'));
assert.equal(plan.record.requestSha256, metrics.sha256);
assert.equal(plan.record.estimatedInputTokens, metrics.estimatedInputTokens);
assert.ok(metrics.estimatedInputTokens + plan.record.budget.outputReservation + plan.record.budget.safetyTokens <= plan.record.budget.contextLimit);
assertToolPairs(plan.messages);
}
test('native seed framing reserves capacity before optional history without changing the selected window',t=>{
const f=setup(t,{contextLimit:6000,maxOutputTokens:512});
for(let i=0;i<4;i++)append(f,i,'Owned original observation.\n'.repeat(110));
const plan=f.planner.prepare(f.session,f.run,f.config,{system:'Keep complete task requirements.',tools:[],transportReservationTokens:2048});
assert.equal(plan.record.budget.contextLimit,6000);assert.equal(plan.record.budget.transportReservationTokens,2048);
assert.equal(plan.record.budget.inputBudget,6000-512-600-2048);assert.ok(plan.record.archivedGroups>0);
assert.ok(plan.messages.some(message=>message.role==='user'&&message.content==='Keep the original engineering goal and public API.'));
assert.ok(plan.record.estimatedInputTokens<=plan.record.budget.inputBudget);assert.doesNotThrow(()=>verifyPlanEvidence(f.store,f.session,plan.record));
});
test('direct code evidence remains selected while recent historical-index browsing is archived within the complete request budget', t => {
const f = setup(t, { contextLimit: 10000, maxOutputTokens: 2048 });
for (const name of ['normalize', 'summary', 'render', 'cli']) append(f, name, 'Actual implementation ' + name + '\n'.repeat(150), { path: 'src/' + name + '.mjs', start_line: 421, end_line: 450 });
const artifact = f.store.artifactStore.write(f.session, f.run, 'Index fixture', { kind: 'context-history-index' });
f.store.core.artifact.set(artifact.id, artifact);
for (let index = 0; index < 20; index++) append(f, 'index-' + index, 'metadata'.repeat(1500), { artifact_id: artifact.id, offset: index * 2000, limit: 2000 }, 'read_artifact');
append(f, 'latest-index', 'Recent bounded index page', { artifact_id: artifact.id, offset: 40000, limit: 2000 }, 'read_artifact');
const plan = f.prepare(); assertBounded(plan);
for (const name of ['normalize', 'summary', 'render', 'cli']) assert.ok(plan.messages.some(message => message.tool_calls?.some(value => value.id === 'history-' + name)), name + ' actual code group must remain');
assert.ok(plan.messages.some(message => message.tool_calls?.some(value => value.id === 'history-latest-index')));
assert.ok(plan.record.archivedGroups > 10); assert.ok(plan.record.mandatoryInputTokens < plan.record.budget.inputBudget);
assert.ok(plan.messages.some(message => message.role === 'system' && message.content.includes('working_evidence')));
});
test('one long user turn becomes bounded complete groups with exact UTF-8 history and recent JSON arguments', t => {
const f = setup(t);
for (let index = 0; index < 60; index++) append(f, index, ('行' + index + '🙂\r\n').repeat(1200), { path: 'README.md', payload: '原参数🙂'.repeat(800) });
append(f, 'latest', 'The most recent verified evidence.');
const original = structuredClone(f.session.providerMessages), plan = f.prepare();
assertBounded(plan); assert.ok(plan.record.archivedGroups > 40);
assert.ok(plan.record.requestBytes < plan.record.beforeBytes / 4);
assert.deepEqual(f.session.providerMessages, original);
assert.deepEqual(plan.messages.slice(-2), original.slice(-2));
for (const message of plan.messages) for (const tool of message.tool_calls ?? []) {
const saved = original.flatMap(message => message.tool_calls ?? []).find(value => value.id === tool.id);
assert.equal(tool.function.arguments, saved.function.arguments); assert.doesNotThrow(() => JSON.parse(tool.function.arguments));
}
const index = archived(f, plan); assert.equal(index.history_sha256, sha(JSON.stringify(original)));
for (const group of index.archivedGroups) {
const artifact = f.store.owned('artifact', group.archive.artifact_id, f.session.id);
const text = f.store.artifactStore.read(artifact);
assert.equal(sha(text), group.sha256); assert.deepEqual(JSON.parse(text), original.slice(group.startMessage, group.endMessage + 1));
}
assert.equal(fs.existsSync(path.join(f.projectRoot, 'WORKBENCH-NOTES.md')), false);
});
test('tool definitions, exact system bytes, output reserve and remaining Run/Task budget are accounted before requests', t => {
const f = setup(t, { contextLimit: 20000 });
const plan = f.prepare('目标🙂'.repeat(700), toolDefinitions); assertBounded(plan);
assert.ok(plan.record.requestBytes > Buffer.byteLength(JSON.stringify(f.session.providerMessages)) + JSON.stringify(toolDefinitions).length);
assert.equal(plan.body.max_tokens, f.config.maxOutputTokens);
assert.equal(plan.record.estimateMethod, 'utf8-third-plus-framing-v1');
f.run.budget.maxTokens = 100; assert.throws(() => f.prepare('', toolDefinitions), error => error.reason === 'token_limit');
f.run.budget.maxTokens = 200000; const task = f.store.core.task.get(f.run.taskId); task.budget.maxTokens = 150;
assert.throws(() => f.prepare('', toolDefinitions), error => error.reason === 'task_token_limit');
});
test('oversized required user instructions pause intact and can be explicitly resumed after increasing the window', async t => {
const f = fixture(t), provider = new Provider(); let requests = 0, sent;
provider.complete = async (_config, messages) => { requests++; sent = messages; return { message: { role: 'assistant', content: 'Requirements retained.' }, usage: { prompt_tokens: 100, completion_tokens: 20 } }; };
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
const goal = '必须保留完整需求🙂'.repeat(1200);
runtime.start(f.session.id, goal); await runtime.active.get(f.session.id).promise;
const taskId = f.session.taskId; assert.equal(requests, 0); assert.equal(f.session.status, 'paused');
assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'context_limit');
assert.equal(f.store.core.task.get(taskId).goal, goal); assert.equal(f.session.providerMessages[0].content, goal);
f.store.saveConfig({ contextLimit: 64000 });
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
assert.equal(requests, 1); assert.equal(f.session.taskId, taskId); assert.equal(f.session.status, 'completed');
assert.equal(sent.findLast(message => message.role === 'user').content, goal);
assert.ok(sent[0].content.includes(goal));
});
test('all task instructions, constraints and acceptance remain in the runtime system instead of an eight-item or byte prefix', async t => {
const f = fixture(t), provider = new Provider(); let round = 0, sent;
provider.complete = async (_config, messages) => { sent = messages; return { message: { role: 'assistant', content: 'Read requirements.' }, usage: { prompt_tokens: 30, completion_tokens: 5 } }; };
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
const run = f.store.beginRun(f.session, 'Original task goal.', false, f.store.config), task = f.store.core.task.get(run.taskId);
task.instructions = Array.from({ length: 12 }, (_, index) => ({ id: randomUUID(), content: 'Requirement ' + index + ' must remain.', createdAt: new Date().toISOString() }));
task.constraints = ['Do not change the protected fixture.']; task.acceptance = ['Four actual tests must pass.'];
f.session.providerMessages.push({ role: 'user', content: task.goal }); f.session.status = 'paused'; run.status = 'paused'; f.store.save(f.session);
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
for (const value of task.instructions) assert.ok(sent[0].content.includes(value.content));
for (const value of [...task.constraints, ...task.acceptance]) assert.ok(sent[0].content.includes(value));
assert.ok(sent[0].content.includes(f.project.id)); assert.ok(sent[0].content.includes(f.session.id));
});
test('a newest manageable multi-tool group stays exact even beyond the working target; oversized earlier groups are archived', t => {
const f = setup(t, { contextLimit: 50000 });
append(f, 'huge', 'a'.repeat(400000));
const first = call('pair-a', 'read_file', { path: 'a', source: 'x'.repeat(11000) }), second = call('pair-b', 'read_file', { path: 'b' });
const latest = [{ role: 'assistant', content: '', tool_calls: [first, second] },
{ role: 'tool', tool_call_id: second.id, content: JSON.stringify({ status: 'error', message: 'Actual failed read', details: 'y'.repeat(41000) }) },
{ role: 'tool', tool_call_id: first.id, content: JSON.stringify({ status: 'unknown', requires_verification: true }) }];
f.session.providerMessages.push(...latest);
const plan = f.prepare(); assertBounded(plan);
assert.ok(plan.record.estimatedInputTokens > plan.record.budget.targetInputTokens);
assert.deepEqual(plan.messages.slice(-3), latest);
});
for (const [label, messages] of [
['orphan result', [{ role: 'tool', tool_call_id: 'x', content: '{}' }]],
['missing result', [{ role: 'assistant', tool_calls: [call('x', 'read_file', {})] }]],
['wrong result', [{ role: 'assistant', tool_calls: [call('x', 'read_file', {})] }, { role: 'tool', tool_call_id: 'y', content: '{}' }]],
['duplicate call', [{ role: 'assistant', tool_calls: [call('x', 'read_file', {}), call('x', 'read_file', {})] }]],
]) test('invalid ' + label + ' is rejected before archive or network preparation', t => {
const f = setup(t); f.session.providerMessages.push(...messages);
assert.throws(() => f.prepare(), error => error.diagnostic?.stage === 'context_plan');
assert.equal(f.store.core.context_plan.size, 0); assert.equal(f.store.core.artifact.size, 0);
});
test('a group larger than the artifact limit uses verified Unicode-safe chunks without replacing tool JSON', t => {
const f = setup(t);
append(f, 'giant', 'actual text', { path: 'README.md', payload: '汉🙂'.repeat(1_100_000) });
const original = JSON.stringify(f.session.providerMessages.slice(-2));
assert.ok(Buffer.byteLength(original) > 7 * 1024 * 1024);
const plan = f.prepare(); assertBounded(plan); const group = archived(f, plan).archivedGroups.find(group => group.tools?.[0].call_id === 'history-giant');
verifyPlanEvidence(f.store, f.session, plan.record);
const manifest = JSON.parse(f.store.artifactStore.read(f.store.owned('artifact', group.archive.artifact_id, f.session.id)));
assert.equal(manifest.kind, 'context-group-chunks'); assert.ok(manifest.chunks.length > 1);
const restored = manifest.chunks.map(chunk => {
const text = f.store.artifactStore.read(f.store.owned('artifact', chunk.artifact_id, f.session.id));
assert.equal(sha(text), chunk.sha256); assert.equal(text, original.slice(chunk.offset, chunk.offset + chunk.characters)); return text;
}).join('');
assert.equal(restored, original); assert.equal(sha(restored), manifest.group_sha256);
assert.equal(JSON.parse(JSON.parse(restored)[0].tool_calls[0].function.arguments).payload, '汉🙂'.repeat(1_100_000));
});
test('changed or missing archived evidence stops reuse and is never silently recreated', t => {
const f = setup(t); append(f, 'large', 'Original durable evidence'.repeat(10000));
const plan = f.prepare(), group = archived(f, plan).archivedGroups.at(-1);
const artifact = f.store.owned('artifact', group.archive.artifact_id, f.session.id), file = path.join(f.data, 'artifacts', artifact.relative);
const original = structuredClone(f.session.providerMessages); fs.writeFileSync(file, 'external corruption');
assert.throws(() => f.prepare(), /工件校验失败/); assert.equal(fs.readFileSync(file, 'utf8'), 'external corruption');
assert.deepEqual(f.session.providerMessages, original);
fs.unlinkSync(file); assert.throws(() => f.prepare(), /ENOENT/); assert.equal(fs.existsSync(file), false);
});
test('transaction failure prevents publishing a plan and any corresponding model request', t => {
const f = setup(t), before = structuredClone(f.session); append(f, 'large', 'saved text'.repeat(10000)); f.store.save(f.session);
const saved = structuredClone(f.session); f.store.storage.faultInjector = point => { if (point === 'after_event') throw new Error('context transaction fault'); };
assert.throws(() => f.prepare(), /context transaction fault/);
assert.equal(f.store.core.context_plan.size, 0); assert.deepEqual(f.session, saved); f.store.storage.faultInjector = null;
});
test('the Provider rejects body mutation after a sealed plan before calling HTTP', async t => {
const f = setup(t), plan = f.prepare('Pinned system', toolDefinitions); let fetched = false;
const provider = new Provider({ fetch: async () => { fetched = true; throw new Error('Must not fetch'); } });
await assert.rejects(provider.complete({ ...f.config, mode: 'compatible', model: 'changed-model' }, plan.messages, f.project, null, null, plan.record), /正文与已保存计划不一致/);
assert.equal(fetched, false);
});
test('HTTP failure and explicit same-Task resume reuse exact completed results and request hashes without repeating a command', { timeout: 45000 }, async t => {
const f = fixture(t); fs.writeFileSync(path.join(f.projectRoot, 'README.md'), ('protected line🙂' + 'x'.repeat(190) + '\n').repeat(400));
let failed = false;
const model = await compatibleFixture(t, (_url, input, requests) => {
const record = [...f.store.core.context_plan.values()].at(-1); assert.equal(record.requestSha256, requestMetrics(input).sha256);
assert.ok(['prepared', 'requesting'].includes(record.status)); assertToolPairs(input.messages);
if (requests.length === 1) return completion([call('only-effect', 'run_command', { command: 'node -e "const fs=require(\'node:fs\'); const file=\'request-effects.txt\'; fs.writeFileSync(file,String(Number(fs.existsSync(file)?fs.readFileSync(file,\'utf8\'):0)+1));"', reason: 'Record one controlled fixture effect.' })]);
if (requests.length < 6) {
const start = (requests.length - 2) * 100 + 1;
return completion([call('read-round-' + requests.length, 'read_file', { path: 'README.md', start_line: start, end_line: start + 99 })]);
}
if (!failed) { failed = true; return { status: 503, body: { error: { message: 'controlled request failure' } } }; }
assert.ok(input.messages[0].content.includes('Do not replay the completed command.'));
return completion([], 'Original task completed from saved evidence.');
});
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture-model', maxSteps: 12 });
const provider = new Provider({ retryPolicy: { attempts: 1 } }), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Read evidence. Do not replay the completed command.');
await until(() => f.session.approvals.some(value => value.status === 'pending')); runtime.decide(f.session.id, f.session.approvals.at(-1).id, true);
await runtime.active.get(f.session.id).promise; assert.equal(f.session.status, 'failed');
assert.equal(model.requests.length, 6); assert.equal(failed, true);
const pages = f.session.events.filter(event => event.tool === 'read_file');
assert.deepEqual(pages.map(event => [event.input.start_line, event.input.end_line]), [[1, 100], [101, 200], [201, 300], [301, 400]]);
const original = structuredClone(f.session.providerMessages), taskId = f.session.taskId, firstRun = f.session.runId;
assert.ok([...f.store.core.context_plan.values()].some(plan => plan.archivedGroups > 0));
assert.equal([...f.store.core.context_plan.values()].at(-1).status, 'failed');
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'completed'); assert.equal(f.session.taskId, taskId); assert.notEqual(f.session.runId, firstRun);
assert.deepEqual(f.session.providerMessages.slice(0, original.length), original);
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'request-effects.txt'), 'utf8'), '1');
assert.equal(f.session.events.filter(event => event.toolCallId === 'only-effect').length, 1);
const plans = [...f.store.core.context_plan.values()]; assert.equal(plans.length, model.requests.length);
for (const plan of plans) {
assert.ok(plan.estimatedInputTokens + plan.budget.outputReservation + plan.budget.safetyTokens <= plan.budget.contextLimit);
const actual = model.requests.find(request => requestMetrics(request.input).sha256 === plan.requestSha256); assert.ok(actual);
}
});
test('a real process exit after plan persistence marks interruption and never issues an automatic model request', { timeout: 30000 }, async t => {
const f = fixture(t); f.store.close();
const env = { ...process.env }; delete env.AGENT_WORKBENCH_API_KEY;
const child = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/context-plan-crash-worker.mjs'), f.data, f.session.id], { env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = ''; child.stderr.on('data', data => output += data.toString('utf8'));
t.after(() => { if (child.exitCode === null && child.signalCode === null) child.kill(); });
await until(() => child.exitCode !== null || child.signalCode !== null, 15000); assert.equal(child.exitCode, 67, output);
const recovered = new Store(f.data); f.onCleanup(() => recovered.close());
const session = recovered.session(f.session.id), plans = [...recovered.core.context_plan.values()];
assert.equal(plans.length, 1); assert.equal(plans[0].status, 'interrupted'); assert.equal(plans[0].requestAttempts, 0);
assert.equal(session.status, 'paused'); assert.equal(recovered.core.run.get(session.runId).stopReason, 'restart');
assert.equal(recovered.core.tool.size, 0); assertToolPairs(session.providerMessages);
assert.equal(fs.existsSync(path.join(f.projectRoot, 'request-effects.txt')), false);
const planner = new RequestPlanner(recovered), run = recovered.beginRun(session, '', true, recovered.config);
const next = planner.prepare(session, run, recovered.config, { system: 'Preserve restarted Task.', tools: [] });
assert.equal(next.record.taskId, plans[0].taskId); assert.notEqual(next.record.runId, plans[0].runId);
});