247 lines
20 KiB
JavaScript
247 lines
20 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { spawn } from 'node:child_process';
|
|
import { RequestPlanner, historyGroups, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
|
import { requestMetrics } from '../src/shared/contracts/model-request.mjs';
|
|
import { Store } from '../src/server/store.mjs';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { Provider, toolDefinitions } from '../src/server/provider.mjs';
|
|
import { fixture, call, assertToolPairs, compatibleFixture, completion, until, workspace } from './helpers.mjs';
|
|
|
|
const sha = text => createHash('sha256').update(text).digest('hex');
|
|
function setup(t, overrides = {}) {
|
|
const f = fixture(t), config = { ...f.store.config, ...overrides };
|
|
const run = f.store.beginRun(f.session, 'Keep the original engineering goal and public API.', false, config);
|
|
f.session.providerMessages.push({ role: 'user', content: 'Keep the original engineering goal and public API.' });
|
|
return { ...f, config, run, planner: new RequestPlanner(f.store), prepare(system = 'Pinned fixture goal, requirements and project identity.', tools = []) {
|
|
return this.planner.prepare(this.session, this.run, this.config, { system, tools });
|
|
} };
|
|
}
|
|
function append(f, index, content, args = { path: 'README.md' }, name = 'read_file') {
|
|
const id = 'history-' + index;
|
|
f.session.providerMessages.push({ role: 'assistant', content: 'Read evidence ' + index, tool_calls: [call(id, name, args)] },
|
|
{ role: 'tool', tool_call_id: id, content: JSON.stringify({ status: 'completed', path: 'README.md', content }) });
|
|
}
|
|
function archived(f, plan) {
|
|
return JSON.parse(f.store.artifactStore.read(f.store.owned('artifact', plan.record.historyIndexArtifactId, f.session.id)));
|
|
}
|
|
function assertBounded(plan) {
|
|
const metrics = requestMetrics(plan.body);
|
|
assert.equal(plan.record.requestBytes, Buffer.byteLength(JSON.stringify(plan.body), 'utf8'));
|
|
assert.equal(plan.record.requestSha256, metrics.sha256);
|
|
assert.equal(plan.record.estimatedInputTokens, metrics.estimatedInputTokens);
|
|
assert.ok(metrics.estimatedInputTokens + plan.record.budget.outputReservation + plan.record.budget.safetyTokens <= plan.record.budget.contextLimit);
|
|
assertToolPairs(plan.messages);
|
|
}
|
|
|
|
test('native seed framing reserves capacity before optional history without changing the selected window',t=>{
|
|
const f=setup(t,{contextLimit:6000,maxOutputTokens:512});
|
|
for(let i=0;i<4;i++)append(f,i,'Owned original observation.\n'.repeat(110));
|
|
const plan=f.planner.prepare(f.session,f.run,f.config,{system:'Keep complete task requirements.',tools:[],transportReservationTokens:2048});
|
|
assert.equal(plan.record.budget.contextLimit,6000);assert.equal(plan.record.budget.transportReservationTokens,2048);
|
|
assert.equal(plan.record.budget.inputBudget,6000-512-600-2048);assert.ok(plan.record.archivedGroups>0);
|
|
assert.ok(plan.messages.some(message=>message.role==='user'&&message.content==='Keep the original engineering goal and public API.'));
|
|
assert.ok(plan.record.estimatedInputTokens<=plan.record.budget.inputBudget);assert.doesNotThrow(()=>verifyPlanEvidence(f.store,f.session,plan.record));
|
|
});
|
|
|
|
test('direct code evidence remains selected while recent historical-index browsing is archived within the complete request budget', t => {
|
|
const f = setup(t, { contextLimit: 10000, maxOutputTokens: 2048 });
|
|
for (const name of ['normalize', 'summary', 'render', 'cli']) append(f, name, 'Actual implementation ' + name + '\n'.repeat(150), { path: 'src/' + name + '.mjs', start_line: 421, end_line: 450 });
|
|
const artifact = f.store.artifactStore.write(f.session, f.run, 'Index fixture', { kind: 'context-history-index' });
|
|
f.store.core.artifact.set(artifact.id, artifact);
|
|
for (let index = 0; index < 20; index++) append(f, 'index-' + index, 'metadata'.repeat(1500), { artifact_id: artifact.id, offset: index * 2000, limit: 2000 }, 'read_artifact');
|
|
append(f, 'latest-index', 'Recent bounded index page', { artifact_id: artifact.id, offset: 40000, limit: 2000 }, 'read_artifact');
|
|
const plan = f.prepare(); assertBounded(plan);
|
|
for (const name of ['normalize', 'summary', 'render', 'cli']) assert.ok(plan.messages.some(message => message.tool_calls?.some(value => value.id === 'history-' + name)), name + ' actual code group must remain');
|
|
assert.ok(plan.messages.some(message => message.tool_calls?.some(value => value.id === 'history-latest-index')));
|
|
assert.ok(plan.record.archivedGroups > 10); assert.ok(plan.record.mandatoryInputTokens < plan.record.budget.inputBudget);
|
|
assert.ok(plan.messages.some(message => message.role === 'system' && message.content.includes('working_evidence')));
|
|
});
|
|
|
|
test('one long user turn becomes bounded complete groups with exact UTF-8 history and recent JSON arguments', t => {
|
|
const f = setup(t);
|
|
for (let index = 0; index < 60; index++) append(f, index, ('行' + index + '🙂\r\n').repeat(1200), { path: 'README.md', payload: '原参数🙂'.repeat(800) });
|
|
append(f, 'latest', 'The most recent verified evidence.');
|
|
const original = structuredClone(f.session.providerMessages), plan = f.prepare();
|
|
assertBounded(plan); assert.ok(plan.record.archivedGroups > 40);
|
|
assert.ok(plan.record.requestBytes < plan.record.beforeBytes / 4);
|
|
assert.deepEqual(f.session.providerMessages, original);
|
|
assert.deepEqual(plan.messages.slice(-2), original.slice(-2));
|
|
for (const message of plan.messages) for (const tool of message.tool_calls ?? []) {
|
|
const saved = original.flatMap(message => message.tool_calls ?? []).find(value => value.id === tool.id);
|
|
assert.equal(tool.function.arguments, saved.function.arguments); assert.doesNotThrow(() => JSON.parse(tool.function.arguments));
|
|
}
|
|
const index = archived(f, plan); assert.equal(index.history_sha256, sha(JSON.stringify(original)));
|
|
for (const group of index.archivedGroups) {
|
|
const artifact = f.store.owned('artifact', group.archive.artifact_id, f.session.id);
|
|
const text = f.store.artifactStore.read(artifact);
|
|
assert.equal(sha(text), group.sha256); assert.deepEqual(JSON.parse(text), original.slice(group.startMessage, group.endMessage + 1));
|
|
}
|
|
assert.equal(fs.existsSync(path.join(f.projectRoot, 'WORKBENCH-NOTES.md')), false);
|
|
});
|
|
|
|
test('tool definitions, exact system bytes, output reserve and remaining Run/Task budget are accounted before requests', t => {
|
|
const f = setup(t, { contextLimit: 20000 });
|
|
const plan = f.prepare('目标🙂'.repeat(700), toolDefinitions); assertBounded(plan);
|
|
assert.ok(plan.record.requestBytes > Buffer.byteLength(JSON.stringify(f.session.providerMessages)) + JSON.stringify(toolDefinitions).length);
|
|
assert.equal(plan.body.max_tokens, f.config.maxOutputTokens);
|
|
assert.equal(plan.record.estimateMethod, 'utf8-third-plus-framing-v1');
|
|
f.run.budget.maxTokens = 100; assert.throws(() => f.prepare('', toolDefinitions), error => error.reason === 'token_limit');
|
|
f.run.budget.maxTokens = 200000; const task = f.store.core.task.get(f.run.taskId); task.budget.maxTokens = 150;
|
|
assert.throws(() => f.prepare('', toolDefinitions), error => error.reason === 'task_token_limit');
|
|
});
|
|
|
|
test('oversized required user instructions pause intact and can be explicitly resumed after increasing the window', async t => {
|
|
const f = fixture(t), provider = new Provider(); let requests = 0, sent;
|
|
provider.complete = async (_config, messages) => { requests++; sent = messages; return { message: { role: 'assistant', content: 'Requirements retained.' }, usage: { prompt_tokens: 100, completion_tokens: 20 } }; };
|
|
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
const goal = '必须保留完整需求🙂'.repeat(1200);
|
|
runtime.start(f.session.id, goal); await runtime.active.get(f.session.id).promise;
|
|
const taskId = f.session.taskId; assert.equal(requests, 0); assert.equal(f.session.status, 'paused');
|
|
assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'context_limit');
|
|
assert.equal(f.store.core.task.get(taskId).goal, goal); assert.equal(f.session.providerMessages[0].content, goal);
|
|
f.store.saveConfig({ contextLimit: 64000 });
|
|
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
|
|
assert.equal(requests, 1); assert.equal(f.session.taskId, taskId); assert.equal(f.session.status, 'completed');
|
|
assert.equal(sent.findLast(message => message.role === 'user').content, goal);
|
|
assert.ok(sent[0].content.includes(goal));
|
|
});
|
|
|
|
test('all task instructions, constraints and acceptance remain in the runtime system instead of an eight-item or byte prefix', async t => {
|
|
const f = fixture(t), provider = new Provider(); let round = 0, sent;
|
|
provider.complete = async (_config, messages) => { sent = messages; return { message: { role: 'assistant', content: 'Read requirements.' }, usage: { prompt_tokens: 30, completion_tokens: 5 } }; };
|
|
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
const run = f.store.beginRun(f.session, 'Original task goal.', false, f.store.config), task = f.store.core.task.get(run.taskId);
|
|
task.instructions = Array.from({ length: 12 }, (_, index) => ({ id: randomUUID(), content: 'Requirement ' + index + ' must remain.', createdAt: new Date().toISOString() }));
|
|
task.constraints = ['Do not change the protected fixture.']; task.acceptance = ['Four actual tests must pass.'];
|
|
f.session.providerMessages.push({ role: 'user', content: task.goal }); f.session.status = 'paused'; run.status = 'paused'; f.store.save(f.session);
|
|
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
|
|
for (const value of task.instructions) assert.ok(sent[0].content.includes(value.content));
|
|
for (const value of [...task.constraints, ...task.acceptance]) assert.ok(sent[0].content.includes(value));
|
|
assert.ok(sent[0].content.includes(f.project.id)); assert.ok(sent[0].content.includes(f.session.id));
|
|
});
|
|
|
|
test('a newest manageable multi-tool group stays exact even beyond the working target; oversized earlier groups are archived', t => {
|
|
const f = setup(t, { contextLimit: 50000 });
|
|
append(f, 'huge', 'a'.repeat(400000));
|
|
const first = call('pair-a', 'read_file', { path: 'a', source: 'x'.repeat(11000) }), second = call('pair-b', 'read_file', { path: 'b' });
|
|
const latest = [{ role: 'assistant', content: '', tool_calls: [first, second] },
|
|
{ role: 'tool', tool_call_id: second.id, content: JSON.stringify({ status: 'error', message: 'Actual failed read', details: 'y'.repeat(41000) }) },
|
|
{ role: 'tool', tool_call_id: first.id, content: JSON.stringify({ status: 'unknown', requires_verification: true }) }];
|
|
f.session.providerMessages.push(...latest);
|
|
const plan = f.prepare(); assertBounded(plan);
|
|
assert.ok(plan.record.estimatedInputTokens > plan.record.budget.targetInputTokens);
|
|
assert.deepEqual(plan.messages.slice(-3), latest);
|
|
});
|
|
|
|
for (const [label, messages] of [
|
|
['orphan result', [{ role: 'tool', tool_call_id: 'x', content: '{}' }]],
|
|
['missing result', [{ role: 'assistant', tool_calls: [call('x', 'read_file', {})] }]],
|
|
['wrong result', [{ role: 'assistant', tool_calls: [call('x', 'read_file', {})] }, { role: 'tool', tool_call_id: 'y', content: '{}' }]],
|
|
['duplicate call', [{ role: 'assistant', tool_calls: [call('x', 'read_file', {}), call('x', 'read_file', {})] }]],
|
|
]) test('invalid ' + label + ' is rejected before archive or network preparation', t => {
|
|
const f = setup(t); f.session.providerMessages.push(...messages);
|
|
assert.throws(() => f.prepare(), error => error.diagnostic?.stage === 'context_plan');
|
|
assert.equal(f.store.core.context_plan.size, 0); assert.equal(f.store.core.artifact.size, 0);
|
|
});
|
|
|
|
test('a group larger than the artifact limit uses verified Unicode-safe chunks without replacing tool JSON', t => {
|
|
const f = setup(t);
|
|
append(f, 'giant', 'actual text', { path: 'README.md', payload: '汉🙂'.repeat(1_100_000) });
|
|
const original = JSON.stringify(f.session.providerMessages.slice(-2));
|
|
assert.ok(Buffer.byteLength(original) > 7 * 1024 * 1024);
|
|
const plan = f.prepare(); assertBounded(plan); const group = archived(f, plan).archivedGroups.find(group => group.tools?.[0].call_id === 'history-giant');
|
|
verifyPlanEvidence(f.store, f.session, plan.record);
|
|
const manifest = JSON.parse(f.store.artifactStore.read(f.store.owned('artifact', group.archive.artifact_id, f.session.id)));
|
|
assert.equal(manifest.kind, 'context-group-chunks'); assert.ok(manifest.chunks.length > 1);
|
|
const restored = manifest.chunks.map(chunk => {
|
|
const text = f.store.artifactStore.read(f.store.owned('artifact', chunk.artifact_id, f.session.id));
|
|
assert.equal(sha(text), chunk.sha256); assert.equal(text, original.slice(chunk.offset, chunk.offset + chunk.characters)); return text;
|
|
}).join('');
|
|
assert.equal(restored, original); assert.equal(sha(restored), manifest.group_sha256);
|
|
assert.equal(JSON.parse(JSON.parse(restored)[0].tool_calls[0].function.arguments).payload, '汉🙂'.repeat(1_100_000));
|
|
});
|
|
|
|
test('changed or missing archived evidence stops reuse and is never silently recreated', t => {
|
|
const f = setup(t); append(f, 'large', 'Original durable evidence'.repeat(10000));
|
|
const plan = f.prepare(), group = archived(f, plan).archivedGroups.at(-1);
|
|
const artifact = f.store.owned('artifact', group.archive.artifact_id, f.session.id), file = path.join(f.data, 'artifacts', artifact.relative);
|
|
const original = structuredClone(f.session.providerMessages); fs.writeFileSync(file, 'external corruption');
|
|
assert.throws(() => f.prepare(), /工件校验失败/); assert.equal(fs.readFileSync(file, 'utf8'), 'external corruption');
|
|
assert.deepEqual(f.session.providerMessages, original);
|
|
fs.unlinkSync(file); assert.throws(() => f.prepare(), /ENOENT/); assert.equal(fs.existsSync(file), false);
|
|
});
|
|
|
|
test('transaction failure prevents publishing a plan and any corresponding model request', t => {
|
|
const f = setup(t), before = structuredClone(f.session); append(f, 'large', 'saved text'.repeat(10000)); f.store.save(f.session);
|
|
const saved = structuredClone(f.session); f.store.storage.faultInjector = point => { if (point === 'after_event') throw new Error('context transaction fault'); };
|
|
assert.throws(() => f.prepare(), /context transaction fault/);
|
|
assert.equal(f.store.core.context_plan.size, 0); assert.deepEqual(f.session, saved); f.store.storage.faultInjector = null;
|
|
});
|
|
|
|
test('the Provider rejects body mutation after a sealed plan before calling HTTP', async t => {
|
|
const f = setup(t), plan = f.prepare('Pinned system', toolDefinitions); let fetched = false;
|
|
const provider = new Provider({ fetch: async () => { fetched = true; throw new Error('Must not fetch'); } });
|
|
await assert.rejects(provider.complete({ ...f.config, mode: 'compatible', model: 'changed-model' }, plan.messages, f.project, null, null, plan.record), /正文与已保存计划不一致/);
|
|
assert.equal(fetched, false);
|
|
});
|
|
|
|
test('HTTP failure and explicit same-Task resume reuse exact completed results and request hashes without repeating a command', { timeout: 45000 }, async t => {
|
|
const f = fixture(t); fs.writeFileSync(path.join(f.projectRoot, 'README.md'), ('protected line🙂' + 'x'.repeat(190) + '\n').repeat(400));
|
|
let failed = false;
|
|
const model = await compatibleFixture(t, (_url, input, requests) => {
|
|
const record = [...f.store.core.context_plan.values()].at(-1); assert.equal(record.requestSha256, requestMetrics(input).sha256);
|
|
assert.ok(['prepared', 'requesting'].includes(record.status)); assertToolPairs(input.messages);
|
|
if (requests.length === 1) return completion([call('only-effect', 'run_command', { command: 'node -e "const fs=require(\'node:fs\'); const file=\'request-effects.txt\'; fs.writeFileSync(file,String(Number(fs.existsSync(file)?fs.readFileSync(file,\'utf8\'):0)+1));"', reason: 'Record one controlled fixture effect.' })]);
|
|
if (requests.length < 6) {
|
|
const start = (requests.length - 2) * 100 + 1;
|
|
return completion([call('read-round-' + requests.length, 'read_file', { path: 'README.md', start_line: start, end_line: start + 99 })]);
|
|
}
|
|
if (!failed) { failed = true; return { status: 503, body: { error: { message: 'controlled request failure' } } }; }
|
|
assert.ok(input.messages[0].content.includes('Do not replay the completed command.'));
|
|
return completion([], 'Original task completed from saved evidence.');
|
|
});
|
|
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture-model', maxSteps: 12 });
|
|
const provider = new Provider({ retryPolicy: { attempts: 1 } }), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
runtime.start(f.session.id, 'Read evidence. Do not replay the completed command.');
|
|
await until(() => f.session.approvals.some(value => value.status === 'pending')); runtime.decide(f.session.id, f.session.approvals.at(-1).id, true);
|
|
await runtime.active.get(f.session.id).promise; assert.equal(f.session.status, 'failed');
|
|
assert.equal(model.requests.length, 6); assert.equal(failed, true);
|
|
const pages = f.session.events.filter(event => event.tool === 'read_file');
|
|
assert.deepEqual(pages.map(event => [event.input.start_line, event.input.end_line]), [[1, 100], [101, 200], [201, 300], [301, 400]]);
|
|
const original = structuredClone(f.session.providerMessages), taskId = f.session.taskId, firstRun = f.session.runId;
|
|
assert.ok([...f.store.core.context_plan.values()].some(plan => plan.archivedGroups > 0));
|
|
assert.equal([...f.store.core.context_plan.values()].at(-1).status, 'failed');
|
|
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
|
|
assert.equal(f.session.status, 'completed'); assert.equal(f.session.taskId, taskId); assert.notEqual(f.session.runId, firstRun);
|
|
assert.deepEqual(f.session.providerMessages.slice(0, original.length), original);
|
|
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'request-effects.txt'), 'utf8'), '1');
|
|
assert.equal(f.session.events.filter(event => event.toolCallId === 'only-effect').length, 1);
|
|
const plans = [...f.store.core.context_plan.values()]; assert.equal(plans.length, model.requests.length);
|
|
for (const plan of plans) {
|
|
assert.ok(plan.estimatedInputTokens + plan.budget.outputReservation + plan.budget.safetyTokens <= plan.budget.contextLimit);
|
|
const actual = model.requests.find(request => requestMetrics(request.input).sha256 === plan.requestSha256); assert.ok(actual);
|
|
}
|
|
});
|
|
|
|
test('a real process exit after plan persistence marks interruption and never issues an automatic model request', { timeout: 30000 }, async t => {
|
|
const f = fixture(t); f.store.close();
|
|
const env = { ...process.env }; delete env.AGENT_WORKBENCH_API_KEY;
|
|
const child = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/context-plan-crash-worker.mjs'), f.data, f.session.id], { env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
let output = ''; child.stderr.on('data', data => output += data.toString('utf8'));
|
|
t.after(() => { if (child.exitCode === null && child.signalCode === null) child.kill(); });
|
|
await until(() => child.exitCode !== null || child.signalCode !== null, 15000); assert.equal(child.exitCode, 67, output);
|
|
const recovered = new Store(f.data); f.onCleanup(() => recovered.close());
|
|
const session = recovered.session(f.session.id), plans = [...recovered.core.context_plan.values()];
|
|
assert.equal(plans.length, 1); assert.equal(plans[0].status, 'interrupted'); assert.equal(plans[0].requestAttempts, 0);
|
|
assert.equal(session.status, 'paused'); assert.equal(recovered.core.run.get(session.runId).stopReason, 'restart');
|
|
assert.equal(recovered.core.tool.size, 0); assertToolPairs(session.providerMessages);
|
|
assert.equal(fs.existsSync(path.join(f.projectRoot, 'request-effects.txt')), false);
|
|
const planner = new RequestPlanner(recovered), run = recovered.beginRun(session, '', true, recovered.config);
|
|
const next = planner.prepare(session, run, recovered.config, { system: 'Preserve restarted Task.', tools: [] });
|
|
assert.equal(next.record.taskId, plans[0].taskId); assert.notEqual(next.record.runId, plans[0].runId);
|
|
});
|