110 lines
8.2 KiB
JavaScript
110 lines
8.2 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { RequestPlanner, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
|
import { Provider, toolDefinitions } from '../src/server/provider.mjs';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { fixture, call, completion, compatibleFixture, until, assertToolPairs } from './helpers.mjs';
|
|
|
|
const sha = text => createHash('sha256').update(text).digest('hex');
|
|
const feedbackFrom = messages => messages.filter(message => message.role === 'system').map(message => {
|
|
try { return JSON.parse(message.content.split('\n').at(-1)); } catch { return {}; }
|
|
}).find(value => value.latest_tools)?.data;
|
|
const workingFrom = messages => messages.filter(message => message.role === 'system').map(message => {
|
|
try { return JSON.parse(message.content.split('\n').at(-1)); } catch { return {}; }
|
|
}).find(value => value.working_set)?.data;
|
|
function setup(t) {
|
|
const f = fixture(t), config = { ...f.store.config, contextLimit: 16000, maxOutputTokens: 2048 };
|
|
const run = f.store.beginRun(f.session, 'Repair four source modules; preserve tests and report actual verification.', false, config);
|
|
f.session.providerMessages.push({ role: 'user', content: f.store.core.task.get(run.taskId).goal });
|
|
const planner = new RequestPlanner(f.store);
|
|
const add = items => {
|
|
const calls = items.map(item => call(randomUUID(), item.tool, item.args ?? {})), results = [];
|
|
for (const [index, invocation] of calls.entries()) {
|
|
const event = f.store.event(f.session, { kind: 'tool', tool: invocation.function.name, toolCallId: invocation.id,
|
|
runId: run.id, phase: 'queued', label: 'Actual recorded tool result' });
|
|
f.store.registerInvocation(f.session, invocation, run, event); f.store.beginInvocation(f.session, invocation, run, event);
|
|
f.store.finishInvocation(f.session, event, items[index].result);
|
|
results.push({ role: 'tool', tool_call_id: invocation.id, content: JSON.stringify(items[index].result) });
|
|
}
|
|
f.session.providerMessages.push({ role: 'assistant', content: '', tool_calls: calls }, ...results);
|
|
f.store.save(f.session); return calls;
|
|
};
|
|
const prepare = () => planner.prepare(f.session, run, config, { system: 'Current project and sealed Task requirements.', tools: toolDefinitions });
|
|
return { ...f, run, add, prepare };
|
|
}
|
|
|
|
test('the latest oversized session result reaches the next request together with already read source modules', t => {
|
|
const f = setup(t);
|
|
const files = ['normalize', 'summary', 'render', 'cli'].map(name => ({ path: 'src/' + name + '.mjs', content: 'export const ' + name + ' = value => value;\n' }));
|
|
for (const file of files) f.add([{ tool: 'read_file', args: { path: file.path }, result: { ...file, sha256: sha(file.content), start_line: 1, end_line: 1 } }]);
|
|
const calls = f.add([{ tool: 'inspect_session', result: { source: 'agent_workbench_runtime',
|
|
session: { id: f.session.id, status: 'running' }, changes: [], lastFailure: null,
|
|
execution: { task: { id: f.run.taskId, status: 'running', verification: { status: 'failed', checkJobId: 'actual-failed-check' } },
|
|
progress: { totalTools: 5, completedTools: 4, requiresVerification: [] },
|
|
checks: [{ id: 'actual-failed-check', status: 'failed', results: [{ name: 'test', status: 'failed', exitCode: 1,
|
|
tests: { total: 14, passed: 1, failed: 13, skipped: 0, cancelled: 0 }, logArtifactId: 'actual-failure-log' }] }],
|
|
contextPlans: [{ request: 'Historical request metadata '.repeat(16000) }] } } }]);
|
|
const original = structuredClone(f.session.providerMessages), plan = f.prepare();
|
|
const feedback = feedbackFrom(plan.messages);
|
|
assert.ok(feedback, 'An oversized completed tool group must have mandatory actual-result feedback');
|
|
assert.equal(feedback.entries[0].tool, 'inspect_session');
|
|
assert.equal(feedback.entries[0].result.session.status, 'running');
|
|
assert.equal(feedback.entries[0].result.execution.task.verification.status, 'failed');
|
|
assert.equal(feedback.entries[0].result.execution.checks[0].results[0].tests.failed, 13);
|
|
const document = JSON.parse(f.store.artifactStore.read(f.store.owned('artifact', plan.record.latestToolsArtifactId, f.session.id)));
|
|
assert.equal(document.entries[0].source.call_id, calls[0].id);
|
|
const working = workingFrom(plan.messages);
|
|
for (const file of files) assert.equal(working.entries.find(entry => entry.observation.path === file.path)?.content, file.content);
|
|
assert.deepEqual(f.session.providerMessages, original); assertToolPairs(plan.messages);
|
|
assert.ok(plan.record.estimatedInputTokens <= plan.record.budget.inputBudget);
|
|
verifyPlanEvidence(f.store, f.session, plan.record);
|
|
});
|
|
|
|
test('every call in a large latest group keeps its actual outcome, including denied and unknown results', t => {
|
|
const f = setup(t);
|
|
f.add([{ tool: 'query_code', args: { query: 'module' }, result: { totalMatches: 1, results: [{ path: 'src/main.mjs', line: 4, name: 'actual' }],
|
|
index: { metadata: 'Oversized index '.repeat(24000) } } },
|
|
{ tool: 'run_command', args: { command: 'node actual.mjs', reason: 'Saved fixture result' }, result: { status: 'denied', message: 'Command was not executed.' } },
|
|
{ tool: 'propose_file', args: { path: 'src/main.mjs', content: 'Saved proposal', reason: 'Fixture proposal' },
|
|
result: { status: 'unknown', requires_verification: true, change_id: 'unconfirmed-change', message: 'Original result requires verification.' } }]);
|
|
const plan = f.prepare(), entries = feedbackFrom(plan.messages).entries;
|
|
assert.deepEqual(entries.map(entry => entry.tool), ['query_code', 'run_command', 'propose_file']);
|
|
assert.equal(entries[0].result.results[0].path, 'src/main.mjs');
|
|
assert.equal(entries[1].result.status, 'denied'); assert.equal(entries[2].result.requires_verification, true);
|
|
verifyPlanEvidence(f.store, f.session, plan.record);
|
|
const document = f.store.owned('artifact', plan.record.latestToolsArtifactId, f.session.id);
|
|
const saved = JSON.parse(f.store.artifactStore.read(document));
|
|
const original = f.store.owned('artifact', saved.entries[0].source.result_artifact_id, f.session.id);
|
|
fs.writeFileSync(path.join(f.data, 'artifacts', original.relative), 'changed original bytes');
|
|
assert.throws(() => verifyPlanEvidence(f.store, f.session, plan.record), /执行工件校验失败/);
|
|
});
|
|
|
|
test('a small complete latest tool group is sent once and needs no duplicate feedback view', t => {
|
|
const f = setup(t), calls = f.add([{ tool: 'inspect_session', result: { session: { id: f.session.id, status: 'running' } } }]);
|
|
const plan = f.prepare();
|
|
assert.equal(plan.record.latestToolsArtifactId, null); assert.equal(feedbackFrom(plan.messages), undefined);
|
|
assert.equal(plan.messages.filter(message => message.tool_call_id === calls[0].id).length, 1);
|
|
verifyPlanEvidence(f.store, f.session, plan.record);
|
|
f.add([{ tool: 'inspect_session', result: { session: { id: f.session.id, status: 'completed' } } }]);
|
|
verifyPlanEvidence(f.store, f.session, plan.record);
|
|
});
|
|
|
|
test('real HTTP repeated state inspection pauses after three identical engineering outcomes and preserves tool pairing', { timeout: 30000 }, async t => {
|
|
const f = fixture(t); let requests = 0;
|
|
const model = await compatibleFixture(t, (_route, body) => {
|
|
requests++; assertToolPairs(body.messages);
|
|
return { body: completion([call('state-query-' + requests, 'inspect_session', {})]) };
|
|
});
|
|
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'state-query-fixture', maxSteps: 12 });
|
|
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
|
|
runtime.start(f.session.id, 'Inspect the state, then finish without changing files.');
|
|
await until(() => !runtime.isActive(f.session.id), 20000);
|
|
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'no_progress');
|
|
assert.equal(requests, 3); assert.equal(f.session.events.filter(event => event.tool === 'inspect_session' && event.phase === 'completed').length, 3);
|
|
assertToolPairs(f.session.providerMessages);
|
|
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'README.md'), 'utf8'), '# Fixture project\n\nA local test project with TODO work.\n');
|
|
});
|