Files
Agent-Workbench/tests/tool-feedback.test.mjs

110 lines
8.2 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { RequestPlanner, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
import { Provider, toolDefinitions } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { fixture, call, completion, compatibleFixture, until, assertToolPairs } from './helpers.mjs';
const sha = text => createHash('sha256').update(text).digest('hex');
const feedbackFrom = messages => messages.filter(message => message.role === 'system').map(message => {
try { return JSON.parse(message.content.split('\n').at(-1)); } catch { return {}; }
}).find(value => value.latest_tools)?.data;
const workingFrom = messages => messages.filter(message => message.role === 'system').map(message => {
try { return JSON.parse(message.content.split('\n').at(-1)); } catch { return {}; }
}).find(value => value.working_set)?.data;
function setup(t) {
const f = fixture(t), config = { ...f.store.config, contextLimit: 16000, maxOutputTokens: 2048 };
const run = f.store.beginRun(f.session, 'Repair four source modules; preserve tests and report actual verification.', false, config);
f.session.providerMessages.push({ role: 'user', content: f.store.core.task.get(run.taskId).goal });
const planner = new RequestPlanner(f.store);
const add = items => {
const calls = items.map(item => call(randomUUID(), item.tool, item.args ?? {})), results = [];
for (const [index, invocation] of calls.entries()) {
const event = f.store.event(f.session, { kind: 'tool', tool: invocation.function.name, toolCallId: invocation.id,
runId: run.id, phase: 'queued', label: 'Actual recorded tool result' });
f.store.registerInvocation(f.session, invocation, run, event); f.store.beginInvocation(f.session, invocation, run, event);
f.store.finishInvocation(f.session, event, items[index].result);
results.push({ role: 'tool', tool_call_id: invocation.id, content: JSON.stringify(items[index].result) });
}
f.session.providerMessages.push({ role: 'assistant', content: '', tool_calls: calls }, ...results);
f.store.save(f.session); return calls;
};
const prepare = () => planner.prepare(f.session, run, config, { system: 'Current project and sealed Task requirements.', tools: toolDefinitions });
return { ...f, run, add, prepare };
}
test('the latest oversized session result reaches the next request together with already read source modules', t => {
const f = setup(t);
const files = ['normalize', 'summary', 'render', 'cli'].map(name => ({ path: 'src/' + name + '.mjs', content: 'export const ' + name + ' = value => value;\n' }));
for (const file of files) f.add([{ tool: 'read_file', args: { path: file.path }, result: { ...file, sha256: sha(file.content), start_line: 1, end_line: 1 } }]);
const calls = f.add([{ tool: 'inspect_session', result: { source: 'agent_workbench_runtime',
session: { id: f.session.id, status: 'running' }, changes: [], lastFailure: null,
execution: { task: { id: f.run.taskId, status: 'running', verification: { status: 'failed', checkJobId: 'actual-failed-check' } },
progress: { totalTools: 5, completedTools: 4, requiresVerification: [] },
checks: [{ id: 'actual-failed-check', status: 'failed', results: [{ name: 'test', status: 'failed', exitCode: 1,
tests: { total: 14, passed: 1, failed: 13, skipped: 0, cancelled: 0 }, logArtifactId: 'actual-failure-log' }] }],
contextPlans: [{ request: 'Historical request metadata '.repeat(16000) }] } } }]);
const original = structuredClone(f.session.providerMessages), plan = f.prepare();
const feedback = feedbackFrom(plan.messages);
assert.ok(feedback, 'An oversized completed tool group must have mandatory actual-result feedback');
assert.equal(feedback.entries[0].tool, 'inspect_session');
assert.equal(feedback.entries[0].result.session.status, 'running');
assert.equal(feedback.entries[0].result.execution.task.verification.status, 'failed');
assert.equal(feedback.entries[0].result.execution.checks[0].results[0].tests.failed, 13);
const document = JSON.parse(f.store.artifactStore.read(f.store.owned('artifact', plan.record.latestToolsArtifactId, f.session.id)));
assert.equal(document.entries[0].source.call_id, calls[0].id);
const working = workingFrom(plan.messages);
for (const file of files) assert.equal(working.entries.find(entry => entry.observation.path === file.path)?.content, file.content);
assert.deepEqual(f.session.providerMessages, original); assertToolPairs(plan.messages);
assert.ok(plan.record.estimatedInputTokens <= plan.record.budget.inputBudget);
verifyPlanEvidence(f.store, f.session, plan.record);
});
test('every call in a large latest group keeps its actual outcome, including denied and unknown results', t => {
const f = setup(t);
f.add([{ tool: 'query_code', args: { query: 'module' }, result: { totalMatches: 1, results: [{ path: 'src/main.mjs', line: 4, name: 'actual' }],
index: { metadata: 'Oversized index '.repeat(24000) } } },
{ tool: 'run_command', args: { command: 'node actual.mjs', reason: 'Saved fixture result' }, result: { status: 'denied', message: 'Command was not executed.' } },
{ tool: 'propose_file', args: { path: 'src/main.mjs', content: 'Saved proposal', reason: 'Fixture proposal' },
result: { status: 'unknown', requires_verification: true, change_id: 'unconfirmed-change', message: 'Original result requires verification.' } }]);
const plan = f.prepare(), entries = feedbackFrom(plan.messages).entries;
assert.deepEqual(entries.map(entry => entry.tool), ['query_code', 'run_command', 'propose_file']);
assert.equal(entries[0].result.results[0].path, 'src/main.mjs');
assert.equal(entries[1].result.status, 'denied'); assert.equal(entries[2].result.requires_verification, true);
verifyPlanEvidence(f.store, f.session, plan.record);
const document = f.store.owned('artifact', plan.record.latestToolsArtifactId, f.session.id);
const saved = JSON.parse(f.store.artifactStore.read(document));
const original = f.store.owned('artifact', saved.entries[0].source.result_artifact_id, f.session.id);
fs.writeFileSync(path.join(f.data, 'artifacts', original.relative), 'changed original bytes');
assert.throws(() => verifyPlanEvidence(f.store, f.session, plan.record), /执行工件校验失败/);
});
test('a small complete latest tool group is sent once and needs no duplicate feedback view', t => {
const f = setup(t), calls = f.add([{ tool: 'inspect_session', result: { session: { id: f.session.id, status: 'running' } } }]);
const plan = f.prepare();
assert.equal(plan.record.latestToolsArtifactId, null); assert.equal(feedbackFrom(plan.messages), undefined);
assert.equal(plan.messages.filter(message => message.tool_call_id === calls[0].id).length, 1);
verifyPlanEvidence(f.store, f.session, plan.record);
f.add([{ tool: 'inspect_session', result: { session: { id: f.session.id, status: 'completed' } } }]);
verifyPlanEvidence(f.store, f.session, plan.record);
});
test('real HTTP repeated state inspection pauses after three identical engineering outcomes and preserves tool pairing', { timeout: 30000 }, async t => {
const f = fixture(t); let requests = 0;
const model = await compatibleFixture(t, (_route, body) => {
requests++; assertToolPairs(body.messages);
return { body: completion([call('state-query-' + requests, 'inspect_session', {})]) };
});
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'state-query-fixture', maxSteps: 12 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Inspect the state, then finish without changing files.');
await until(() => !runtime.isActive(f.session.id), 20000);
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'no_progress');
assert.equal(requests, 3); assert.equal(f.session.events.filter(event => event.tool === 'inspect_session' && event.phase === 'completed').length, 3);
assertToolPairs(f.session.providerMessages);
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'README.md'), 'utf8'), '# Fixture project\n\nA local test project with TODO work.\n');
});