Files
Agent-Workbench/tests/reminder-runtime.test.mjs

155 lines
14 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { fixture, compatibleFixture, call, completion, assertToolPairs, until } from './helpers.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Store } from '../src/server/store.mjs';
import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
import { DeliveryReporter } from '../src/core/delivery-report.mjs';
const marker = '工作台可选进展建议';
const hasAdvice = body => JSON.stringify(body).includes(marker);
const goal = '修改README.md,先读取核对原内容;无需测试。';
const script = n => n < 4 ? completion([call('read-' + n, 'read_file', { path: 'README.md', offset: 0, limit: 10 })])
: n === 4 ? completion([call('acknowledge-and-read', 'read_file', { path: 'README.md', offset: 0, limit: 10 })], '明白,已知悉。')
: n === 5 ? completion([call('new-range', 'read_file', { path: 'README.md', offset: 10, limit: 10 })]) : completion([], '读取过程结束,未修改源码。');
async function setup(t, enabled = true, respond = n => ({ body: script(n) })) {
const f = fixture(t, { modernDefaults: true }); let requests = 0;
const model = await compatibleFixture(t, (url, body) => respond(++requests, body));
f.store.saveConfig({ mode: 'compatible', model: 'reminder-local-fixture', baseUrl: model.baseUrl, progressRemindersEnabled: enabled });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
return { ...f, runtime, model, start(content = goal) { runtime.start(f.session.id, content); return runtime.active.get(f.session.id).promise; } };
}
test('default-off runtime preserves original history, tools and unlimited requests without advice receipts', async t => {
const f = await setup(t, false); await f.start();
const run = f.store.core.run.get(f.session.runId);
assert.equal(f.model.requests.length, 6); assert.ok(f.model.requests.every(row => !hasAdvice(row.input)));
assert.equal(run.progressReminders, undefined); assert.equal(run.budget.maxTokens, 0); assert.equal(run.stopReason, null);
for (const row of f.model.requests) assert.equal(row.input.max_tokens, undefined);
assertToolPairs(f.session.providerMessages); assert.equal(f.session.status, 'completed');
});
test('actual HTTP submission, separate acknowledgement and original readback survive reopening and verify every request', async t => {
const f = await setup(t); const original = fs.readFileSync(path.join(f.projectRoot, 'README.md')); await f.start();
const run = f.store.core.run.get(f.session.runId), item = run.progressReminders.items[0];
assert.deepEqual(f.model.requests.map(row => hasAdvice(row.input)), [false, false, false, true, false, false]);
assert.equal(item.deliveries.length, 1); assert.equal(item.deliveries[0].attempts.length, 1); assert.equal(item.status, 'resolved');
assert.equal(item.resolution.relation, 'observed_after_dispatch'); assert.equal(item.resolution.causality, 'unproven');
assert.ok(item.resolution.source.ordinal > item.deliveries[0].ordinal + 1, 'acknowledgement plus repeated read did not resolve advice');
assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, 'README.md')), original); assertToolPairs(f.session.providerMessages);
for (const plan of f.store.core.context_plan.values()) assert.doesNotThrow(() => verifyPlanEvidence(f.store, f.session, plan));
assert.equal(f.runtime.reminderManager().inspect(f.session, run).evidenceVerified, true);
const report = await new DeliveryReporter(f.store).create(f.session);
assert.equal(report.report.reminderObservations[0].evidenceVerified, true);
assert.equal(report.report.reminderObservations[0].items[0].status, 'resolved');
assert.match(f.store.artifactStore.read(f.store.core.artifact.get(report.markdownArtifactId)), /提醒与实际回读/);
assert.notEqual(report.report.status, 'verified_committed', 'reminder readback cannot prove file delivery');
const snapshot = structuredClone(run.progressReminders); f.store.close();
const reopened = new Store(f.data); f.onCleanup(() => reopened.close()); const nextRuntime = new Runtime(reopened, new Provider());
assert.deepEqual(reopened.core.run.get(run.id).progressReminders, snapshot);
assert.equal(reopened.config.progressRemindersEnabled, true);
assert.equal(nextRuntime.reminderManager().inspect(reopened.session(f.session.id), reopened.core.run.get(run.id)).evidenceVerified, true);
});
test('research and diagnosis remain legal read-only work without nudges', async t => {
const f = await setup(t); await f.start('只研究README.md的原内容,不修改文件。');
assert.ok(f.model.requests.every(row => !hasAdvice(row.input))); assert.equal(f.store.core.run.get(f.session.runId).progressReminders.items.length, 0);
assert.equal(f.session.status, 'completed');
});
test('turning off while a sent request waits preserves its dispatch but prevents later advice and readback resolution', async t => {
let release; const gate = new Promise(resolve => { release = resolve; });
const f = await setup(t, true, async n => { if (n === 4) await gate; return { body: script(n) }; });
const running = f.start(); await until(() => f.model.requests.length === 4, 15000);
const run = f.store.core.run.get(f.session.runId); assert.equal(run.progressReminders.items[0].status, 'delivered');
f.store.saveConfig({ progressRemindersEnabled: false }); f.runtime.progressRemindersChanged(); release(); await running;
const item = run.progressReminders.items[0]; assert.equal(item.status, 'expired'); assert.equal(item.exitReason, 'disabled'); assert.equal(item.resolution, undefined);
assert.equal(item.deliveries.length, 1); assert.deepEqual(f.model.requests.map(row => hasAdvice(row.input)), [false, false, false, true, false, false]);
assert.equal(f.runtime.reminderManager().inspect(f.session, run).evidenceVerified, true);
});
test('cancelled requests and a new Task cannot receive late reminder readback', async t => {
let release; const gate = new Promise(resolve => { release = resolve; });
const f = await setup(t, true, async n => { if (n === 4) await gate; return { body: script(n) }; });
const running = f.start(); await until(() => f.model.requests.length === 4, 15000); f.runtime.stop(f.session.id); await running; release();
const run = f.store.core.run.get(f.session.runId); assert.equal(run.stopReason, 'cancelled'); assert.equal(run.progressReminders.items[0].exitReason, 'cancelled');
const original = structuredClone(run.progressReminders); const other = f.store.beginRun(f.session, '新任务,仅查看历史', false, f.store.config, { forceNewTask: true });
assert.notEqual(other.taskId, run.taskId); f.runtime.reminderManager().observe(f.session, run);
assert.deepEqual(run.progressReminders, original); assert.equal(f.model.requests.length, 4);
});
test('reminder artifact failure does not fail or repeat original model requests or tools', async t => {
const f = await setup(t), write = f.store.artifactStore.write.bind(f.store.artifactStore);
f.store.artifactStore.write = (session, run, value, options) => {
if (options?.kind?.startsWith('progress-reminder')) throw new Error('injected reminder storage unavailable');
return write(session, run, value, options);
};
await f.start(); assert.equal(f.model.requests.length, 6); assert.equal(f.session.status, 'completed'); assert.equal(f.session.changes.length, 0);
assert.ok(f.model.requests.every(row => !hasAdvice(row.input))); assertToolPairs(f.session.providerMessages);
assert.equal(f.store.core.run.get(f.session.runId).progressReminderError.kind, 'reminder_unavailable');
});
test('optional request planning failure falls back to original complete request before dispatch', async t => {
const f = await setup(t), planner = f.runtime.requestPlanner(), prepare = planner.prepare.bind(planner); let failures = 0;
planner.prepare = (session, run, config, options) => { if (options.progressReminder) { failures++; throw new Error('injected optional planning failure'); } return prepare(session, run, config, options); };
await f.start(); assert.equal(f.model.requests.length, 6); assert.ok(failures > 0); assert.ok(f.model.requests.every(row => !hasAdvice(row.input)));
const run = f.store.core.run.get(f.session.runId); assert.equal(run.progressReminderError.phase, 'request_plan'); assert.equal(run.progressReminders.items[0].resolution.relation, 'naturally_resolved');
assert.equal(f.session.status, 'completed'); assertToolPairs(f.session.providerMessages);
});
test('actual invalid arguments resolve only after a successful original tool result', async t => {
const f = await setup(t, true, n => ({ body: n < 4 ? completion([call('invalid-' + n, 'read_file', { path: 17 })])
: n === 4 ? completion([call('corrected', 'read_file', { path: 'README.md', offset: 0, limit: 10 })]) : completion([], '读取已核对。') }));
await f.start(); const run = f.store.core.run.get(f.session.runId), item = run.progressReminders.items[0];
assert.equal(f.model.requests.length, 5); assert.equal(item.category, 'repeated_invalid_input'); assert.equal(item.status, 'resolved');
assert.equal(item.resolution.kind, 'corrected_tool_result'); assert.equal(f.runtime.reminderManager().inspect(f.session, run).evidenceVerified, true);
});
test('upstream error after submitting advice stays an upstream failure with unknown usage', async t => {
const f = await setup(t, true, n => n === 4 ? { status: 503, body: { error: { message: 'original supplier unavailable', code: 'D_FIXTURE_503' } } } : { body: script(n) });
await f.start(); const run = f.store.core.run.get(f.session.runId), item = run.progressReminders.items[0];
assert.equal(f.session.status, 'failed'); assert.equal(f.session.lastFailure.httpStatus, 503); assert.equal(f.model.requests.length, 4);
assert.equal(item.status, 'expired'); assert.equal(item.resolution, undefined); assert.equal(item.deliveries.length, 1);
assert.equal(run.usage.unknownRequests, 1); assert.equal(f.runtime.reminderManager().inspect(f.session, run).evidenceVerified, true);
});
test('tampered observation, queue, message or wire request remains unverified without granting completion', async t => {
const f = await setup(t); await f.start(); const run = f.store.core.run.get(f.session.runId), manager = f.runtime.reminderManager();
const item = run.progressReminders.items[0], request = f.store.core.context_plan.get(item.deliveries[0].requestId);
const original = request.progressReminderDispatches[0].wireRequestSha256; request.progressReminderDispatches[0].wireRequestSha256 = '0'.repeat(64);
assert.equal(manager.inspect(f.session, run).evidenceVerified, false); request.progressReminderDispatches[0].wireRequestSha256 = original;
const message = f.store.core.artifact.get(request.progressReminder.artifactId), old = message.sha256; message.sha256 = '0'.repeat(64);
assert.equal(manager.inspect(f.session, run).evidenceVerified, false); message.sha256 = old;
item.status = 'pending'; assert.equal(manager.inspect(f.session, run).evidenceVerified, false); item.status = 'resolved';
const planArtifactId = request.artifactId; request.artifactId = run.progressReminderReceiptId;
assert.equal(manager.inspect(f.session, run).evidenceVerified, false); request.artifactId = planArtifactId;
const receipt = f.store.core.artifact.get(item.source[0].artifactId), file = path.join(f.store.artifactStore.root, receipt.relative), bytes = fs.readFileSync(file);
fs.writeFileSync(file, 'source bytes corrupted'); assert.equal(manager.inspect(f.session, run).evidenceVerified, false); fs.writeFileSync(file, bytes);
assert.equal(manager.inspect(f.session, run).evidenceVerified, true);
});
for (const protocol of ['responses', 'anthropic-messages']) test(protocol + ' routes exact submitted advice through the selected native connection', async t => {
const f = fixture(t, { modernDefaults: true }); let n = 0;
const model = await compatibleFixture(t, () => {
const original = script(++n).choices[0].message, calls = original.tool_calls ?? [];
if (protocol === 'responses') return { body: { id: 'response-' + n, status: 'completed', model: 'owned-model',
output: [...(original.content ? [{ type: 'message', id: 'msg-' + n, status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: original.content }] }] : []),
...calls.map(call => ({ type: 'function_call', id: 'fn-' + call.id, call_id: call.id, status: 'completed', name: call.function.name, arguments: call.function.arguments }))],
usage: { input_tokens: 100, output_tokens: 20, total_tokens: 120 } } };
return { body: { type: 'message', id: 'msg-' + n, role: 'assistant', model: 'owned-model',
content: [...(original.content ? [{ type: 'text', text: original.content }] : []), ...calls.map(call => ({ type: 'tool_use', id: call.id, name: call.function.name, input: JSON.parse(call.function.arguments) }))],
stop_reason: calls.length ? 'tool_use' : 'end_turn', stop_sequence: null, usage: { input_tokens: 100, output_tokens: 20 } } };
});
const provider = new Provider(), hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Local D ' + protocol, model: 'owned-model', protocol, baseUrl: model.baseUrl, requiresKey: false });
hub.select({ type: 'connection', id: connection.id }); f.store.saveConfig({ mode: 'compatible', progressRemindersEnabled: true });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, goal); await runtime.active.get(f.session.id).promise;
const run = f.store.core.run.get(f.session.runId); assert.equal(f.session.status, 'completed'); assert.equal(model.requests.length, 6);
assert.deepEqual(model.requests.map(row => hasAdvice(row.input)), [false, false, false, true, false, false]);
assert.equal(run.progressReminders.items[0].deliveries[0].attempts.length, 1); assert.equal(runtime.reminderManager().inspect(f.session, run).evidenceVerified, true);
for (const plan of f.store.core.context_plan.values()) assert.doesNotThrow(() => verifyPlanEvidence(f.store, f.session, plan));
assertToolPairs(f.session.providerMessages);
});