Files
Agent-Workbench/tests/runtime.test.mjs
T

246 lines
16 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { Provider } from '../src/server/provider.mjs';
import { RequestPlanner } from '../src/core/context/request-plan.mjs';
import { Runtime, validateToolArgs } from '../src/server/runtime.mjs';
import { fixture, compatibleFixture, call, completion, until, assertToolPairs } from './helpers.mjs';
function commandProvider(command) {
const provider = new Provider();
provider.complete = async (_config, messages) => {
const latestUser = messages.findLastIndex(message => message.role === 'user');
if (!messages.slice(latestUser + 1).some(message => message.role === 'tool')) {
return { message: { role: 'assistant', content: '', tool_calls: [call('command-' + latestUser, 'run_command', { command, reason: 'Controlled fixture command' })] }, usage: null };
}
return { message: { role: 'assistant', content: 'Fixture finished.' }, usage: null };
};
return provider;
}
test('a stop from the event loop interrupts immediate model replies before later tools', async t => {
const f = fixture(t), provider = new Provider();
let replies = 0, stopped;
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
provider.complete = async () => {
replies++;
if (replies === 1) setImmediate(() => { stopped = runtime.stop(f.session.id); });
return { message: { role: 'assistant', content: 'Controlled immediate reply.',
...(replies <= 6 ? { tool_calls: [call('immediate-' + replies, 'list_files', { path: '', depth: 1 })] } : {}) }, usage: null };
};
runtime.start(f.session.id, 'Inspect this owned fixture; allow an incoming stop to take effect.');
await runtime.active.get(f.session.id).promise;
await new Promise(resolve => setImmediate(resolve)); await stopped;
assert.equal(f.session.status, 'paused', 'The event loop must process the stop before the task finishes.');
assert.equal(replies, 1, 'A completed tool must be persisted before stop prevents the next model reply.');
const tools = [...f.store.core.tool.values()].filter(value => value.sessionId === f.session.id);
assert.equal(tools.length, 1); assert.equal(tools[0].state, 'completed');
assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'cancelled');
assertToolPairs(f.session.providerMessages);
});
test('a stop after the first tool preserves a complete interrupted multi-call protocol group', async t => {
const f = fixture(t), provider = new Provider(), runtime = new Runtime(f.store, provider);
f.onCleanup(() => runtime.stopAll());
let scheduled = false, stopped;
f.store.on('update', () => {
if (!scheduled && f.session.events.some(value => value.toolCallId === 'fair-batch-0' && value.phase === 'completed')) {
scheduled = true; setImmediate(() => { stopped = runtime.stop(f.session.id); });
}
});
provider.complete = async () => ({ message: { role: 'assistant', content: 'Controlled immediate multi-tool reply.',
tool_calls: Array.from({ length: 6 }, (_, index) => call('fair-batch-' + index, 'list_files', { path: '', depth: 1 })) }, usage: null });
runtime.start(f.session.id, 'Inspect this fixture and accept a stop before later batched calls.');
await runtime.active.get(f.session.id).promise;
await new Promise(resolve => setImmediate(resolve)); await stopped;
assert.equal(f.session.status, 'paused');
const tools = [...f.store.core.tool.values()].filter(value => value.sessionId === f.session.id);
assert.equal(tools.length, 6); assert.equal(tools[0].state, 'completed');
assert.ok(tools.slice(1).every(value => value.state === 'interrupted' && !value.requiresVerification));
assertToolPairs(f.session.providerMessages);
assert.equal(f.session.providerMessages.filter(value => value.role === 'tool').length, 6);
});
test('the compatible HTTP loop preserves call IDs, validates errors and waits before executing commands', async t => {
const f = fixture(t);
const command = 'node -e "require(\'node:fs\').writeFileSync(\'approved.txt\',\'ok\'); console.log(\'fixture-command\')"';
let round = 0;
const model = await compatibleFixture(t, (_url, input) => {
assert.equal(input.model, 'fixture-agent'); assert.equal(input.stream, false);
assert.ok(input.tools.some(tool => tool.function.name === 'propose_file'));
assertToolPairs(input.messages);
round++;
if (round === 1) return completion([
call('list', 'list_files', { path: '', depth: 2 }),
call('malformed', 'search_files', { query: 42 }),
call('unknown', 'unknown_tool', {}),
]);
if (round === 2) {
assert.equal(JSON.parse(input.messages.find(item => item.tool_call_id === 'malformed').content).status, 'error');
assert.equal(JSON.parse(input.messages.find(item => item.tool_call_id === 'unknown').content).status, 'error');
return completion([call('read', 'read_file', { path: 'README.md' })]);
}
if (round === 3) return completion([
call('proposal', 'propose_file', { path: 'FIXTURE-NOTES.md', content: '# Fixture notes\n', reason: 'Controlled proposal' }),
call('command', 'run_command', { command, reason: 'Controlled fixture command' }),
]);
return completion([], 'Fixture task completed.');
});
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture-agent' });
const provider = new Provider(); provider.setKey('fixture-only-secret');
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Read the fixture and prepare a change.');
const promise = runtime.active.get(f.session.id).promise;
await until(() => f.session.status === 'awaiting_approval');
assert.equal(fs.existsSync(path.join(f.projectRoot, 'approved.txt')), false);
assert.equal(fs.existsSync(path.join(f.projectRoot, 'FIXTURE-NOTES.md')), false);
assert.equal(f.session.events.find(event => event.toolCallId === 'command').phase, 'waiting');
const approval = f.session.approvals[0];
runtime.decide(f.session.id, approval.id, true);
assert.throws(() => runtime.decide(f.session.id, approval.id, true), { status: 409 });
await promise;
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'approved.txt'), 'utf8'), 'ok');
assert.equal(f.session.status, 'needs_review');
assert.equal(f.session.usage.prompt_tokens, 123);
assertToolPairs(f.session.providerMessages);
assert.match(f.session.events.find(event => event.toolCallId === 'command').output, /fixture-command/);
assert.equal(runtime.isActive(f.session.id), false);
assert.ok(model.requests.every(request => request.headers.authorization === 'Bearer fixture-only-secret'));
assert.ok(!JSON.stringify(f.store.storage.load()).includes('fixture-only-secret'));
for (const artifact of f.store.core.artifact.values()) assert.ok(!f.store.artifactStore.read(artifact).includes('fixture-only-secret'));
assert.ok(!fs.readFileSync(path.join(f.data, 'config.json'), 'utf8').includes('fixture-only-secret'));
});
test('denial prevents command execution and produces a paired denied result', async t => {
const f = fixture(t), runtime = new Runtime(f.store, commandProvider('node --version'));
f.onCleanup(() => runtime.stopAll());
let executions = 0; runtime.executeCommand = () => { executions++; throw new Error('Must not execute'); };
runtime.start(f.session.id, 'test'); const promise = runtime.active.get(f.session.id).promise;
await until(() => f.session.status === 'awaiting_approval');
assert.throws(() => runtime.start(f.session.id, 'overlap'), { status: 409 });
runtime.decide(f.session.id, f.session.approvals[0].id, false); await promise;
assert.equal(executions, 0); assert.equal(f.session.events[0].phase, 'denied');
assert.equal(f.session.approvals[0].status, 'denied');
assertToolPairs(f.session.providerMessages);
});
test('stopping during approval interrupts pending calls without executing or replaying them', async t => {
const f = fixture(t), runtime = new Runtime(f.store, commandProvider('node --version'));
f.onCleanup(() => runtime.stopAll());
let executions = 0; runtime.executeCommand = () => { executions++; throw new Error('Must not execute'); };
runtime.start(f.session.id, 'test');
await until(() => f.session.status === 'awaiting_approval');
const approvalId = f.session.approvals[0].id;
await runtime.stop(f.session.id);
assert.equal(executions, 0); assert.equal(f.session.status, 'paused');
assert.equal(f.session.approvals[0].status, 'interrupted');
assert.equal(f.session.events[0].phase, 'interrupted');
assert.throws(() => runtime.decide(f.session.id, approvalId, true), { status: 409 });
assertToolPairs(f.session.providerMessages);
});
test('stopping a running fixture command terminates its own process tree and retains uncertain state', async t => {
const f = fixture(t);
fs.writeFileSync(path.join(f.projectRoot, 'wait.mjs'), "import fs from 'node:fs'; console.log('fixture-started'); setTimeout(() => { fs.writeFileSync('late.txt', 'late') }, 20000);\n");
const runtime = new Runtime(f.store, commandProvider('node wait.mjs')); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'test');
await until(() => f.session.status === 'awaiting_approval');
runtime.decide(f.session.id, f.session.approvals[0].id, true);
await until(() => f.session.events.some(event => event.output?.includes('fixture-started')));
const start = Date.now(); await runtime.stop(f.session.id);
assert.ok(Date.now() - start < 8000);
assert.equal(f.session.status, 'paused'); assert.equal(f.session.events[0].phase, 'interrupted');
assert.ok(f.session.events[0].executedAt); assert.equal(fs.existsSync(path.join(f.projectRoot, 'late.txt')), false);
assertToolPairs(f.session.providerMessages);
});
test('command output decodes split UTF-8 and excludes the workbench key from child environment', async t => {
const f = fixture(t), provider = new Provider(), runtime = new Runtime(f.store, provider);
f.onCleanup(() => runtime.stopAll());
fs.writeFileSync(path.join(f.projectRoot, 'utf8.mjs'), "const bytes = Buffer.from('中文完成'); for (const byte of bytes) { process.stdout.write(Buffer.from([byte])); await new Promise(r => setTimeout(r, 8)); } console.log(''); console.log(process.env.AGENT_WORKBENCH_API_KEY ? 'key-present' : 'key-absent');\n");
const previous = process.env.AGENT_WORKBENCH_API_KEY;
process.env.AGENT_WORKBENCH_API_KEY = 'fixture-child-environment-key';
try {
const result = await runtime.executeCommand('node utf8.mjs', f.projectRoot, new AbortController().signal, () => {});
assert.equal(result.status, 'completed'); assert.match(result.stdout, /中文完成/);
assert.match(result.stdout, /key-absent/); assert.ok(!result.stdout.includes('�'));
} finally {
if (previous === undefined) delete process.env.AGENT_WORKBENCH_API_KEY; else process.env.AGENT_WORKBENCH_API_KEY = previous;
}
});
test('model-list testing uses a tentative key without changing the saved connection and redacts service errors', async t => {
const model = await compatibleFixture(t, (url, _input, requests) => {
if (url === '/v1/models') return { data: [{ id: 'fixture-model' }] };
return { status: 401, body: { error: { message: 'Rejected fixture-saved-key' } } };
});
const provider = new Provider(); provider.setKey('fixture-saved-key');
const config = { mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture-model', maxOutputTokens: 256 };
assert.deepEqual(await provider.models(config, 'fixture-tentative-key'), ['fixture-model']);
assert.equal(provider.apiKey, 'fixture-saved-key');
assert.equal(model.requests[0].headers.authorization, 'Bearer fixture-tentative-key');
await assert.rejects(provider.complete(config, [{ role: 'user', content: 'test' }], null), error => error.message.includes('[已隐藏]') && !error.message.includes('fixture-saved-key'));
});
test('truncated or repeated model calls fail before execution; tool input validation rejects unknown capabilities', async t => {
const f = fixture(t);
const model = await compatibleFixture(t, () => ({ choices: [{ message: { role: 'assistant', tool_calls: [call('cut', 'run_command', { command: 'node --version', reason: 'test' })] }, finish_reason: 'length' }] }));
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture-model' });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'test'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'failed'); assert.equal(f.session.approvals.length, 0);
assertToolPairs(f.session.providerMessages);
assert.throws(() => validateToolArgs('__proto__', {}));
assert.throws(() => validateToolArgs('read_file', { path: 123 }));
assert.throws(() => validateToolArgs('list_files', { path: '', depth: -1 }));
const repeatedProvider = new Provider();
repeatedProvider.complete = async () => ({ message: { role: 'assistant', content: '', tool_calls: [call('same', 'list_files', { path: '' })] }, usage: null });
const other = new Runtime(f.store, repeatedProvider); f.onCleanup(() => other.stopAll());
other.start(f.session.id, 'another task'); await other.active.get(f.session.id).promise;
assert.equal(f.session.status, 'failed'); assert.match(f.session.messages.at(-1).content, /重复/);
assertToolPairs(f.session.providerMessages);
});
test('request projection keeps exact tool pairs and complete recent user instructions while preserving disk history', t => {
const f = fixture(t), config = { ...f.store.config, contextLimit: 32000 };
const run = f.store.beginRun(f.session, 'Preserve complete user instructions', false, config);
const messages = [];
for (let index = 0; index < 8; index++) {
messages.push({ role: 'user', content: index === 7 ? '最新指令'.repeat(2500) : 'User turn ' + index });
messages.push({ role: 'assistant', content: 'Answer', tool_calls: [call('read-' + index, 'read_file', { path: 'README.md', historical: 'x'.repeat(14000) })] });
messages.push({ role: 'tool', tool_call_id: 'read-' + index, content: 'result'.repeat(7000) });
}
f.session.providerMessages = messages;
const original = structuredClone(messages), projected = new RequestPlanner(f.store).prepare(f.session, run, config, { system: 'Current fixture task', tools: [] });
assert.ok(projected.record.archivedGroups > 0); assert.ok(projected.record.estimatedInputTokens < projected.record.beforeInputTokens);
assertToolPairs(projected.messages);
assert.equal(projected.messages.findLast(message => message.role === 'user').content, messages.findLast(message => message.role === 'user').content);
assert.deepEqual(messages, original);
for (const message of projected.messages) for (const tool of message.tool_calls ?? []) assert.doesNotThrow(() => JSON.parse(tool.function.arguments));
});
test('incoming mutations wait only for the ended run report and do not replay model work or wait on an active tool loop', async t => {
const f = fixture(t), provider = new Provider(); let releaseModel, releaseReport, requests = 0;
provider.complete = async () => {
requests++; await new Promise(resolve => { releaseModel = resolve; });
return { message: { role: 'assistant', content: 'Owned task finished.' }, usage: null };
};
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.finalizeDeliveryReport = async () => { await new Promise(resolve => { releaseReport = resolve; }); };
runtime.start(f.session.id, 'Inspect the owned fixture.');
try {
await until(() => releaseModel);
await runtime.awaitFinalization(f.session.id);
assert.equal(runtime.active.get(f.session.id).finalizing, undefined);
releaseModel(); await until(() => releaseReport);
let settled = false;
const waiting = runtime.awaitFinalization(f.session.id).then(() => { settled = true; });
await Promise.resolve(); assert.equal(settled, false);
assert.equal(runtime.active.get(f.session.id).finalizing, true);
releaseReport(); await waiting;
assert.equal(runtime.active.has(f.session.id), false); assert.equal(requests, 1);
assert.equal(f.session.status, 'completed');
} finally { releaseModel?.(); releaseReport?.(); }
});