Files
Agent-Workbench/tests/execution.test.mjs
2026-10-29 21:09:50 +08:00

184 lines
14 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { Store, repairToolPairs } from '../src/server/store.mjs';
import { StateDatabase } from '../src/core/storage/state-database.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { fixture, workspace, call, until, assertToolPairs } from './helpers.mjs';
function legacyCase(f, name) {
f.store.message(f.session, 'user', 'Original fixture requirement');
fs.writeFileSync(path.join(f.projectRoot, 'worker-effect.mjs'), "import fs from 'node:fs'; const file='executions.txt'; const count=fs.existsSync(file)?Number(fs.readFileSync(file,'utf8')):0; fs.writeFileSync(file,String(count+1)); console.log('fixture-result');\n");
const directory = path.join(f.root, name); f.store.exportLegacyData(directory);
return directory;
}
async function crash(t, directory, phase) {
const env = { ...process.env }; delete env.AGENT_WORKBENCH_API_KEY;
const child = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/core-crash-worker.mjs'), directory, phase], { env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = ''; child.stdout.on('data', bytes => output += bytes.toString('utf8')); child.stderr.on('data', bytes => output += bytes.toString('utf8'));
t.after(() => { if (child.exitCode === null && child.signalCode === null) child.kill(); });
await until(() => child.exitCode !== null || child.signalCode !== null, 15000);
assert.equal(child.exitCode, 66, output);
}
function reopen(f, directory) { const store = new Store(directory); f.onCleanup(() => store.close()); return store; }
test('JSON migration survives process exit before commit and retains exact legacy backups', { timeout: 30000 }, async t => {
const f = fixture(t), directory = legacyCase(f, 'migration');
const sessionFile = path.join(directory, 'sessions', f.session.id + '.json');
const before = fs.readFileSync(sessionFile);
await crash(t, directory, 'migration');
const raw = new StateDatabase(directory);
assert.equal(raw.initialized, false); assert.equal(raw.cursor, 0); assert.equal(raw.load().sessions.length, 0); raw.close();
assert.deepEqual(fs.readFileSync(sessionFile), before);
const recovered = reopen(f, directory), session = recovered.session(f.session.id);
assert.equal(session.messages[0].content, 'Original fixture requirement');
assert.equal(recovered.core.task.size, 1);
const migration = recovered.storage.meta('migration');
assert.deepEqual(fs.readFileSync(path.join(directory, migration.backupRelative, 'sessions', f.session.id + '.json')), before);
const expected = structuredClone(session); recovered.close();
fs.writeFileSync(sessionFile, '{}');
const again = reopen(f, directory);
assert.deepEqual(again.session(f.session.id), expected, 'A stale JSON mirror cannot overwrite migrated SQLite state');
const exported = path.join(f.root, 'rollback-export'); again.exportLegacyData(exported);
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(exported, 'sessions', session.id + '.json'), 'utf8')), expected);
});
for (const point of ['after_state', 'after_event']) test('process exit ' + point + ' rolls back state, event and outbox together', { timeout: 30000 }, async t => {
const f = fixture(t), directory = legacyCase(f, point), initial = reopen(f, directory);
const before = structuredClone(initial.session(f.session.id)), cursor = initial.eventCursor;
initial.close(); await crash(t, directory, point);
const recovered = reopen(f, directory);
assert.deepEqual(recovered.session(f.session.id), before); assert.equal(recovered.eventCursor, cursor);
assert.equal(recovered.eventsAfter(cursor).events.length, 0);
});
for (const phase of ['accepted', 'queued', 'approval_wait', 'approved', 'effect_unrecorded', 'result_committed', 'before_next_model']) {
test('crash recovery at ' + phase + ' retains execution identity and never replays side effects', { timeout: 30000 }, async t => {
const f = fixture(t), directory = legacyCase(f, phase);
await crash(t, directory, phase);
const recovered = reopen(f, directory), session = recovered.session(f.session.id), execution = recovered.executionSnapshot(session);
assert.equal(session.status, 'paused'); assert.equal(execution.task.goal, 'Crash fixture task');
assert.equal(execution.run.status, 'paused'); assert.equal(execution.run.stopReason, 'restart');
const marker = path.join(f.projectRoot, 'executions.txt');
const effectOccurred = ['effect_unrecorded', 'result_committed', 'before_next_model'].includes(phase);
assert.equal(fs.existsSync(marker), effectOccurred);
if (effectOccurred) assert.equal(fs.readFileSync(marker, 'utf8'), '1');
assertToolPairs(session.providerMessages);
const invocation = [...recovered.core.tool.values()].find(value => value.callId === 'crash-' + phase);
if (phase !== 'accepted') {
assert.ok(invocation); assert.equal(invocation.runId, execution.run.id); assert.equal(invocation.taskId, execution.task.id);
if (['result_committed', 'before_next_model'].includes(phase)) {
assert.equal(invocation.state, 'completed'); assert.equal(invocation.requiresVerification, false);
const result = JSON.parse(session.providerMessages.find(value => value.tool_call_id === invocation.callId).content);
assert.equal(result.status, 'completed'); assert.match(result.stdout, /fixture-result/);
} else if (['approved', 'effect_unrecorded'].includes(phase)) {
assert.equal(invocation.state, 'unknown'); assert.equal(invocation.requiresVerification, true);
assert.ok(execution.run.checkpoint.requiresVerification.includes(invocation.id));
} else { assert.equal(invocation.state, 'interrupted'); assert.equal(invocation.requiresVerification, false); }
}
const oldApproval = session.approvals[0]?.id;
const provider = new Provider(); let requests = 0;
provider.complete = async () => { requests++; return { message: { role: 'assistant', content: 'Recovered records inspected.' }, usage: null }; };
const runtime = new Runtime(recovered, provider); f.onCleanup(() => runtime.stopAll());
if (oldApproval) assert.throws(() => runtime.decide(session.id, oldApproval, true), { status: 409 });
if (phase !== 'accepted') {
const previousRunId = session.runId, taskId = session.taskId;
runtime.resume(session.id); await runtime.active.get(session.id).promise;
assert.equal(requests, 1); assert.equal(session.taskId, taskId); assert.notEqual(session.runId, previousRunId);
assert.equal(recovered.core.run.get(session.runId).resumeOf, previousRunId);
if (oldApproval) assert.throws(() => runtime.decide(session.id, oldApproval, true), { status: 409 });
assert.equal(fs.existsSync(marker), effectOccurred);
if (effectOccurred) assert.equal(fs.readFileSync(marker, 'utf8'), '1');
}
});
}
test('state and event failures publish no success and preserve the last committed memory snapshot', t => {
const f = fixture(t), before = structuredClone(f.session), cursor = f.store.eventCursor;
let notifications = 0; f.store.on('update', () => notifications++);
f.store.storage.faultInjector = point => { if (point === 'after_event') throw new Error('Controlled transaction failure'); };
assert.throws(() => f.store.message(f.session, 'user', 'must not publish'), /Controlled transaction failure/);
assert.deepEqual(f.session, before); assert.equal(f.store.eventCursor, cursor); assert.equal(notifications, 0);
f.store.storage.faultInjector = null;
f.store.message(f.session, 'user', 'committed'); assert.equal(notifications, 1);
});
test('tool result artifacts and event cursors retain project ownership across restart', async t => {
const f = fixture(t), provider = new Provider(); let round = 0;
provider.complete = async () => ({ message: round++ === 0 ? { role: 'assistant', content: '', tool_calls: [call('owned-read', 'read_file', { path: 'README.md' })] }
: { role: 'assistant', content: 'Read complete.' }, usage: { prompt_tokens: 11, completion_tokens: 3 } });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
const initial = f.store.eventCursor;
runtime.start(f.session.id, 'Read the fixture'); await runtime.active.get(f.session.id).promise;
const snapshot = f.store.executionSnapshot(f.session);
assert.equal(snapshot.run.usage.requests, 2); assert.equal(snapshot.run.usage.totalTokens, 28);
const artifact = snapshot.artifacts.find(value => value.kind === 'tool-result'); assert.equal(artifact.projectId, f.project.id);
assert.match(JSON.parse(f.store.artifactStore.read(artifact)).content, /Fixture project/);
const other = f.store.createSession(f.project.id);
assert.throws(() => f.store.owned('artifact', artifact.id, other.id), { status: 404 });
const page = f.store.eventsAfter(initial, { sessionId: f.session.id, limit: 3 });
assert.equal(page.events.length, 3); assert.equal(page.hasMore, true);
assert.ok(page.events.every(event => event.projectId === f.project.id && event.sessionId === f.session.id));
const next = f.store.eventsAfter(page.nextCursor, { sessionId: f.session.id });
assert.ok(next.events.every(event => event.seq > page.nextCursor));
const all = [...page.events, ...next.events]; assert.equal(new Set(all.map(event => event.seq)).size, all.length);
const expectedCursor = f.store.eventCursor;
f.store.close(); const recovered = reopen(f, f.data);
assert.equal(recovered.eventCursor, expectedCursor);
assert.deepEqual(recovered.owned('artifact', artifact.id, f.session.id), artifact);
});
test('token budgets pause before tools execute and continuation keeps original task and accumulated usage', async t => {
const f = fixture(t), provider = new Provider(); let round = 0;
f.store.saveConfig({ maxRunTokens: 12000 });
provider.complete = async () => ({ message: { role: 'assistant', content: '', tool_calls: [call('budget-' + round++, 'run_command', { command: 'node --version', reason: 'must not execute over budget' })] },
usage: { prompt_tokens: 12020, completion_tokens: 10 } });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Original goal'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'paused'); assert.equal(f.session.approvals.length, 0);
const originalTaskId = f.session.taskId;
assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'token_limit');
assert.equal(f.store.core.task.get(originalTaskId).usage.totalTokens, 12030); assertToolPairs(f.session.providerMessages);
f.store.saveConfig({ maxRunTokens: 16000 });
provider.complete = async () => ({ message: { role: 'assistant', content: 'Finished.' }, usage: { prompt_tokens: 5, completion_tokens: 5 } });
runtime.start(f.session.id, 'Also preserve the public API.'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.taskId, originalTaskId); assert.equal(f.store.core.task.get(originalTaskId).goal, 'Original goal');
assert.equal(f.store.core.task.get(originalTaskId).instructions[0].content, 'Also preserve the public API.');
assert.equal(f.store.core.task.get(originalTaskId).usage.totalTokens, 12040);
assert.equal(f.store.core.run.get(f.session.runId).budget.maxTokens, 16000);
});
test('a time budget interrupts an unresponsive model and persists a resumable run', async t => {
const f = fixture(t), provider = new Provider(); f.store.saveConfig({ maxRunDurationMs: 80 });
provider.complete = async (_config, _messages, _project, signal) => new Promise((resolve, reject) => {
signal.addEventListener('abort', () => reject(signal.reason), { once: true });
if (signal.aborted) reject(signal.reason);
});
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Keep this goal after timeout'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'time_limit');
assert.match(f.session.messages.at(-1).content, /时间预算/); assert.equal(runtime.isActive(f.session.id), false);
});
test('a corrupted large result artifact cannot become a recovered success', t => {
const f = fixture(t), run = f.store.beginRun(f.session, 'Read a large fixture', false, f.store.config);
const tool = call('large-result', 'read_file', { path: 'README.md' });
f.session.providerMessages.push({ role: 'assistant', content: '', tool_calls: [tool] });
const event = { id: crypto.randomUUID(), kind: 'tool', tool: 'read_file', phase: 'queued', toolCallId: tool.id, runId: run.id, createdAt: new Date().toISOString() };
f.session.events.push(event); f.store.registerInvocation(f.session, tool, run, event);
f.store.beginInvocation(f.session, tool, run, event); f.store.invocationState(f.session, event, 'running');
event.phase = 'completed';
const artifact = f.store.finishInvocation(f.session, event, { status: 'completed', content: 'x'.repeat(20000) });
fs.writeFileSync(path.join(f.data, 'artifacts', artifact.relative), 'corrupted fixture result');
f.store.close(); const recovered = reopen(f, f.data), session = recovered.session(f.session.id);
const result = JSON.parse(session.providerMessages.find(value => value.tool_call_id === tool.id).content);
assert.equal(result.status, 'unknown'); assert.equal(result.requires_verification, true);
assert.equal(recovered.executionSnapshot(session).progress.requiresVerification.length, 1);
assert.equal(session.events.find(value => value.id === event.id).phase, 'unknown');
assert.equal(recovered.core.run.get(run.id).checkpoint.completedToolIds.includes(event.invocationId), false);
assert.throws(() => recovered.artifactStore.write(session, recovered.core.run.get(run.id), { status: 'completed', content: 'x'.repeat(20000) }), /已有执行工件校验失败/);
});