import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { spawn } from 'node:child_process'; import { Store, repairToolPairs } from '../src/server/store.mjs'; import { StateDatabase } from '../src/core/storage/state-database.mjs'; import { Runtime } from '../src/server/runtime.mjs'; import { Provider } from '../src/server/provider.mjs'; import { fixture, workspace, call, until, assertToolPairs } from './helpers.mjs'; function legacyCase(f, name) { f.store.message(f.session, 'user', 'Original fixture requirement'); fs.writeFileSync(path.join(f.projectRoot, 'worker-effect.mjs'), "import fs from 'node:fs'; const file='executions.txt'; const count=fs.existsSync(file)?Number(fs.readFileSync(file,'utf8')):0; fs.writeFileSync(file,String(count+1)); console.log('fixture-result');\n"); const directory = path.join(f.root, name); f.store.exportLegacyData(directory); return directory; } async function crash(t, directory, phase) { const env = { ...process.env }; delete env.AGENT_WORKBENCH_API_KEY; const child = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/core-crash-worker.mjs'), directory, phase], { env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); let output = ''; child.stdout.on('data', bytes => output += bytes.toString('utf8')); child.stderr.on('data', bytes => output += bytes.toString('utf8')); t.after(() => { if (child.exitCode === null && child.signalCode === null) child.kill(); }); await until(() => child.exitCode !== null || child.signalCode !== null, 15000); assert.equal(child.exitCode, 66, output); } function reopen(f, directory) { const store = new Store(directory); f.onCleanup(() => store.close()); return store; } test('JSON migration survives process exit before commit and retains exact legacy backups', { timeout: 30000 }, async t => { const f = fixture(t), directory = legacyCase(f, 'migration'); const sessionFile = path.join(directory, 'sessions', f.session.id + '.json'); const before = fs.readFileSync(sessionFile); await crash(t, directory, 'migration'); const raw = new StateDatabase(directory); assert.equal(raw.initialized, false); assert.equal(raw.cursor, 0); assert.equal(raw.load().sessions.length, 0); raw.close(); assert.deepEqual(fs.readFileSync(sessionFile), before); const recovered = reopen(f, directory), session = recovered.session(f.session.id); assert.equal(session.messages[0].content, 'Original fixture requirement'); assert.equal(recovered.core.task.size, 1); const migration = recovered.storage.meta('migration'); assert.deepEqual(fs.readFileSync(path.join(directory, migration.backupRelative, 'sessions', f.session.id + '.json')), before); const expected = structuredClone(session); recovered.close(); fs.writeFileSync(sessionFile, '{}'); const again = reopen(f, directory); assert.deepEqual(again.session(f.session.id), expected, 'A stale JSON mirror cannot overwrite migrated SQLite state'); const exported = path.join(f.root, 'rollback-export'); again.exportLegacyData(exported); assert.deepEqual(JSON.parse(fs.readFileSync(path.join(exported, 'sessions', session.id + '.json'), 'utf8')), expected); }); for (const point of ['after_state', 'after_event']) test('process exit ' + point + ' rolls back state, event and outbox together', { timeout: 30000 }, async t => { const f = fixture(t), directory = legacyCase(f, point), initial = reopen(f, directory); const before = structuredClone(initial.session(f.session.id)), cursor = initial.eventCursor; initial.close(); await crash(t, directory, point); const recovered = reopen(f, directory); assert.deepEqual(recovered.session(f.session.id), before); assert.equal(recovered.eventCursor, cursor); assert.equal(recovered.eventsAfter(cursor).events.length, 0); }); for (const phase of ['accepted', 'queued', 'approval_wait', 'approved', 'effect_unrecorded', 'result_committed', 'before_next_model']) { test('crash recovery at ' + phase + ' retains execution identity and never replays side effects', { timeout: 30000 }, async t => { const f = fixture(t), directory = legacyCase(f, phase); await crash(t, directory, phase); const recovered = reopen(f, directory), session = recovered.session(f.session.id), execution = recovered.executionSnapshot(session); assert.equal(session.status, 'paused'); assert.equal(execution.task.goal, 'Crash fixture task'); assert.equal(execution.run.status, 'paused'); assert.equal(execution.run.stopReason, 'restart'); const marker = path.join(f.projectRoot, 'executions.txt'); const effectOccurred = ['effect_unrecorded', 'result_committed', 'before_next_model'].includes(phase); assert.equal(fs.existsSync(marker), effectOccurred); if (effectOccurred) assert.equal(fs.readFileSync(marker, 'utf8'), '1'); assertToolPairs(session.providerMessages); const invocation = [...recovered.core.tool.values()].find(value => value.callId === 'crash-' + phase); if (phase !== 'accepted') { assert.ok(invocation); assert.equal(invocation.runId, execution.run.id); assert.equal(invocation.taskId, execution.task.id); if (['result_committed', 'before_next_model'].includes(phase)) { assert.equal(invocation.state, 'completed'); assert.equal(invocation.requiresVerification, false); const result = JSON.parse(session.providerMessages.find(value => value.tool_call_id === invocation.callId).content); assert.equal(result.status, 'completed'); assert.match(result.stdout, /fixture-result/); } else if (['approved', 'effect_unrecorded'].includes(phase)) { assert.equal(invocation.state, 'unknown'); assert.equal(invocation.requiresVerification, true); assert.ok(execution.run.checkpoint.requiresVerification.includes(invocation.id)); } else { assert.equal(invocation.state, 'interrupted'); assert.equal(invocation.requiresVerification, false); } } const oldApproval = session.approvals[0]?.id; const provider = new Provider(); let requests = 0; provider.complete = async () => { requests++; return { message: { role: 'assistant', content: 'Recovered records inspected.' }, usage: null }; }; const runtime = new Runtime(recovered, provider); f.onCleanup(() => runtime.stopAll()); if (oldApproval) assert.throws(() => runtime.decide(session.id, oldApproval, true), { status: 409 }); if (phase !== 'accepted') { const previousRunId = session.runId, taskId = session.taskId; runtime.resume(session.id); await runtime.active.get(session.id).promise; assert.equal(requests, 1); assert.equal(session.taskId, taskId); assert.notEqual(session.runId, previousRunId); assert.equal(recovered.core.run.get(session.runId).resumeOf, previousRunId); if (oldApproval) assert.throws(() => runtime.decide(session.id, oldApproval, true), { status: 409 }); assert.equal(fs.existsSync(marker), effectOccurred); if (effectOccurred) assert.equal(fs.readFileSync(marker, 'utf8'), '1'); } }); } test('state and event failures publish no success and preserve the last committed memory snapshot', t => { const f = fixture(t), before = structuredClone(f.session), cursor = f.store.eventCursor; let notifications = 0; f.store.on('update', () => notifications++); f.store.storage.faultInjector = point => { if (point === 'after_event') throw new Error('Controlled transaction failure'); }; assert.throws(() => f.store.message(f.session, 'user', 'must not publish'), /Controlled transaction failure/); assert.deepEqual(f.session, before); assert.equal(f.store.eventCursor, cursor); assert.equal(notifications, 0); f.store.storage.faultInjector = null; f.store.message(f.session, 'user', 'committed'); assert.equal(notifications, 1); }); test('tool result artifacts and event cursors retain project ownership across restart', async t => { const f = fixture(t), provider = new Provider(); let round = 0; provider.complete = async () => ({ message: round++ === 0 ? { role: 'assistant', content: '', tool_calls: [call('owned-read', 'read_file', { path: 'README.md' })] } : { role: 'assistant', content: 'Read complete.' }, usage: { prompt_tokens: 11, completion_tokens: 3 } }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); const initial = f.store.eventCursor; runtime.start(f.session.id, 'Read the fixture'); await runtime.active.get(f.session.id).promise; const snapshot = f.store.executionSnapshot(f.session); assert.equal(snapshot.run.usage.requests, 2); assert.equal(snapshot.run.usage.totalTokens, 28); const artifact = snapshot.artifacts[0]; assert.equal(artifact.projectId, f.project.id); assert.match(JSON.parse(f.store.artifactStore.read(artifact)).content, /Fixture project/); const other = f.store.createSession(f.project.id); assert.throws(() => f.store.owned('artifact', artifact.id, other.id), { status: 404 }); const page = f.store.eventsAfter(initial, { sessionId: f.session.id, limit: 3 }); assert.equal(page.events.length, 3); assert.equal(page.hasMore, true); assert.ok(page.events.every(event => event.projectId === f.project.id && event.sessionId === f.session.id)); const next = f.store.eventsAfter(page.nextCursor, { sessionId: f.session.id }); assert.ok(next.events.every(event => event.seq > page.nextCursor)); const all = [...page.events, ...next.events]; assert.equal(new Set(all.map(event => event.seq)).size, all.length); const expectedCursor = f.store.eventCursor; f.store.close(); const recovered = reopen(f, f.data); assert.equal(recovered.eventCursor, expectedCursor); assert.deepEqual(recovered.owned('artifact', artifact.id, f.session.id), artifact); }); test('token budgets pause before tools execute and continuation keeps original task and accumulated usage', async t => { const f = fixture(t), provider = new Provider(); let round = 0; f.store.saveConfig({ maxRunTokens: 25 }); provider.complete = async () => ({ message: { role: 'assistant', content: '', tool_calls: [call('budget-' + round++, 'run_command', { command: 'node --version', reason: 'must not execute over budget' })] }, usage: { prompt_tokens: 20, completion_tokens: 10 } }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, 'Original goal'); await runtime.active.get(f.session.id).promise; assert.equal(f.session.status, 'paused'); assert.equal(f.session.approvals.length, 0); const originalTaskId = f.session.taskId; assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'token_limit'); assert.equal(f.store.core.task.get(originalTaskId).usage.totalTokens, 30); assertToolPairs(f.session.providerMessages); f.store.saveConfig({ maxRunTokens: 1000 }); provider.complete = async () => ({ message: { role: 'assistant', content: 'Finished.' }, usage: { prompt_tokens: 5, completion_tokens: 5 } }); runtime.start(f.session.id, 'Also preserve the public API.'); await runtime.active.get(f.session.id).promise; assert.equal(f.session.taskId, originalTaskId); assert.equal(f.store.core.task.get(originalTaskId).goal, 'Original goal'); assert.equal(f.store.core.task.get(originalTaskId).instructions[0].content, 'Also preserve the public API.'); assert.equal(f.store.core.task.get(originalTaskId).usage.totalTokens, 40); assert.equal(f.store.core.run.get(f.session.runId).budget.maxTokens, 1000); }); test('a time budget interrupts an unresponsive model and persists a resumable run', async t => { const f = fixture(t), provider = new Provider(); f.store.saveConfig({ maxRunDurationMs: 80 }); provider.complete = async (_config, _messages, _project, signal) => new Promise((resolve, reject) => { signal.addEventListener('abort', () => reject(signal.reason), { once: true }); if (signal.aborted) reject(signal.reason); }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, 'Keep this goal after timeout'); await runtime.active.get(f.session.id).promise; assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'time_limit'); assert.match(f.session.messages.at(-1).content, /时间预算/); assert.equal(runtime.isActive(f.session.id), false); }); test('a corrupted large result artifact cannot become a recovered success', t => { const f = fixture(t), run = f.store.beginRun(f.session, 'Read a large fixture', false, f.store.config); const tool = call('large-result', 'read_file', { path: 'README.md' }); f.session.providerMessages.push({ role: 'assistant', content: '', tool_calls: [tool] }); const event = { id: crypto.randomUUID(), kind: 'tool', tool: 'read_file', phase: 'queued', toolCallId: tool.id, runId: run.id, createdAt: new Date().toISOString() }; f.session.events.push(event); f.store.registerInvocation(f.session, tool, run, event); f.store.beginInvocation(f.session, tool, run, event); f.store.invocationState(f.session, event, 'running'); event.phase = 'completed'; const artifact = f.store.finishInvocation(f.session, event, { status: 'completed', content: 'x'.repeat(20000) }); fs.writeFileSync(path.join(f.data, 'artifacts', artifact.relative), 'corrupted fixture result'); f.store.close(); const recovered = reopen(f, f.data), session = recovered.session(f.session.id); const result = JSON.parse(session.providerMessages.find(value => value.tool_call_id === tool.id).content); assert.equal(result.status, 'unknown'); assert.equal(result.requires_verification, true); assert.equal(recovered.executionSnapshot(session).progress.requiresVerification.length, 1); });