337 lines
32 KiB
JavaScript
337 lines
32 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { spawn, execFileSync } from 'node:child_process';
|
|
import { fixture, call, until, workspace } from './helpers.mjs';
|
|
import { Store } from '../src/server/store.mjs';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { readOwnedProcessTree } from '../src/core/owned-process-tree.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { workerUsage } from '../src/shared/contracts/workers.mjs';
|
|
import { proposeFile } from '../src/server/files.mjs';
|
|
import { prepareWorkerFixture, workerDefinition, completedWorkers, ownerBytes } from './fixtures/workers-project.mjs';
|
|
import { deliveryGit, writeDeliveryFile } from './fixtures/delivery-project.mjs';
|
|
|
|
const alive = pid => { try { process.kill(pid, 0); return true; } catch { return false; } };
|
|
async function ready(t, options = {}, definition = workerDefinition()) {
|
|
const f = fixture(t), context = prepareWorkerFixture(f, options), group = await context.manager.plan(f.session, definition);
|
|
await (await context.manager.prepare(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.status, 'ready', group.summary); return { f, ...context, group };
|
|
}
|
|
function reopen(f, id) {
|
|
f.store.close(); const store = new Store(f.data), provider = new Provider(), requests = [];
|
|
provider.complete = async () => { requests.push('unexpected'); throw new Error('Recovery must never request a model'); };
|
|
const runtime = new Runtime(store, provider), manager = runtime.workerManager(), group = store.core.collaboration.get(id), session = store.session(group.sessionId);
|
|
f.onCleanup(() => store.close()); f.onCleanup(() => runtime.stopAll());
|
|
return { store, provider, requests, runtime, manager, group, session };
|
|
}
|
|
async function evidence(manager, session, group) {
|
|
await (await manager.reconcile(session, group.id, group.fingerprint)).promise;
|
|
assert.ok(group.evidence, group.summary); return manager.readEvidence(session, group);
|
|
}
|
|
function longCheckProject(f) {
|
|
writeDeliveryFile(f, 'test/add.test.mjs', "import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { spawn } from 'node:child_process'; import { add } from '../src/add.mjs';\nconst child = spawn(process.execPath, ['-e', \"const fs = require('node:fs'); setInterval(() => fs.appendFileSync('.awb-heartbeat', 'tick'), 30);\"], { windowsHide: true, stdio: 'ignore' }); fs.writeFileSync('.awb-child-pid', String(child.pid));\ntest('owned long running functional check', async () => { assert.equal(add(2, 3), 5); await new Promise(resolve => setTimeout(resolve, 45000)); child.kill(); });\n");
|
|
deliveryGit(f, ['add', '--', 'test/add.test.mjs']); deliveryGit(f, ['commit', '--no-gpg-sign', '-m', 'Own a cancellable check tree']);
|
|
}
|
|
|
|
test('stopping a parent accounts late responses and executes none of their proposed tools', async t => {
|
|
const releases = [], context = await ready(t, { respond: ({ project }) => new Promise(resolve => releases.push(() => resolve({
|
|
message: { role: 'assistant', tool_calls: [call(randomUUID(), 'propose_file', { path: 'src/a.mjs', content: 'late', reason: 'late' })] }, usage: { total_tokens: 91 },
|
|
}))) });
|
|
const { f, manager, runtime, group } = context, initial = ownerBytes(f);
|
|
const started = manager.start(f.session, group.id, group.fingerprint); await until(() => releases.length === 2);
|
|
const stopping = runtime.stop(f.session.id); await until(() => group.status === 'stopping');
|
|
for (const release of releases) release(); await stopping; await started.promise;
|
|
assert.equal(group.status, 'paused', group.summary); assert.equal(runtime.hasActivity(), false); assert.deepEqual(ownerBytes(f), initial);
|
|
for (const child of group.children) {
|
|
const worker = f.store.core.worker.get(child.id), session = f.store.session(child.sessionId);
|
|
assert.equal(session.changes.length, 0); assert.equal(workerUsage(f.store, worker).totalTokens, 91); assert.equal(workerUsage(f.store, worker).requests, 1);
|
|
assert.equal(workerUsage(f.store, worker).toolCalls, 0); assert.equal(fs.existsSync(path.join(f.store.project(child.projectId).root, 'src/a.mjs')), false);
|
|
}
|
|
});
|
|
|
|
test('unreported interrupted requests reserve estimated usage and the total deadline cannot reset on resume', async t => {
|
|
let bothDispatched;
|
|
const dispatched = new Promise(resolve => { bothDispatched = resolve; });
|
|
const { f, manager, group, requests } = await ready(t, { respond: ({ signal }) => new Promise((resolve, reject) => {
|
|
if (requests.length === 2) bothDispatched();
|
|
const stop = () => reject(Object.assign(new Error('Owned fixture aborted without upstream usage'), { name: 'AbortError' }));
|
|
signal.addEventListener('abort', stop, { once: true }); if (signal.aborted) stop();
|
|
}) }, { ...workerDefinition(), budget: { maxDurationMs: 1000 } });
|
|
// Workspace verification may use the whole one-second budget on Windows.
|
|
// Advance the deadline only after both actual fixture requests are in flight.
|
|
t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: Date.now() });
|
|
const started = manager.start(f.session, group.id, group.fingerprint);
|
|
await Promise.race([dispatched, started.promise.then(() => assert.fail('Workers ended before the two fixture requests were dispatched.'))]);
|
|
t.mock.timers.tick(1001); await started.promise;
|
|
const deadline = group.deadline; assert.equal(requests.length, 2); assert.ok(Date.now() >= deadline);
|
|
for (const child of group.children) { const usage = workerUsage(f.store, f.store.core.worker.get(child.id)); assert.equal(usage.unknownRequests, 1); assert.ok(usage.estimatedUnknownTokens > 0); }
|
|
await (await manager.start(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.deadline, deadline); assert.equal(requests.length, 2); assert.match(group.summary, /时限/);
|
|
});
|
|
|
|
test('a deadline before worker request dispatch records no invented usage and cannot reset on resume', async t => {
|
|
const { f, runtime, manager, group, requests } = await ready(t, {}, { ...workerDefinition(), budget: { maxDurationMs: 1000 } });
|
|
let release, bothEntered, count = 0;
|
|
const held = new Promise(resolve => { release = resolve; }), entered = new Promise(resolve => { bothEntered = resolve; });
|
|
const run = runtime.run.bind(runtime);
|
|
runtime.run = async (...args) => { if (++count === 2) bothEntered(); await held; return run(...args); };
|
|
t.after(() => release());
|
|
t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: Date.now() });
|
|
const started = manager.start(f.session, group.id, group.fingerprint);
|
|
await Promise.race([entered, started.promise.then(() => assert.fail('Workers ended before reaching the controlled pre-dispatch boundary.'))]);
|
|
t.mock.timers.tick(1001); release(); await started.promise;
|
|
const deadline = group.deadline; assert.equal(requests.length, 0); assert.ok(Date.now() >= deadline);
|
|
for (const child of group.children) {
|
|
const usage = workerUsage(f.store, f.store.core.worker.get(child.id));
|
|
assert.equal(usage.requests, 0); assert.equal(usage.unknownRequests, 0); assert.equal(usage.estimatedUnknownTokens, 0);
|
|
}
|
|
await (await manager.start(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.deadline, deadline); assert.equal(requests.length, 0); assert.match(group.summary, /时限/);
|
|
});
|
|
|
|
test('a response exceeding the cumulative token allocation stops before registering or executing its tools', async t => {
|
|
// Keep the request sendable as the tool contract grows; exceed the same explicit
|
|
// per-worker allocation by one reported token to exercise response accounting.
|
|
const allocation = 20000;
|
|
const { f, manager, group, requests } = await ready(t, { respond: async () => ({ message: { role: 'assistant', tool_calls: [call(randomUUID(), 'propose_file', { path: 'src/a.mjs', content: 'over budget', reason: 'fixture' })] }, usage: { total_tokens: allocation + 1 } }) }, { ...workerDefinition(), budget: { maxTokens: allocation * 2 } });
|
|
await (await manager.start(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.status, 'paused', group.summary);
|
|
assert.equal(requests.length, 2, 'Both responses must actually be dispatched before testing their recorded usage.');
|
|
for (const child of group.children) { const worker = f.store.core.worker.get(child.id); assert.equal(workerUsage(f.store, worker).accountedTokens, allocation + 1); assert.equal(workerUsage(f.store, worker).toolCalls, 0); assert.equal(f.store.session(child.sessionId).changes.length, 0); }
|
|
await (await manager.start(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(requests.length, 2); assert.match(group.summary, /累计预算/);
|
|
});
|
|
|
|
for (const [label, budget, count] of [['cumulative tool allocation', { maxToolCalls: 2 }, 2], ['single response 16 tool ceiling', {}, 17]]) {
|
|
test(label + ' rejects the entire returned batch before any tool side effect', async t => {
|
|
const { f, manager, group } = await ready(t, { respond: async () => ({ message: { role: 'assistant', tool_calls: Array.from({ length: count }, () => call(randomUUID(), 'inspect_project', {})) }, usage: { total_tokens: 100 } }) }, { ...workerDefinition(), budget });
|
|
await (await manager.start(f.session, group.id, group.fingerprint)).promise;
|
|
for (const child of group.children) { const worker = f.store.core.worker.get(child.id); assert.equal(workerUsage(f.store, worker).toolCalls, 0); assert.equal(f.store.session(child.sessionId).changes.length, 0); assert.match(f.store.core.run.get(f.store.session(child.sessionId).runId).stopReason, /worker_tool_limit/); }
|
|
});
|
|
}
|
|
|
|
test('consumed request steps remain consumed across explicit worker resumes', async t => {
|
|
const { f, manager, group, requests } = await ready(t, { respond: async () => ({ message: { role: 'assistant', content: 'No verified work' }, usage: { total_tokens: 10 } }) }, { ...workerDefinition(), budget: { maxSteps: 2 } });
|
|
await (await manager.start(f.session, group.id, group.fingerprint)).promise;
|
|
await (await manager.start(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(requests.length, 2); assert.match(group.summary, /累计预算/);
|
|
for (const child of group.children) assert.equal(workerUsage(f.store, f.store.core.worker.get(child.id)).steps, 1);
|
|
});
|
|
|
|
test('a saved completed response with missing usage is conservatively accounted exactly once during recovery', async t => {
|
|
const { f, group } = await ready(t), worker = f.store.core.worker.get(group.children[0].id), session = f.store.session(worker.sessionId), run = f.store.core.run.get(session.runId);
|
|
const record = { id: randomUUID(), projectId: session.projectId, sessionId: session.id, taskId: session.taskId, runId: run.id,
|
|
status: 'completed', workerDispatchedAt: new Date().toISOString(), estimatedInputTokens: 1234, workerUsageAccounted: false };
|
|
f.store.putRecord('context_plan', session, record, 'fixture.response_saved_before_usage');
|
|
const first = reopen(f, group.id), recovered = first.store.core.context_plan.get(record.id);
|
|
assert.equal(recovered.status, 'completed'); assert.equal(recovered.workerUsageRecovered, true);
|
|
const original = workerUsage(first.store, first.store.core.worker.get(worker.id)); assert.equal(original.unknownRequests, 1); assert.equal(original.requests, 1);
|
|
first.store.close(); const second = new Store(f.data); f.onCleanup(() => second.close());
|
|
assert.deepEqual(workerUsage(second, second.core.worker.get(worker.id)), original);
|
|
});
|
|
|
|
test('an unexecuted second workspace plan resumes its original identity without recreating the first checkout', async t => {
|
|
const f = fixture(t), { manager } = prepareWorkerFixture(f), group = await manager.plan(f.session, workerDefinition());
|
|
let count = 0; manager.workspaces.faultInjector = phase => { if (phase === 'prepared' && ++count === 2) throw new Error('Fixture interrupted after saving the second unexecuted plan'); };
|
|
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.status, 'unknown', group.summary); const first = f.store.core.worker.get(group.children[0].id), second = f.store.core.workspace.get(group.children[1].workspaceId);
|
|
assert.equal(second.status, 'prepared'); assert.equal(second.creationAttempted, false);
|
|
manager.workspaces.faultInjector = null; await evidence(manager, f.session, group);
|
|
await manager.acknowledge(f.session, group.id, group.fingerprint, group.evidence.sha256);
|
|
assert.equal(group.status, 'partially_prepared');
|
|
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.status, 'ready', group.summary); assert.equal(f.store.core.worker.get(first.id).runId, first.runId);
|
|
assert.equal(f.store.core.workspace.get(second.id).id, second.id); assert.equal(f.store.core.workspace.size, 2);
|
|
});
|
|
|
|
test('a verified interrupted parent proposal resumes sealing without duplicating or replacing its original change', async t => {
|
|
const f = fixture(t), { manager, group } = await completedWorkers(f);
|
|
await (await manager.collect(f.session, group.id, group.fingerprint)).promise;
|
|
manager.faultInjector = phase => { if (phase === 'merge_proposal') throw new Error('Fixture interruption after the first persisted parent proposal'); };
|
|
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.status, 'unknown'); const originalId = group.merge.changeIds[0]; assert.equal(group.merge.changeIds.length, 1);
|
|
manager.faultInjector = null; await evidence(manager, f.session, group);
|
|
await manager.acknowledge(f.session, group.id, group.fingerprint, group.evidence.sha256);
|
|
assert.equal(group.status, 'collected', group.summary);
|
|
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.status, 'merge_prepared', group.summary); assert.equal(group.merge.changeIds[0], originalId);
|
|
assert.equal(f.session.changes.filter(change => change.sourceBinding?.collaborationId === group.id).length, 2);
|
|
});
|
|
|
|
test('recovery attaches a sealed parent batch saved before its group reference without creating another batch', async t => {
|
|
const f = fixture(t), { manager, group } = await completedWorkers(f);
|
|
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise;
|
|
const setId = group.merge.changeSetId; delete group.merge.changeSetId; delete group.merge.fingerprint; group.merge.status = 'proposing'; group.status = 'merge_preparing'; f.store.save(f.session);
|
|
const restored = reopen(f, group.id); await evidence(restored.manager, restored.session, restored.group);
|
|
await restored.manager.acknowledge(restored.session, group.id, group.fingerprint, restored.group.evidence.sha256);
|
|
assert.equal(restored.group.status, 'merge_prepared', restored.group.summary); assert.equal(restored.group.merge.changeSetId, setId);
|
|
assert.equal([...restored.store.core.change_set.values()].filter(set => set.collaborationBinding?.id === group.id).length, 1); assert.equal(restored.requests.length, 0);
|
|
});
|
|
|
|
test('explicitly retaining an unexecuted parent merge cancels its batch and frees the next parent goal', async t => {
|
|
const f = fixture(t), { manager, group } = await completedWorkers(f), initial = ownerBytes(f);
|
|
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise;
|
|
const set = f.store.core.change_set.get(group.merge.changeSetId); group.status = 'unknown'; group.requiresVerification = true; f.store.save(f.session);
|
|
await evidence(manager, f.session, group); await manager.acknowledge(f.session, group.id, group.fingerprint, group.evidence.sha256, true);
|
|
assert.equal(group.status, 'resolved_external'); assert.equal(set.status, 'cancelled'); assert.equal(set.attemptedAt, undefined); assert.deepEqual(ownerBytes(f), initial);
|
|
assert.equal(f.session.changes.some(change => ['pending', 'conflict'].includes(change.status)), false);
|
|
const next = await manager.plan(f.session, { ...workerDefinition(), goal: 'The next explicitly planned goal' }); assert.notEqual(next.taskId, group.taskId);
|
|
});
|
|
|
|
test('unknown collaboration blocks ordinary proposals but permits only original batch recovery and rollback', async t => {
|
|
const f = fixture(t), { runtime, manager, group } = await completedWorkers(f), initial = ownerBytes(f);
|
|
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise;
|
|
const set = f.store.core.change_set.get(group.merge.changeSetId);
|
|
await (await runtime.changeSetManager().start(f.session, set.id, set.fingerprint)).promise;
|
|
set.status = 'unknown'; set.requiresVerification = true; group.status = 'unknown'; group.requiresVerification = true; f.store.save(f.session);
|
|
assert.throws(() => proposeFile(f.store, f.session, { path: 'src/new.mjs', content: 'blocked', reason: 'fixture' }), /待核对/);
|
|
await runtime.changeSetManager().reconcile(f.session, set.id); assert.equal(set.status, 'completed');
|
|
await runtime.changeSetManager().rollback(f.session, set.id, set.fingerprint); assert.equal(set.status, 'rolled_back');
|
|
assert.deepEqual(ownerBytes(f), initial); assert.equal(fs.existsSync(path.join(f.projectRoot, 'src/a.mjs')), false);
|
|
await evidence(manager, f.session, group);
|
|
await assert.rejects(manager.acknowledge(f.session, group.id, group.fingerprint, group.evidence.sha256), /批次|保留现场/);
|
|
await evidence(manager, f.session, group); await manager.acknowledge(f.session, group.id, group.fingerprint, group.evidence.sha256, true);
|
|
assert.equal(group.status, 'resolved_external');
|
|
});
|
|
|
|
test('after a restart the batch application lazily reinstates the worker result guard before any parent write', async t => {
|
|
const f = fixture(t), { manager, group } = await completedWorkers(f);
|
|
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise;
|
|
const setId = group.merge.changeSetId; f.store.close(); const store = new Store(f.data), runtime = new Runtime(store, new Provider()); f.onCleanup(() => store.close()); f.onCleanup(() => runtime.stopAll());
|
|
assert.equal(runtime.workers, undefined);
|
|
fs.writeFileSync(path.join(store.project(group.children[0].projectId).root, 'src/a.mjs'), 'changed after original preview\n');
|
|
const session = store.session(group.sessionId), set = store.core.change_set.get(setId); await assert.rejects(runtime.changeSetManager().start(session, set.id, set.fingerprint), /原成果|提案|不一致|变化/);
|
|
assert.ok(runtime.workers); assert.equal(set.attemptedAt, undefined); assert.equal(fs.existsSync(path.join(f.projectRoot, 'src/a.mjs')), false);
|
|
});
|
|
|
|
test('unknown original tool inputs must match before acknowledgement, and reviewed checkpoint IDs preserve the unknown fact', async t => {
|
|
const { f, group } = await ready(t), child = group.children[0], worker = f.store.core.worker.get(child.id), session = f.store.session(child.sessionId);
|
|
const run = f.store.beginRun(session, '', true, f.store.config); worker.currentRunId = run.id; worker.runIds.push(run.id); worker.status = 'running';
|
|
const originalCall = call(randomUUID(), 'inspect_project', {}); session.providerMessages.push({ role: 'assistant', tool_calls: [originalCall] });
|
|
const event = { id: randomUUID(), toolCallId: originalCall.id }; session.events.push(event); f.store.registerInvocation(session, originalCall, run, event); f.store.invocationState(session, event, 'running');
|
|
const restored = reopen(f, group.id), restoredChild = restored.store.session(child.sessionId), tool = [...restored.store.core.tool.values()].find(value => value.callId === originalCall.id);
|
|
assert.equal(restored.group.status, 'unknown'); await evidence(restored.manager, restored.session, restored.group);
|
|
const savedCall = restoredChild.providerMessages.flatMap(message => message.tool_calls ?? []).find(value => value.id === originalCall.id), originalArgs = savedCall.function.arguments;
|
|
savedCall.function.arguments = '{"changed":true}'; restored.store.save(restoredChild);
|
|
await assert.rejects(restored.manager.acknowledge(restored.session, group.id, group.fingerprint, restored.group.evidence.sha256), /输入哈希/);
|
|
savedCall.function.arguments = originalArgs; restored.store.save(restoredChild); await evidence(restored.manager, restored.session, restored.group);
|
|
await restored.manager.acknowledge(restored.session, group.id, group.fingerprint, restored.group.evidence.sha256);
|
|
assert.equal(tool.state, 'resolved_external'); assert.equal(tool.originalState, 'unknown'); assert.equal(tool.requiresVerification, false);
|
|
assert.equal(restored.store.core.run.get(run.id).checkpoint.requiresVerification.includes(tool.id), false);
|
|
assert.equal(restored.store.core.run.get(run.id).checkpoint.reviewedUnknownToolIds.includes(tool.id), true);
|
|
assert.equal(restored.store.savedToolResult(child.sessionId, originalCall.id).status, 'unknown'); assert.equal(restored.requests.length, 0);
|
|
});
|
|
|
|
test('a parent stop observes and waits for a manually approved child check and its real descendant process', async t => {
|
|
const f = fixture(t), { manager, runtime } = prepareWorkerFixture(f); longCheckProject(f); const initial = ownerBytes(f), group = await manager.plan(f.session, workerDefinition());
|
|
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise;
|
|
const child = group.children[0], session = f.store.session(child.sessionId), root = f.store.project(child.projectId).root;
|
|
const plan = await runtime.checkRunner().plan(session, ['test']), started = await runtime.checkRunner().start(session, plan.id);
|
|
await until(() => fs.existsSync(path.join(root, '.awb-heartbeat')) || started.job.finishedAt, 15000);
|
|
assert.equal(fs.existsSync(path.join(root, '.awb-heartbeat')), true, JSON.stringify(started.job));
|
|
const descendant = Number(fs.readFileSync(path.join(root, '.awb-child-pid'), 'utf8')); assert.equal(alive(descendant), true);
|
|
assert.equal(manager.isActive(f.session.id), true); assert.equal(runtime.projectHasActivity(f.project.id), true);
|
|
await assert.rejects(manager.plan(f.session, workerDefinition()), /执行|等待/);
|
|
await runtime.stop(f.session.id); await started.promise;
|
|
assert.equal(started.job.status, 'cancelled', started.job.summary); assert.equal(started.job.requiresVerification ?? false, false);
|
|
assert.equal(started.job.results[0].process.rootExited, true); assert.equal(started.job.results[0].process.treeTermination, 'confirmed');
|
|
assert.equal(alive(descendant), false); const bytes = fs.readFileSync(path.join(root, '.awb-heartbeat'));
|
|
await new Promise(resolve => setTimeout(resolve, 150)); assert.deepEqual(fs.readFileSync(path.join(root, '.awb-heartbeat')), bytes); assert.deepEqual(ownerBytes(f), initial); assert.equal(runtime.hasActivity(), false);
|
|
});
|
|
|
|
test('a real host crash preserves unknown checks, refuses a still living owned PID and never replays a model or tool', async t => {
|
|
const f = fixture(t), { manager } = prepareWorkerFixture(f); longCheckProject(f); const group = await manager.plan(f.session, workerDefinition());
|
|
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise;
|
|
for (const child of group.children) await manager.runtime.checkRunner().plan(f.store.session(child.sessionId), ['test']);
|
|
f.store.close();
|
|
const host = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/workers-crash.mjs'), f.data, group.id], { cwd: workspace, env: { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot, TEMP: process.env.TEMP }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
|
|
const observed = new Map(); let output = '';
|
|
host.on('message', value => { if (value.kind === 'owned-check' && value.process) observed.set(value.jobId, value.process); });
|
|
host.stderr.on('data', bytes => { output += bytes.toString('utf8'); });
|
|
const ended = new Promise(resolve => host.once('close', resolve));
|
|
const stopOwnedChecks = async () => {
|
|
if (host.exitCode === null && host.signalCode === null) host.kill('SIGKILL');
|
|
await ended;
|
|
const deadline = Date.now() + 20000;
|
|
while (true) {
|
|
const roots = [...observed.values()];
|
|
for (const info of roots) {
|
|
assert.equal(info.hostPid, host.pid);
|
|
assert.ok(path.resolve(info.cwd).startsWith(path.resolve(f.root) + path.sep));
|
|
}
|
|
const processes = (await Promise.all(roots.map(info => readOwnedProcessTree(info)))).flatMap(tree => tree.processes);
|
|
if (!processes.length) return;
|
|
assert.ok(Date.now() < deadline, 'Owned descendants did not exit after QA cleanup');
|
|
for (const info of processes) { try { execFileSync('taskkill.exe', ['/PID', String(info.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' }); } catch {} }
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
}
|
|
};
|
|
// The fixture disposer runs before removal, including when an assertion fails.
|
|
f.onCleanup(stopOwnedChecks);
|
|
await until(() => observed.size || host.exitCode !== null, 15000); assert.ok(observed.size, output);
|
|
const childRoots = group.children.map(child => manager.store.project(child.projectId).root);
|
|
await until(() => childRoots.every(root => fs.existsSync(path.join(root, '.awb-heartbeat')) && fs.existsSync(path.join(root, '.awb-child-pid'))), 15000);
|
|
const heartbeatPids = childRoots.map(root => Number(fs.readFileSync(path.join(root, '.awb-child-pid'), 'utf8')));
|
|
await until(() => heartbeatPids.every(pid => [...observed.values()].some(info => info.descendants?.some(value => value.pid === pid))) || host.exitCode !== null, 20000);
|
|
assert.equal(observed.size, 2, output);
|
|
for (const pid of heartbeatPids) assert.ok([...observed.values()].some(info => info.descendants?.some(value => value.pid === pid)), 'Both actual heartbeat descendants must be owned before the deliberate host crash');
|
|
host.kill('SIGKILL'); await ended;
|
|
const store = new Store(f.data), runtime = new Runtime(store, new Provider()), recovered = runtime.workerManager(), parent = store.session(group.sessionId), record = store.core.collaboration.get(group.id);
|
|
f.onCleanup(() => store.close()); f.onCleanup(() => runtime.stopAll());
|
|
const jobs = [...store.core.check_job.values()].filter(job => group.children.some(child => child.taskId === job.taskId));
|
|
const pids = jobs.flatMap(job => job.currentCheck?.process ? [job.currentCheck.process, ...(job.currentCheck.process.descendants ?? []).map(value => ({ ...value, hostPid: host.pid, cwd: job.currentCheck.process.cwd }))] : []);
|
|
assert.equal(record.status, 'unknown'); assert.ok(jobs.some(job => job.status === 'unknown')); assert.ok(pids.some(info => alive(info.pid)));
|
|
await (await recovered.reconcile(parent, record.id, record.fingerprint)).promise; assert.equal(record.evidence, undefined); assert.match(record.summary, /PID|存活/);
|
|
for (const job of jobs) if (job.currentCheck?.process) observed.set(job.id, job.currentCheck.process);
|
|
await stopOwnedChecks(); assert.ok(heartbeatPids.every(pid => !alive(pid)));
|
|
const stoppedHeartbeat = childRoots.map(root => fs.readFileSync(path.join(root, '.awb-heartbeat')));
|
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
childRoots.forEach((root, index) => assert.deepEqual(fs.readFileSync(path.join(root, '.awb-heartbeat')), stoppedHeartbeat[index], 'Stopped checks cannot continue writing after the process inventory is empty'));
|
|
await evidence(recovered, parent, record); await recovered.acknowledge(parent, record.id, record.fingerprint, record.evidence.sha256);
|
|
assert.equal(record.requiresVerification, false); assert.equal(jobs.some(job => job.status === 'passed'), false); assert.equal([...store.core.tool.values()].some(tool => tool.originalState === 'unknown'), true);
|
|
assert.equal(runtime.hasActivity(), false);
|
|
childRoots.forEach((root, index) => assert.deepEqual(fs.readFileSync(path.join(root, '.awb-heartbeat')), stoppedHeartbeat[index], 'Acknowledgement must not restart either actual heartbeat'));
|
|
});
|
|
|
|
test('a persisted child initialization resumes the original control run after a failure before its worker record', async t => {
|
|
const f = fixture(t), { manager } = prepareWorkerFixture(f), group = await manager.plan(f.session, workerDefinition()), inventory = manager.inventory;
|
|
let failed = false; manager.inventory = async function(project) { if (!failed) { failed = true; throw new Error('Owned failure after child task persistence'); } return inventory.call(this, project); };
|
|
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'unknown', group.summary);
|
|
const child = group.children[0], session = f.store.session(child.sessionId), originalRunId = session.runId; assert.equal(session.taskId, child.taskId); assert.equal(f.store.core.worker.has(child.id), false);
|
|
manager.inventory = inventory; await evidence(manager, f.session, group); await manager.acknowledge(f.session, group.id, group.fingerprint, group.evidence.sha256);
|
|
assert.equal(group.status, 'partially_prepared'); await (await manager.prepare(f.session, group.id, group.fingerprint)).promise;
|
|
assert.equal(group.status, 'ready', group.summary); assert.equal(f.store.core.worker.get(child.id).runId, originalRunId);
|
|
assert.equal([...f.store.core.task.values()].filter(task => task.workerId === child.id).length, 1);
|
|
});
|
|
|
|
test('failed request intent persistence never dispatches a worker request or changes source', async t => {
|
|
const { f, manager, group, requests } = await ready(t, { respond: async () => ({ message: { role: 'assistant', content: 'unexpected' }, usage: { total_tokens: 5 } }) });
|
|
const save = f.store.storage.save, rejected = new Set();
|
|
f.store.storage.save = function(session, records, type, detail) { if (type === 'worker.request_intent' && !rejected.has(session.id)) { rejected.add(session.id); throw new Error('Owned SQLite request intent failure'); } return save.call(this, session, records, type, detail); };
|
|
try { await (await manager.start(f.session, group.id, group.fingerprint)).promise; } finally { f.store.storage.save = save; }
|
|
assert.equal(rejected.size, 2); assert.equal(requests.length, 0);
|
|
for (const child of group.children) { assert.equal(workerUsage(f.store, f.store.core.worker.get(child.id)).requests, 0); assert.equal(f.store.session(child.sessionId).changes.length, 0); }
|
|
});
|
|
|
|
test('failed usage persistence rolls back the uncommitted usage and conservatively reserves one unknown request without tools', async t => {
|
|
const { f, manager, group, requests } = await ready(t, { respond: async () => ({ message: { role: 'assistant', tool_calls: [call(randomUUID(), 'propose_file', { path: 'src/a.mjs', content: 'must not execute', reason: 'fixture' })] }, usage: { total_tokens: 73 } }) });
|
|
const save = f.store.storage.save, rejected = new Set();
|
|
f.store.storage.save = function(session, records, type, detail) {
|
|
if (type === 'session.updated' && records.some(record => record.kind === 'context_plan' && record.value.status === 'completed' && record.value.workerUsageAccounted) && !rejected.has(session.id)) { rejected.add(session.id); throw new Error('Owned SQLite usage persistence failure'); }
|
|
return save.call(this, session, records, type, detail);
|
|
};
|
|
try { await (await manager.start(f.session, group.id, group.fingerprint)).promise; } finally { f.store.storage.save = save; }
|
|
assert.equal(rejected.size, 2); assert.equal(requests.length, 2);
|
|
for (const child of group.children) { const usage = workerUsage(f.store, f.store.core.worker.get(child.id)); assert.equal(usage.requests, 1); assert.equal(usage.unknownRequests, 1); assert.equal(usage.totalTokens, 0); assert.equal(usage.toolCalls, 0); assert.equal(f.store.session(child.sessionId).changes.length, 0); }
|
|
});
|
|
|
|
test('a child file change after observation prevents acknowledgement even when the tool inputs did not change', async t => {
|
|
const { f, manager, group } = await ready(t); group.status = 'unknown'; group.requiresVerification = true; f.store.save(f.session); await evidence(manager, f.session, group);
|
|
fs.appendFileSync(path.join(f.store.project(group.children[0].projectId).root, 'README.md'), 'External owner edit after observation\n');
|
|
await assert.rejects(manager.acknowledge(f.session, group.id, group.fingerprint, group.evidence.sha256), /现场.*变化/);
|
|
assert.equal(group.status, 'unknown'); assert.equal(group.requiresVerification, true);
|
|
});
|