fix: await both owned crash trees and dispose fixtures before removal
This commit is contained in:
Vendored
+1
-1
@@ -11,7 +11,7 @@ provider.complete = async (config, messages, project) => {
|
||||
const save = store.save;
|
||||
store.save = function(session, event) { const result = save.call(this, session, event); if (event?.type === 'worker.check_process_owned') {
|
||||
const info = [...store.core.check_job.values()].find(job => job.id === event.detail.jobId)?.currentCheck?.process;
|
||||
process.send?.({ kind: 'owned-check', descendants: info?.descendants?.length ?? 0 });
|
||||
process.send?.({ kind: 'owned-check', jobId: event.detail.jobId, process: info, descendants: info?.descendants?.length ?? 0 });
|
||||
} return result; };
|
||||
const group = store.core.collaboration.get(groupId), parent = store.session(group.sessionId);
|
||||
const started = runtime.workerManager().start(parent, group.id, group.fingerprint);
|
||||
|
||||
+3
-1
@@ -26,7 +26,9 @@ export function fixture(t) {
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
const resolved = path.resolve(root);
|
||||
assert.ok(resolved.startsWith(path.resolve(fixtureRoot) + path.sep) && path.basename(resolved).startsWith('test-'));
|
||||
fs.rmSync(resolved, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
||||
// Yield while Windows releases handles from owned child processes. A
|
||||
// persistent lock still fails teardown; resource disposal must run first.
|
||||
await fs.promises.rm(resolved, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
|
||||
});
|
||||
return { root, projectRoot, data, store, project, session, onCleanup: dispose => disposers.push(dispose) };
|
||||
}
|
||||
|
||||
@@ -216,37 +216,52 @@ test('a real host crash preserves unknown checks, refuses a still living owned P
|
||||
for (const child of group.children) await manager.runtime.checkRunner().plan(f.store.session(child.sessionId), ['test']);
|
||||
f.store.close();
|
||||
const host = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/workers-crash.mjs'), f.data, group.id], { cwd: workspace, env: { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot, TEMP: process.env.TEMP }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
|
||||
let processStarted = false, treeObserved = false, output = ''; host.on('message', value => { if (value.kind === 'owned-check') { processStarted = true; if (value.descendants >= 3) treeObserved = true; } }); host.stderr.on('data', bytes => { output += bytes.toString('utf8'); });
|
||||
const observed = new Map(); let output = '';
|
||||
host.on('message', value => { if (value.kind === 'owned-check' && value.process) observed.set(value.jobId, value.process); });
|
||||
host.stderr.on('data', bytes => { output += bytes.toString('utf8'); });
|
||||
const ended = new Promise(resolve => host.once('close', resolve));
|
||||
t.after(async () => { if (host.exitCode === null && host.signalCode === null) { host.kill('SIGKILL'); await ended; } });
|
||||
await until(() => processStarted || host.exitCode !== null, 15000); assert.equal(processStarted, true, output);
|
||||
await until(() => group.children.some(child => fs.existsSync(path.join(manager.store.project(child.projectId).root, '.awb-heartbeat'))), 15000);
|
||||
await until(() => treeObserved || host.exitCode !== null, 20000); assert.equal(treeObserved, true, output);
|
||||
const stopOwnedChecks = async () => {
|
||||
if (host.exitCode === null && host.signalCode === null) host.kill('SIGKILL');
|
||||
await ended;
|
||||
const deadline = Date.now() + 20000;
|
||||
while (true) {
|
||||
const roots = [...observed.values()];
|
||||
for (const info of roots) {
|
||||
assert.equal(info.hostPid, host.pid);
|
||||
assert.ok(path.resolve(info.cwd).startsWith(path.resolve(f.root) + path.sep));
|
||||
}
|
||||
const processes = (await Promise.all(roots.map(info => readOwnedProcessTree(info)))).flatMap(tree => tree.processes);
|
||||
if (!processes.length) return;
|
||||
assert.ok(Date.now() < deadline, 'Owned descendants did not exit after QA cleanup');
|
||||
for (const info of processes) { try { execFileSync('taskkill.exe', ['/PID', String(info.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' }); } catch {} }
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
};
|
||||
// The fixture disposer runs before removal, including when an assertion fails.
|
||||
f.onCleanup(stopOwnedChecks);
|
||||
await until(() => observed.size || host.exitCode !== null, 15000); assert.ok(observed.size, output);
|
||||
const childRoots = group.children.map(child => manager.store.project(child.projectId).root);
|
||||
await until(() => childRoots.every(root => fs.existsSync(path.join(root, '.awb-heartbeat')) && fs.existsSync(path.join(root, '.awb-child-pid'))), 15000);
|
||||
const heartbeatPids = childRoots.map(root => Number(fs.readFileSync(path.join(root, '.awb-child-pid'), 'utf8')));
|
||||
await until(() => heartbeatPids.every(pid => [...observed.values()].some(info => info.descendants?.some(value => value.pid === pid))) || host.exitCode !== null, 20000);
|
||||
assert.equal(observed.size, 2, output);
|
||||
for (const pid of heartbeatPids) assert.ok([...observed.values()].some(info => info.descendants?.some(value => value.pid === pid)), 'Both actual heartbeat descendants must be owned before the deliberate host crash');
|
||||
host.kill('SIGKILL'); await ended;
|
||||
const store = new Store(f.data), runtime = new Runtime(store, new Provider()), recovered = runtime.workerManager(), parent = store.session(group.sessionId), record = store.core.collaboration.get(group.id);
|
||||
f.onCleanup(() => store.close()); f.onCleanup(() => runtime.stopAll());
|
||||
const jobs = [...store.core.check_job.values()].filter(job => group.children.some(child => child.taskId === job.taskId));
|
||||
const pids = jobs.flatMap(job => job.currentCheck?.process ? [job.currentCheck.process, ...(job.currentCheck.process.descendants ?? []).map(value => ({ ...value, hostPid: host.pid, cwd: job.currentCheck.process.cwd }))] : []);
|
||||
t.after(() => { for (const info of pids) if (info.hostPid === host.pid && alive(info.pid)) { try { execFileSync('taskkill.exe', ['/PID', String(info.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' }); } catch {} } });
|
||||
assert.equal(record.status, 'unknown'); assert.ok(jobs.some(job => job.status === 'unknown')); assert.ok(pids.some(info => alive(info.pid)));
|
||||
await (await recovered.reconcile(parent, record.id, record.fingerprint)).promise; assert.equal(record.evidence, undefined); assert.match(record.summary, /PID|存活/);
|
||||
for (const info of pids) {
|
||||
assert.equal(info.hostPid, host.pid); assert.ok(path.resolve(info.cwd).startsWith(path.resolve(f.root) + path.sep));
|
||||
if (alive(info.pid)) { try { execFileSync('taskkill.exe', ['/PID', String(info.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' }); } catch { await until(() => !alive(info.pid), 3000); } }
|
||||
}
|
||||
await until(() => pids.every(info => !alive(info.pid)));
|
||||
const roots = jobs.flatMap(job => job.currentCheck?.process ? [job.currentCheck.process] : []);
|
||||
const cleanupDeadline = Date.now() + 20000;
|
||||
while (true) {
|
||||
const current = (await Promise.all(roots.map(info => readOwnedProcessTree(info)))).flatMap(tree => tree.processes);
|
||||
if (!current.length) break;
|
||||
assert.ok(Date.now() < cleanupDeadline, 'Owned late descendant did not exit after QA cleanup');
|
||||
for (const info of current) { try { execFileSync('taskkill.exe', ['/PID', String(info.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' }); } catch {} }
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
for (const job of jobs) if (job.currentCheck?.process) observed.set(job.id, job.currentCheck.process);
|
||||
await stopOwnedChecks(); assert.ok(heartbeatPids.every(pid => !alive(pid)));
|
||||
const stoppedHeartbeat = childRoots.map(root => fs.readFileSync(path.join(root, '.awb-heartbeat')));
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
childRoots.forEach((root, index) => assert.deepEqual(fs.readFileSync(path.join(root, '.awb-heartbeat')), stoppedHeartbeat[index], 'Stopped checks cannot continue writing after the process inventory is empty'));
|
||||
await evidence(recovered, parent, record); await recovered.acknowledge(parent, record.id, record.fingerprint, record.evidence.sha256);
|
||||
assert.equal(record.requiresVerification, false); assert.equal(jobs.some(job => job.status === 'passed'), false); assert.equal([...store.core.tool.values()].some(tool => tool.originalState === 'unknown'), true);
|
||||
assert.equal(runtime.hasActivity(), false);
|
||||
childRoots.forEach((root, index) => assert.deepEqual(fs.readFileSync(path.join(root, '.awb-heartbeat')), stoppedHeartbeat[index], 'Acknowledgement must not restart either actual heartbeat'));
|
||||
});
|
||||
|
||||
test('a persisted child initialization resumes the original control run after a failure before its worker record', async t => {
|
||||
|
||||
Reference in New Issue
Block a user