feat: seal isolated worker recovery, parent integration and review console

This commit is contained in:
Agent Workbench Dev
2026-10-31 02:02:06 +08:00
parent 3abc0e8595
commit 2ada7b899f
34 changed files with 2877 additions and 64 deletions
+54 -12
View File
@@ -3,13 +3,14 @@ import path from 'node:path';
import { spawn, execFile } from 'node:child_process';
import { createHash, randomUUID } from 'node:crypto';
import { StringDecoder } from 'node:string_decoder';
import { readOwnedProcessTree } from './owned-process-tree.mjs';
import { inspectProject, createCheckList } from './project-inspector.mjs';
import { safePath } from '../server/files.mjs';
import { assertChangeSetsResolved } from '../shared/contracts/change-sets.mjs';
import { assertWorkspacesResolved } from '../shared/contracts/workspaces.mjs';
import { verifyWorkspaceProject } from './workspaces/git-workspaces.mjs';
import { assertMcpResolved } from './tools/mcp-contract.mjs';
import { assertWorkersResolved } from '../shared/contracts/workers.mjs';
import { assertWorkersResolved, workerForSession } from '../shared/contracts/workers.mjs';
import { bindPythonChecks, preparePythonInvocation, pythonEnvironment, readPythonEvidence, pythonResultBinding } from './python-runtime.mjs';
import { projectEnvironment } from './process-environment.mjs';
@@ -65,6 +66,7 @@ export class CheckRunner {
async plan(session, names) {
assertWorkersResolved(this.store, session.projectId);
if (workerForSession(this.store, session)?.status === 'completed') throw conflict('子成果已经封存,请读取原检查记录;重新执行需要新的受限任务。');
assertChangeSetsResolved(this.store, session.projectId);
assertWorkspacesResolved(this.store, session.projectId, { includePrepared: true });
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
@@ -105,6 +107,7 @@ export class CheckRunner {
async start(session, planId, { signal, source = 'user' } = {}) {
assertWorkersResolved(this.store, session.projectId);
if (workerForSession(this.store, session)?.status === 'completed') throw conflict('子成果已经封存,不重新运行原检查。');
assertMcpResolved(this.store, session.projectId);
assertChangeSetsResolved(this.store, session.projectId, { includePrepared: true });
assertWorkspacesResolved(this.store, session.projectId, { includePrepared: true });
@@ -130,6 +133,7 @@ export class CheckRunner {
async execute(session, plan, job, run, signal) {
const task = this.store.core.task.get(job.taskId);
let unrecordedProcess = null;
try {
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
const before = await inspectProject(this.store.project(session.projectId));
@@ -144,7 +148,9 @@ export class CheckRunner {
signal.throwIfAborted();
job.currentCheck = { id: check.id, name: check.name, command: check.command, startedAt: at() };
this.store.save(session, { type: 'check.command_started', detail: { jobId: job.id, checkId: check.id } });
const result = await this.executeCheck(check, this.store.project(session.projectId), signal);
const onSpawn = task.workerId ? processInfo => { unrecordedProcess = processInfo; job.currentCheck.process = processInfo; this.store.save(session, { type: 'worker.check_process_owned', detail: { jobId: job.id, pid: processInfo.pid } }); } : null;
const result = await this.executeCheck(check, this.store.project(session.projectId), signal, onSpawn);
unrecordedProcess = result.process ?? unrecordedProcess;
const text = this.redact(result.stdout + (result.stderr ? '\n--- stderr ---\n' + result.stderr : ''));
const artifact = this.store.artifactStore.write(session, run, text, { kind: 'check-log', mediaType: 'text/plain' });
this.store.core.artifact.set(artifact.id, artifact);
@@ -159,6 +165,7 @@ export class CheckRunner {
}
job.results.push({ id: randomUUID(), name: check.name, kind: check.kind, command: check.command, ...record, logArtifactId: artifact.id, logExcerpt: text.slice(-8000) });
this.store.save(session, { type: 'check.result', detail: { jobId: job.id, checkId: check.id, status: record.status, artifactId: artifact.id } });
unrecordedProcess = null;
if (record.status === 'stale') { job.stale = true; break; }
if (['cancelled', 'unknown'].includes(record.status)) break;
const after = await inspectProject(this.store.project(session.projectId));
@@ -166,8 +173,8 @@ export class CheckRunner {
}
const final = await inspectProject(this.store.project(session.projectId));
job.stale ||= final.sourceFingerprint !== job.sourceFingerprint;
if (signal.aborted || job.results.some(result => result.status === 'cancelled')) job.status = 'cancelled';
else if (job.results.some(result => result.status === 'unknown')) { job.status = 'unknown'; job.requiresVerification = true; }
if (job.results.some(result => result.status === 'unknown')) { job.status = 'unknown'; job.requiresVerification = true; }
else if (signal.aborted || job.results.some(result => result.status === 'cancelled')) job.status = 'cancelled';
else if (job.stale) job.status = 'stale';
else if (job.results.some(result => result.status === 'failed')) job.status = 'failed';
else if (!final.scope.complete || job.results.length !== plan.checks.length || job.results.some(result => result.status !== 'passed') || !job.results.some(result => result.kind === 'test' && result.tests?.passed > 0)) job.status = 'unverified';
@@ -177,6 +184,10 @@ export class CheckRunner {
job.status = signal.aborted ? 'cancelled' : error.status === 409 ? 'stale' : 'failed';
job.stale = error.status === 409; job.summary = this.redact(error.message ?? String(error));
} finally {
if (unrecordedProcess) {
job.unreportedProcesses ??= []; job.unreportedProcesses.push(unrecordedProcess);
job.status = 'unknown'; job.requiresVerification = true; job.summary = '所属检查进程没有完整持久结果,保留进程归属供原输入核对。';
}
job.currentCheck = null;
job.finishedAt = at();
task.verification = { status: job.status, checkJobId: job.id, fingerprint: job.fingerprint, sourceFingerprint: job.sourceFingerprint, checkedAt: job.finishedAt };
@@ -186,7 +197,7 @@ export class CheckRunner {
return job;
}
async executeCheck(check, project, signal) {
async executeCheck(check, project, signal, onSpawn = null) {
const started = Date.now();
let invocation;
const isPython = check.engine === 'python';
@@ -196,23 +207,47 @@ export class CheckRunner {
const env = isPython ? pythonEnvironment() : projectEnvironment({ NO_COLOR: '1', CI: '1' });
const child = spawn(invocation.executable, invocation.args, { cwd: safePath(project), env, windowsHide: true, detached: process.platform !== 'win32', stdio: [isPython ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
const streams = { stdout: { head: '', tail: '', characters: 0, decoder: new StringDecoder('utf8') }, stderr: { head: '', tail: '', characters: 0, decoder: new StringDecoder('utf8') } };
let settled = false, timedOut = false, terminating = false, killTimer;
let settled = false, timedOut = false, terminating = false, killTimer, ownershipError = null, treeKill = null, closedResult = null, rootClosed = false, treeTimer = null, treePending = null;
const processInfo = onSpawn ? { pid: child.pid ?? null, hostPid: process.pid, executable: invocation.executable, argsSha256: createHash('sha256').update(JSON.stringify(invocation.args)).digest('hex'), cwd: project.root, startedAt: at() } : null;
const append = (name, text) => { const value = streams[name]; value.characters += text.length; value.head = (value.head + text).slice(0, 240000); value.tail = (value.tail + text).slice(-240000); };
const captured = name => { const value = streams[name]; return value.characters <= 240000 ? value.head : value.characters <= 480000 ? value.head + value.tail.slice(value.head.length - value.characters) : value.head + '\n[中间日志超出保存上限,保留开头和结尾]\n' + value.tail; };
const finish = (code, error, unknown = false) => {
if (settled) return; settled = true; clearTimeout(timeout); clearTimeout(killTimer); signal.removeEventListener('abort', terminate);
const stdout = captured('stdout'), stderr = captured('stderr') + (error ? '\n' + error.message : '');
const captureTree = async (publish = true) => {
const snapshot = await readOwnedProcessTree(processInfo);
if (!snapshot.supported) return snapshot;
if (snapshot.root) processInfo.rootCreatedAt ??= snapshot.root.createdAt;
const known = new Map((processInfo.descendants ?? []).map(value => [value.pid + ':' + value.createdAt, value]));
for (const value of snapshot.processes) if (value.pid !== processInfo.pid) known.set(value.pid + ':' + value.createdAt, value);
processInfo.descendants = [...known.values()].sort((a, b) => a.pid - b.pid);
processInfo.treeObservedAt = at(); processInfo.treeRemainingPids = snapshot.processes.map(value => value.pid);
if (publish && !settled) onSpawn(processInfo);
return snapshot;
};
const monitorTree = () => {
if (settled || treePending) return;
treePending = captureTree().catch(error => { ownershipError = error; processInfo.treeObservationError = error.message; terminate(); }).finally(() => { treePending = null; });
};
const finish = async (code, error, unknown = false) => {
if (onSpawn && terminating && treeKill && !treeKill.done && !unknown) { closedResult = { code, error }; return; }
if (onSpawn && treeKill?.error) unknown = true;
if (settled) return; settled = true; clearTimeout(timeout); clearTimeout(killTimer); clearInterval(treeTimer); signal.removeEventListener('abort', terminate);
if (onSpawn && process.platform === 'win32' && processInfo.pid) {
await treePending;
try { const snapshot = await captureTree(false); if (snapshot.processes.length) unknown = true; }
catch (failure) { ownershipError = failure; processInfo.treeObservationError = failure.message; unknown = true; }
}
const stdout = captured('stdout'), stderr = captured('stderr') + (error ? '\n' + error.message : '') + (ownershipError ? '\n' + ownershipError.message : '');
let collectedEvidence = null, collectedResult = null, evidenceError = null, cleanupError = null;
if (isPython && !unknown && code !== null && !error) {
try { const saved = readPythonEvidence(invocation, code); collectedEvidence = saved.text; collectedResult = saved.result; } catch (failure) { evidenceError = failure.message; }
}
const tests = check.kind === 'test' ? isPython ? collectedResult?.tests ?? null : parseTestEvidence(streams.stdout.tail, check.framework) : null;
const status = unknown ? 'unknown' : signal.aborted ? 'cancelled'
: timedOut || error || code !== 0 || evidenceError || tests?.failed > 0 || collectedResult?.collectionErrors > 0 ? 'failed'
: timedOut || error || ownershipError || code !== 0 || evidenceError || tests?.failed > 0 || collectedResult?.collectionErrors > 0 ? 'failed'
: (check.kind === 'test' && (!tests || tests.total === 0 || tests.passed !== tests.total || tests.cancelled > 0 || collectedResult?.collectionSkipped > 0)
|| isPython && collectedResult?.compilation?.total === 0) ? 'unverified' : 'passed';
if (isPython && !unknown) { try { invocation.dispose(); } catch (failure) { cleanupError = failure.message; } }
resolve({ status, exitCode: code, durationMs: Date.now() - started, tests, stdout, stderr, timedOut, truncated: streams.stdout.characters > 480000 || streams.stderr.characters > 480000,
...(onSpawn ? { process: { ...processInfo, rootExited: rootClosed, treeTermination: terminating ? unknown ? 'unknown' : 'confirmed' : 'not_requested' } } : {}),
...(isPython ? { engine: 'python', python: pythonResultBinding(invocation), collectedEvidence, checkInput: invocation.input.toString('utf8'),
compilation: collectedResult?.compilation ?? null, collectionErrors: collectedResult?.collectionErrors ?? null,
collectionSkipped: collectedResult?.collectionSkipped ?? null, evidenceValidated: !!collectedResult, evidenceError, cleanupError,
@@ -223,13 +258,20 @@ export class CheckRunner {
};
const terminate = () => {
if (settled || terminating) return; terminating = true;
if (child.pid) { if (process.platform === 'win32') execFile('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true }, () => {}); else { try { process.kill(-child.pid, 'SIGTERM'); } catch { child.kill('SIGTERM'); } } }
if (child.pid) { if (process.platform === 'win32') {
if (onSpawn) treeKill = { done: false, error: null };
execFile('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true }, error => {
if (!treeKill) return; treeKill.done = true; treeKill.error = error;
if (closedResult) finish(closedResult.code, closedResult.error);
});
} else { try { process.kill(-child.pid, 'SIGTERM'); } catch { child.kill('SIGTERM'); } } }
killTimer = setTimeout(() => finish(null, null, true), 5000);
};
const timeout = setTimeout(() => { timedOut = true; terminate(); }, this.timeoutMs);
signal.addEventListener('abort', terminate, { once: true });
for (const name of ['stdout', 'stderr']) { child[name].on('data', bytes => append(name, streams[name].decoder.write(bytes))); child[name].on('end', () => append(name, streams[name].decoder.end())); }
child.once('error', error => finish(null, error)); child.once('close', code => finish(code));
child.once('error', error => finish(null, error)); child.once('close', code => { rootClosed = true; finish(code); });
if (onSpawn) child.once('spawn', () => { processInfo.pid = child.pid; try { onSpawn(processInfo); if (process.platform === 'win32') { monitorTree(); treeTimer = setInterval(monitorTree, 1200); } } catch (error) { ownershipError = error; terminate(); } });
if (isPython) { child.stdin.on('error', () => {}); child.stdin.end(invocation.input); }
if (signal.aborted) terminate();
});