feat: seal isolated worker recovery, parent integration and review console
This commit is contained in:
+54
-12
@@ -3,13 +3,14 @@ import path from 'node:path';
|
||||
import { spawn, execFile } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import { readOwnedProcessTree } from './owned-process-tree.mjs';
|
||||
import { inspectProject, createCheckList } from './project-inspector.mjs';
|
||||
import { safePath } from '../server/files.mjs';
|
||||
import { assertChangeSetsResolved } from '../shared/contracts/change-sets.mjs';
|
||||
import { assertWorkspacesResolved } from '../shared/contracts/workspaces.mjs';
|
||||
import { verifyWorkspaceProject } from './workspaces/git-workspaces.mjs';
|
||||
import { assertMcpResolved } from './tools/mcp-contract.mjs';
|
||||
import { assertWorkersResolved } from '../shared/contracts/workers.mjs';
|
||||
import { assertWorkersResolved, workerForSession } from '../shared/contracts/workers.mjs';
|
||||
import { bindPythonChecks, preparePythonInvocation, pythonEnvironment, readPythonEvidence, pythonResultBinding } from './python-runtime.mjs';
|
||||
import { projectEnvironment } from './process-environment.mjs';
|
||||
|
||||
@@ -65,6 +66,7 @@ export class CheckRunner {
|
||||
|
||||
async plan(session, names) {
|
||||
assertWorkersResolved(this.store, session.projectId);
|
||||
if (workerForSession(this.store, session)?.status === 'completed') throw conflict('子成果已经封存,请读取原检查记录;重新执行需要新的受限任务。');
|
||||
assertChangeSetsResolved(this.store, session.projectId);
|
||||
assertWorkspacesResolved(this.store, session.projectId, { includePrepared: true });
|
||||
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
|
||||
@@ -105,6 +107,7 @@ export class CheckRunner {
|
||||
|
||||
async start(session, planId, { signal, source = 'user' } = {}) {
|
||||
assertWorkersResolved(this.store, session.projectId);
|
||||
if (workerForSession(this.store, session)?.status === 'completed') throw conflict('子成果已经封存,不重新运行原检查。');
|
||||
assertMcpResolved(this.store, session.projectId);
|
||||
assertChangeSetsResolved(this.store, session.projectId, { includePrepared: true });
|
||||
assertWorkspacesResolved(this.store, session.projectId, { includePrepared: true });
|
||||
@@ -130,6 +133,7 @@ export class CheckRunner {
|
||||
|
||||
async execute(session, plan, job, run, signal) {
|
||||
const task = this.store.core.task.get(job.taskId);
|
||||
let unrecordedProcess = null;
|
||||
try {
|
||||
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
|
||||
const before = await inspectProject(this.store.project(session.projectId));
|
||||
@@ -144,7 +148,9 @@ export class CheckRunner {
|
||||
signal.throwIfAborted();
|
||||
job.currentCheck = { id: check.id, name: check.name, command: check.command, startedAt: at() };
|
||||
this.store.save(session, { type: 'check.command_started', detail: { jobId: job.id, checkId: check.id } });
|
||||
const result = await this.executeCheck(check, this.store.project(session.projectId), signal);
|
||||
const onSpawn = task.workerId ? processInfo => { unrecordedProcess = processInfo; job.currentCheck.process = processInfo; this.store.save(session, { type: 'worker.check_process_owned', detail: { jobId: job.id, pid: processInfo.pid } }); } : null;
|
||||
const result = await this.executeCheck(check, this.store.project(session.projectId), signal, onSpawn);
|
||||
unrecordedProcess = result.process ?? unrecordedProcess;
|
||||
const text = this.redact(result.stdout + (result.stderr ? '\n--- stderr ---\n' + result.stderr : ''));
|
||||
const artifact = this.store.artifactStore.write(session, run, text, { kind: 'check-log', mediaType: 'text/plain' });
|
||||
this.store.core.artifact.set(artifact.id, artifact);
|
||||
@@ -159,6 +165,7 @@ export class CheckRunner {
|
||||
}
|
||||
job.results.push({ id: randomUUID(), name: check.name, kind: check.kind, command: check.command, ...record, logArtifactId: artifact.id, logExcerpt: text.slice(-8000) });
|
||||
this.store.save(session, { type: 'check.result', detail: { jobId: job.id, checkId: check.id, status: record.status, artifactId: artifact.id } });
|
||||
unrecordedProcess = null;
|
||||
if (record.status === 'stale') { job.stale = true; break; }
|
||||
if (['cancelled', 'unknown'].includes(record.status)) break;
|
||||
const after = await inspectProject(this.store.project(session.projectId));
|
||||
@@ -166,8 +173,8 @@ export class CheckRunner {
|
||||
}
|
||||
const final = await inspectProject(this.store.project(session.projectId));
|
||||
job.stale ||= final.sourceFingerprint !== job.sourceFingerprint;
|
||||
if (signal.aborted || job.results.some(result => result.status === 'cancelled')) job.status = 'cancelled';
|
||||
else if (job.results.some(result => result.status === 'unknown')) { job.status = 'unknown'; job.requiresVerification = true; }
|
||||
if (job.results.some(result => result.status === 'unknown')) { job.status = 'unknown'; job.requiresVerification = true; }
|
||||
else if (signal.aborted || job.results.some(result => result.status === 'cancelled')) job.status = 'cancelled';
|
||||
else if (job.stale) job.status = 'stale';
|
||||
else if (job.results.some(result => result.status === 'failed')) job.status = 'failed';
|
||||
else if (!final.scope.complete || job.results.length !== plan.checks.length || job.results.some(result => result.status !== 'passed') || !job.results.some(result => result.kind === 'test' && result.tests?.passed > 0)) job.status = 'unverified';
|
||||
@@ -177,6 +184,10 @@ export class CheckRunner {
|
||||
job.status = signal.aborted ? 'cancelled' : error.status === 409 ? 'stale' : 'failed';
|
||||
job.stale = error.status === 409; job.summary = this.redact(error.message ?? String(error));
|
||||
} finally {
|
||||
if (unrecordedProcess) {
|
||||
job.unreportedProcesses ??= []; job.unreportedProcesses.push(unrecordedProcess);
|
||||
job.status = 'unknown'; job.requiresVerification = true; job.summary = '所属检查进程没有完整持久结果,保留进程归属供原输入核对。';
|
||||
}
|
||||
job.currentCheck = null;
|
||||
job.finishedAt = at();
|
||||
task.verification = { status: job.status, checkJobId: job.id, fingerprint: job.fingerprint, sourceFingerprint: job.sourceFingerprint, checkedAt: job.finishedAt };
|
||||
@@ -186,7 +197,7 @@ export class CheckRunner {
|
||||
return job;
|
||||
}
|
||||
|
||||
async executeCheck(check, project, signal) {
|
||||
async executeCheck(check, project, signal, onSpawn = null) {
|
||||
const started = Date.now();
|
||||
let invocation;
|
||||
const isPython = check.engine === 'python';
|
||||
@@ -196,23 +207,47 @@ export class CheckRunner {
|
||||
const env = isPython ? pythonEnvironment() : projectEnvironment({ NO_COLOR: '1', CI: '1' });
|
||||
const child = spawn(invocation.executable, invocation.args, { cwd: safePath(project), env, windowsHide: true, detached: process.platform !== 'win32', stdio: [isPython ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
|
||||
const streams = { stdout: { head: '', tail: '', characters: 0, decoder: new StringDecoder('utf8') }, stderr: { head: '', tail: '', characters: 0, decoder: new StringDecoder('utf8') } };
|
||||
let settled = false, timedOut = false, terminating = false, killTimer;
|
||||
let settled = false, timedOut = false, terminating = false, killTimer, ownershipError = null, treeKill = null, closedResult = null, rootClosed = false, treeTimer = null, treePending = null;
|
||||
const processInfo = onSpawn ? { pid: child.pid ?? null, hostPid: process.pid, executable: invocation.executable, argsSha256: createHash('sha256').update(JSON.stringify(invocation.args)).digest('hex'), cwd: project.root, startedAt: at() } : null;
|
||||
const append = (name, text) => { const value = streams[name]; value.characters += text.length; value.head = (value.head + text).slice(0, 240000); value.tail = (value.tail + text).slice(-240000); };
|
||||
const captured = name => { const value = streams[name]; return value.characters <= 240000 ? value.head : value.characters <= 480000 ? value.head + value.tail.slice(value.head.length - value.characters) : value.head + '\n[中间日志超出保存上限,保留开头和结尾]\n' + value.tail; };
|
||||
const finish = (code, error, unknown = false) => {
|
||||
if (settled) return; settled = true; clearTimeout(timeout); clearTimeout(killTimer); signal.removeEventListener('abort', terminate);
|
||||
const stdout = captured('stdout'), stderr = captured('stderr') + (error ? '\n' + error.message : '');
|
||||
const captureTree = async (publish = true) => {
|
||||
const snapshot = await readOwnedProcessTree(processInfo);
|
||||
if (!snapshot.supported) return snapshot;
|
||||
if (snapshot.root) processInfo.rootCreatedAt ??= snapshot.root.createdAt;
|
||||
const known = new Map((processInfo.descendants ?? []).map(value => [value.pid + ':' + value.createdAt, value]));
|
||||
for (const value of snapshot.processes) if (value.pid !== processInfo.pid) known.set(value.pid + ':' + value.createdAt, value);
|
||||
processInfo.descendants = [...known.values()].sort((a, b) => a.pid - b.pid);
|
||||
processInfo.treeObservedAt = at(); processInfo.treeRemainingPids = snapshot.processes.map(value => value.pid);
|
||||
if (publish && !settled) onSpawn(processInfo);
|
||||
return snapshot;
|
||||
};
|
||||
const monitorTree = () => {
|
||||
if (settled || treePending) return;
|
||||
treePending = captureTree().catch(error => { ownershipError = error; processInfo.treeObservationError = error.message; terminate(); }).finally(() => { treePending = null; });
|
||||
};
|
||||
const finish = async (code, error, unknown = false) => {
|
||||
if (onSpawn && terminating && treeKill && !treeKill.done && !unknown) { closedResult = { code, error }; return; }
|
||||
if (onSpawn && treeKill?.error) unknown = true;
|
||||
if (settled) return; settled = true; clearTimeout(timeout); clearTimeout(killTimer); clearInterval(treeTimer); signal.removeEventListener('abort', terminate);
|
||||
if (onSpawn && process.platform === 'win32' && processInfo.pid) {
|
||||
await treePending;
|
||||
try { const snapshot = await captureTree(false); if (snapshot.processes.length) unknown = true; }
|
||||
catch (failure) { ownershipError = failure; processInfo.treeObservationError = failure.message; unknown = true; }
|
||||
}
|
||||
const stdout = captured('stdout'), stderr = captured('stderr') + (error ? '\n' + error.message : '') + (ownershipError ? '\n' + ownershipError.message : '');
|
||||
let collectedEvidence = null, collectedResult = null, evidenceError = null, cleanupError = null;
|
||||
if (isPython && !unknown && code !== null && !error) {
|
||||
try { const saved = readPythonEvidence(invocation, code); collectedEvidence = saved.text; collectedResult = saved.result; } catch (failure) { evidenceError = failure.message; }
|
||||
}
|
||||
const tests = check.kind === 'test' ? isPython ? collectedResult?.tests ?? null : parseTestEvidence(streams.stdout.tail, check.framework) : null;
|
||||
const status = unknown ? 'unknown' : signal.aborted ? 'cancelled'
|
||||
: timedOut || error || code !== 0 || evidenceError || tests?.failed > 0 || collectedResult?.collectionErrors > 0 ? 'failed'
|
||||
: timedOut || error || ownershipError || code !== 0 || evidenceError || tests?.failed > 0 || collectedResult?.collectionErrors > 0 ? 'failed'
|
||||
: (check.kind === 'test' && (!tests || tests.total === 0 || tests.passed !== tests.total || tests.cancelled > 0 || collectedResult?.collectionSkipped > 0)
|
||||
|| isPython && collectedResult?.compilation?.total === 0) ? 'unverified' : 'passed';
|
||||
if (isPython && !unknown) { try { invocation.dispose(); } catch (failure) { cleanupError = failure.message; } }
|
||||
resolve({ status, exitCode: code, durationMs: Date.now() - started, tests, stdout, stderr, timedOut, truncated: streams.stdout.characters > 480000 || streams.stderr.characters > 480000,
|
||||
...(onSpawn ? { process: { ...processInfo, rootExited: rootClosed, treeTermination: terminating ? unknown ? 'unknown' : 'confirmed' : 'not_requested' } } : {}),
|
||||
...(isPython ? { engine: 'python', python: pythonResultBinding(invocation), collectedEvidence, checkInput: invocation.input.toString('utf8'),
|
||||
compilation: collectedResult?.compilation ?? null, collectionErrors: collectedResult?.collectionErrors ?? null,
|
||||
collectionSkipped: collectedResult?.collectionSkipped ?? null, evidenceValidated: !!collectedResult, evidenceError, cleanupError,
|
||||
@@ -223,13 +258,20 @@ export class CheckRunner {
|
||||
};
|
||||
const terminate = () => {
|
||||
if (settled || terminating) return; terminating = true;
|
||||
if (child.pid) { if (process.platform === 'win32') execFile('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true }, () => {}); else { try { process.kill(-child.pid, 'SIGTERM'); } catch { child.kill('SIGTERM'); } } }
|
||||
if (child.pid) { if (process.platform === 'win32') {
|
||||
if (onSpawn) treeKill = { done: false, error: null };
|
||||
execFile('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true }, error => {
|
||||
if (!treeKill) return; treeKill.done = true; treeKill.error = error;
|
||||
if (closedResult) finish(closedResult.code, closedResult.error);
|
||||
});
|
||||
} else { try { process.kill(-child.pid, 'SIGTERM'); } catch { child.kill('SIGTERM'); } } }
|
||||
killTimer = setTimeout(() => finish(null, null, true), 5000);
|
||||
};
|
||||
const timeout = setTimeout(() => { timedOut = true; terminate(); }, this.timeoutMs);
|
||||
signal.addEventListener('abort', terminate, { once: true });
|
||||
for (const name of ['stdout', 'stderr']) { child[name].on('data', bytes => append(name, streams[name].decoder.write(bytes))); child[name].on('end', () => append(name, streams[name].decoder.end())); }
|
||||
child.once('error', error => finish(null, error)); child.once('close', code => finish(code));
|
||||
child.once('error', error => finish(null, error)); child.once('close', code => { rootClosed = true; finish(code); });
|
||||
if (onSpawn) child.once('spawn', () => { processInfo.pid = child.pid; try { onSpawn(processInfo); if (process.platform === 'win32') { monitorTree(); treeTimer = setInterval(monitorTree, 1200); } } catch (error) { ownershipError = error; terminate(); } });
|
||||
if (isPython) { child.stdin.on('error', () => {}); child.stdin.end(invocation.input); }
|
||||
if (signal.aborted) terminate();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user