feat: seal isolated worker recovery, parent integration and review console
This commit is contained in:
+54
-12
@@ -3,13 +3,14 @@ import path from 'node:path';
|
||||
import { spawn, execFile } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import { readOwnedProcessTree } from './owned-process-tree.mjs';
|
||||
import { inspectProject, createCheckList } from './project-inspector.mjs';
|
||||
import { safePath } from '../server/files.mjs';
|
||||
import { assertChangeSetsResolved } from '../shared/contracts/change-sets.mjs';
|
||||
import { assertWorkspacesResolved } from '../shared/contracts/workspaces.mjs';
|
||||
import { verifyWorkspaceProject } from './workspaces/git-workspaces.mjs';
|
||||
import { assertMcpResolved } from './tools/mcp-contract.mjs';
|
||||
import { assertWorkersResolved } from '../shared/contracts/workers.mjs';
|
||||
import { assertWorkersResolved, workerForSession } from '../shared/contracts/workers.mjs';
|
||||
import { bindPythonChecks, preparePythonInvocation, pythonEnvironment, readPythonEvidence, pythonResultBinding } from './python-runtime.mjs';
|
||||
import { projectEnvironment } from './process-environment.mjs';
|
||||
|
||||
@@ -65,6 +66,7 @@ export class CheckRunner {
|
||||
|
||||
async plan(session, names) {
|
||||
assertWorkersResolved(this.store, session.projectId);
|
||||
if (workerForSession(this.store, session)?.status === 'completed') throw conflict('子成果已经封存,请读取原检查记录;重新执行需要新的受限任务。');
|
||||
assertChangeSetsResolved(this.store, session.projectId);
|
||||
assertWorkspacesResolved(this.store, session.projectId, { includePrepared: true });
|
||||
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
|
||||
@@ -105,6 +107,7 @@ export class CheckRunner {
|
||||
|
||||
async start(session, planId, { signal, source = 'user' } = {}) {
|
||||
assertWorkersResolved(this.store, session.projectId);
|
||||
if (workerForSession(this.store, session)?.status === 'completed') throw conflict('子成果已经封存,不重新运行原检查。');
|
||||
assertMcpResolved(this.store, session.projectId);
|
||||
assertChangeSetsResolved(this.store, session.projectId, { includePrepared: true });
|
||||
assertWorkspacesResolved(this.store, session.projectId, { includePrepared: true });
|
||||
@@ -130,6 +133,7 @@ export class CheckRunner {
|
||||
|
||||
async execute(session, plan, job, run, signal) {
|
||||
const task = this.store.core.task.get(job.taskId);
|
||||
let unrecordedProcess = null;
|
||||
try {
|
||||
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
|
||||
const before = await inspectProject(this.store.project(session.projectId));
|
||||
@@ -144,7 +148,9 @@ export class CheckRunner {
|
||||
signal.throwIfAborted();
|
||||
job.currentCheck = { id: check.id, name: check.name, command: check.command, startedAt: at() };
|
||||
this.store.save(session, { type: 'check.command_started', detail: { jobId: job.id, checkId: check.id } });
|
||||
const result = await this.executeCheck(check, this.store.project(session.projectId), signal);
|
||||
const onSpawn = task.workerId ? processInfo => { unrecordedProcess = processInfo; job.currentCheck.process = processInfo; this.store.save(session, { type: 'worker.check_process_owned', detail: { jobId: job.id, pid: processInfo.pid } }); } : null;
|
||||
const result = await this.executeCheck(check, this.store.project(session.projectId), signal, onSpawn);
|
||||
unrecordedProcess = result.process ?? unrecordedProcess;
|
||||
const text = this.redact(result.stdout + (result.stderr ? '\n--- stderr ---\n' + result.stderr : ''));
|
||||
const artifact = this.store.artifactStore.write(session, run, text, { kind: 'check-log', mediaType: 'text/plain' });
|
||||
this.store.core.artifact.set(artifact.id, artifact);
|
||||
@@ -159,6 +165,7 @@ export class CheckRunner {
|
||||
}
|
||||
job.results.push({ id: randomUUID(), name: check.name, kind: check.kind, command: check.command, ...record, logArtifactId: artifact.id, logExcerpt: text.slice(-8000) });
|
||||
this.store.save(session, { type: 'check.result', detail: { jobId: job.id, checkId: check.id, status: record.status, artifactId: artifact.id } });
|
||||
unrecordedProcess = null;
|
||||
if (record.status === 'stale') { job.stale = true; break; }
|
||||
if (['cancelled', 'unknown'].includes(record.status)) break;
|
||||
const after = await inspectProject(this.store.project(session.projectId));
|
||||
@@ -166,8 +173,8 @@ export class CheckRunner {
|
||||
}
|
||||
const final = await inspectProject(this.store.project(session.projectId));
|
||||
job.stale ||= final.sourceFingerprint !== job.sourceFingerprint;
|
||||
if (signal.aborted || job.results.some(result => result.status === 'cancelled')) job.status = 'cancelled';
|
||||
else if (job.results.some(result => result.status === 'unknown')) { job.status = 'unknown'; job.requiresVerification = true; }
|
||||
if (job.results.some(result => result.status === 'unknown')) { job.status = 'unknown'; job.requiresVerification = true; }
|
||||
else if (signal.aborted || job.results.some(result => result.status === 'cancelled')) job.status = 'cancelled';
|
||||
else if (job.stale) job.status = 'stale';
|
||||
else if (job.results.some(result => result.status === 'failed')) job.status = 'failed';
|
||||
else if (!final.scope.complete || job.results.length !== plan.checks.length || job.results.some(result => result.status !== 'passed') || !job.results.some(result => result.kind === 'test' && result.tests?.passed > 0)) job.status = 'unverified';
|
||||
@@ -177,6 +184,10 @@ export class CheckRunner {
|
||||
job.status = signal.aborted ? 'cancelled' : error.status === 409 ? 'stale' : 'failed';
|
||||
job.stale = error.status === 409; job.summary = this.redact(error.message ?? String(error));
|
||||
} finally {
|
||||
if (unrecordedProcess) {
|
||||
job.unreportedProcesses ??= []; job.unreportedProcesses.push(unrecordedProcess);
|
||||
job.status = 'unknown'; job.requiresVerification = true; job.summary = '所属检查进程没有完整持久结果,保留进程归属供原输入核对。';
|
||||
}
|
||||
job.currentCheck = null;
|
||||
job.finishedAt = at();
|
||||
task.verification = { status: job.status, checkJobId: job.id, fingerprint: job.fingerprint, sourceFingerprint: job.sourceFingerprint, checkedAt: job.finishedAt };
|
||||
@@ -186,7 +197,7 @@ export class CheckRunner {
|
||||
return job;
|
||||
}
|
||||
|
||||
async executeCheck(check, project, signal) {
|
||||
async executeCheck(check, project, signal, onSpawn = null) {
|
||||
const started = Date.now();
|
||||
let invocation;
|
||||
const isPython = check.engine === 'python';
|
||||
@@ -196,23 +207,47 @@ export class CheckRunner {
|
||||
const env = isPython ? pythonEnvironment() : projectEnvironment({ NO_COLOR: '1', CI: '1' });
|
||||
const child = spawn(invocation.executable, invocation.args, { cwd: safePath(project), env, windowsHide: true, detached: process.platform !== 'win32', stdio: [isPython ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
|
||||
const streams = { stdout: { head: '', tail: '', characters: 0, decoder: new StringDecoder('utf8') }, stderr: { head: '', tail: '', characters: 0, decoder: new StringDecoder('utf8') } };
|
||||
let settled = false, timedOut = false, terminating = false, killTimer;
|
||||
let settled = false, timedOut = false, terminating = false, killTimer, ownershipError = null, treeKill = null, closedResult = null, rootClosed = false, treeTimer = null, treePending = null;
|
||||
const processInfo = onSpawn ? { pid: child.pid ?? null, hostPid: process.pid, executable: invocation.executable, argsSha256: createHash('sha256').update(JSON.stringify(invocation.args)).digest('hex'), cwd: project.root, startedAt: at() } : null;
|
||||
const append = (name, text) => { const value = streams[name]; value.characters += text.length; value.head = (value.head + text).slice(0, 240000); value.tail = (value.tail + text).slice(-240000); };
|
||||
const captured = name => { const value = streams[name]; return value.characters <= 240000 ? value.head : value.characters <= 480000 ? value.head + value.tail.slice(value.head.length - value.characters) : value.head + '\n[中间日志超出保存上限,保留开头和结尾]\n' + value.tail; };
|
||||
const finish = (code, error, unknown = false) => {
|
||||
if (settled) return; settled = true; clearTimeout(timeout); clearTimeout(killTimer); signal.removeEventListener('abort', terminate);
|
||||
const stdout = captured('stdout'), stderr = captured('stderr') + (error ? '\n' + error.message : '');
|
||||
const captureTree = async (publish = true) => {
|
||||
const snapshot = await readOwnedProcessTree(processInfo);
|
||||
if (!snapshot.supported) return snapshot;
|
||||
if (snapshot.root) processInfo.rootCreatedAt ??= snapshot.root.createdAt;
|
||||
const known = new Map((processInfo.descendants ?? []).map(value => [value.pid + ':' + value.createdAt, value]));
|
||||
for (const value of snapshot.processes) if (value.pid !== processInfo.pid) known.set(value.pid + ':' + value.createdAt, value);
|
||||
processInfo.descendants = [...known.values()].sort((a, b) => a.pid - b.pid);
|
||||
processInfo.treeObservedAt = at(); processInfo.treeRemainingPids = snapshot.processes.map(value => value.pid);
|
||||
if (publish && !settled) onSpawn(processInfo);
|
||||
return snapshot;
|
||||
};
|
||||
const monitorTree = () => {
|
||||
if (settled || treePending) return;
|
||||
treePending = captureTree().catch(error => { ownershipError = error; processInfo.treeObservationError = error.message; terminate(); }).finally(() => { treePending = null; });
|
||||
};
|
||||
const finish = async (code, error, unknown = false) => {
|
||||
if (onSpawn && terminating && treeKill && !treeKill.done && !unknown) { closedResult = { code, error }; return; }
|
||||
if (onSpawn && treeKill?.error) unknown = true;
|
||||
if (settled) return; settled = true; clearTimeout(timeout); clearTimeout(killTimer); clearInterval(treeTimer); signal.removeEventListener('abort', terminate);
|
||||
if (onSpawn && process.platform === 'win32' && processInfo.pid) {
|
||||
await treePending;
|
||||
try { const snapshot = await captureTree(false); if (snapshot.processes.length) unknown = true; }
|
||||
catch (failure) { ownershipError = failure; processInfo.treeObservationError = failure.message; unknown = true; }
|
||||
}
|
||||
const stdout = captured('stdout'), stderr = captured('stderr') + (error ? '\n' + error.message : '') + (ownershipError ? '\n' + ownershipError.message : '');
|
||||
let collectedEvidence = null, collectedResult = null, evidenceError = null, cleanupError = null;
|
||||
if (isPython && !unknown && code !== null && !error) {
|
||||
try { const saved = readPythonEvidence(invocation, code); collectedEvidence = saved.text; collectedResult = saved.result; } catch (failure) { evidenceError = failure.message; }
|
||||
}
|
||||
const tests = check.kind === 'test' ? isPython ? collectedResult?.tests ?? null : parseTestEvidence(streams.stdout.tail, check.framework) : null;
|
||||
const status = unknown ? 'unknown' : signal.aborted ? 'cancelled'
|
||||
: timedOut || error || code !== 0 || evidenceError || tests?.failed > 0 || collectedResult?.collectionErrors > 0 ? 'failed'
|
||||
: timedOut || error || ownershipError || code !== 0 || evidenceError || tests?.failed > 0 || collectedResult?.collectionErrors > 0 ? 'failed'
|
||||
: (check.kind === 'test' && (!tests || tests.total === 0 || tests.passed !== tests.total || tests.cancelled > 0 || collectedResult?.collectionSkipped > 0)
|
||||
|| isPython && collectedResult?.compilation?.total === 0) ? 'unverified' : 'passed';
|
||||
if (isPython && !unknown) { try { invocation.dispose(); } catch (failure) { cleanupError = failure.message; } }
|
||||
resolve({ status, exitCode: code, durationMs: Date.now() - started, tests, stdout, stderr, timedOut, truncated: streams.stdout.characters > 480000 || streams.stderr.characters > 480000,
|
||||
...(onSpawn ? { process: { ...processInfo, rootExited: rootClosed, treeTermination: terminating ? unknown ? 'unknown' : 'confirmed' : 'not_requested' } } : {}),
|
||||
...(isPython ? { engine: 'python', python: pythonResultBinding(invocation), collectedEvidence, checkInput: invocation.input.toString('utf8'),
|
||||
compilation: collectedResult?.compilation ?? null, collectionErrors: collectedResult?.collectionErrors ?? null,
|
||||
collectionSkipped: collectedResult?.collectionSkipped ?? null, evidenceValidated: !!collectedResult, evidenceError, cleanupError,
|
||||
@@ -223,13 +258,20 @@ export class CheckRunner {
|
||||
};
|
||||
const terminate = () => {
|
||||
if (settled || terminating) return; terminating = true;
|
||||
if (child.pid) { if (process.platform === 'win32') execFile('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true }, () => {}); else { try { process.kill(-child.pid, 'SIGTERM'); } catch { child.kill('SIGTERM'); } } }
|
||||
if (child.pid) { if (process.platform === 'win32') {
|
||||
if (onSpawn) treeKill = { done: false, error: null };
|
||||
execFile('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true }, error => {
|
||||
if (!treeKill) return; treeKill.done = true; treeKill.error = error;
|
||||
if (closedResult) finish(closedResult.code, closedResult.error);
|
||||
});
|
||||
} else { try { process.kill(-child.pid, 'SIGTERM'); } catch { child.kill('SIGTERM'); } } }
|
||||
killTimer = setTimeout(() => finish(null, null, true), 5000);
|
||||
};
|
||||
const timeout = setTimeout(() => { timedOut = true; terminate(); }, this.timeoutMs);
|
||||
signal.addEventListener('abort', terminate, { once: true });
|
||||
for (const name of ['stdout', 'stderr']) { child[name].on('data', bytes => append(name, streams[name].decoder.write(bytes))); child[name].on('end', () => append(name, streams[name].decoder.end())); }
|
||||
child.once('error', error => finish(null, error)); child.once('close', code => finish(code));
|
||||
child.once('error', error => finish(null, error)); child.once('close', code => { rootClosed = true; finish(code); });
|
||||
if (onSpawn) child.once('spawn', () => { processInfo.pid = child.pid; try { onSpawn(processInfo); if (process.platform === 'win32') { monitorTree(); treeTimer = setInterval(monitorTree, 1200); } } catch (error) { ownershipError = error; terminate(); } });
|
||||
if (isPython) { child.stdin.on('error', () => {}); child.stdin.end(invocation.input); }
|
||||
if (signal.aborted) terminate();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { gitEnvironment } from './git-process.mjs';
|
||||
|
||||
// Read only: recovery never terminates a process based on a historical PID.
|
||||
// Command lines and environments are deliberately excluded from the inventory.
|
||||
export async function readOwnedProcessTree(info) {
|
||||
if (process.platform !== 'win32') return { processes: [], supported: false };
|
||||
if (!Number.isSafeInteger(info.pid) || info.pid <= 0 || !Number.isFinite(Date.parse(info.startedAt))) throw new Error('原检查进程身份无效。');
|
||||
const script = "$ErrorActionPreference='Stop'; [Console]::OutputEncoding=New-Object System.Text.UTF8Encoding($false); @(Get-CimInstance Win32_Process | ForEach-Object { [pscustomobject]@{ pid=[int]$_.ProcessId; parentPid=[int]$_.ParentProcessId; createdAt=$_.CreationDate.ToUniversalTime().ToString('o'); executable=$_.ExecutablePath } }) | ConvertTo-Json -Compress";
|
||||
const inventory = await new Promise((resolve, reject) => execFile('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script],
|
||||
{ windowsHide: true, timeout: 8000, maxBuffer: 2 * 1024 * 1024, env: gitEnvironment() }, (error, stdout) => {
|
||||
if (error) reject(new Error('原检查进程树无法只读核对:' + error.message.slice(0, 1200)));
|
||||
else { try { resolve(JSON.parse(stdout)); } catch { reject(new Error('原检查进程树输出格式无效,不能声明进程已结束。')); } }
|
||||
}));
|
||||
const all = Array.isArray(inventory) ? inventory : [inventory], selected = new Map(), seeds = new Set(), since = Date.parse(info.startedAt) - 3000;
|
||||
const same = (actual, original) => actual.pid === original.pid && actual.createdAt === original.createdAt;
|
||||
const root = all.find(value => value.pid === info.pid);
|
||||
const rootMatches = root && (info.rootCreatedAt ? root.createdAt === info.rootCreatedAt
|
||||
: Date.parse(root.createdAt) >= since && Date.parse(root.createdAt) <= Date.parse(info.startedAt) + 3000);
|
||||
if (rootMatches) { selected.set(root.pid, root); seeds.add(root.pid); }
|
||||
else if (!root) seeds.add(info.pid);
|
||||
for (const original of info.descendants ?? []) {
|
||||
const actual = all.find(value => same(value, original));
|
||||
if (actual) { selected.set(actual.pid, actual); seeds.add(actual.pid); }
|
||||
else if (!all.some(value => value.pid === original.pid)) seeds.add(original.pid);
|
||||
}
|
||||
for (let depth = 0; depth < 32; depth++) {
|
||||
let added = false;
|
||||
for (const value of all) if (seeds.has(value.parentPid) && !selected.has(value.pid) && Date.parse(value.createdAt) >= since) {
|
||||
selected.set(value.pid, value); seeds.add(value.pid); added = true;
|
||||
if (selected.size > 256) throw new Error('原检查进程树超过256项,不能省略进程后声明结束。');
|
||||
}
|
||||
if (!added) break;
|
||||
if (depth === 31) throw new Error('原检查进程树超过32层,结束状态仍需核对。');
|
||||
}
|
||||
return { supported: true, processes: [...selected.values()].sort((a, b) => a.pid - b.pid), root: rootMatches ? root : null };
|
||||
}
|
||||
@@ -10,6 +10,11 @@ import { assertMcpResolved } from '../tools/mcp-contract.mjs';
|
||||
import { inspectProject } from '../project-inspector.mjs';
|
||||
import { safePath, readText, proposeFile } from '../../server/files.mjs';
|
||||
import { verifiedCheck } from '../delivery-report.mjs';
|
||||
import { ChangeSetManager } from '../workspaces/change-sets.mjs';
|
||||
import { parseTestEvidence } from '../check-runner.mjs';
|
||||
import { verifyPlanEvidence } from '../context/request-plan.mjs';
|
||||
import { repairToolPairs } from '../../server/store.mjs';
|
||||
import { readOwnedProcessTree } from '../owned-process-tree.mjs';
|
||||
|
||||
const at = () => new Date().toISOString();
|
||||
const sourceIdentity = source => Object.fromEntries(['root', 'head', 'reference', 'branch', 'gitDir', 'commonDir', 'indexSha256',
|
||||
@@ -31,10 +36,14 @@ export class WorkerManager {
|
||||
return record ? { ...record, usage: workerUsage(this.store, record), sessionStatus: this.store.session(record.sessionId).status } : { ...child, status: 'unprepared', usage: null };
|
||||
}) })), worker: workerForProject(this.store, session.projectId) };
|
||||
}
|
||||
isActive(sessionId) { return [...this.active.values()].some(value => value.sessionId === sessionId); }
|
||||
childActive(id) { return this.runtime.active.has(id) || !!this.runtime.checks?.isActive(id) || !!this.runtime.changeSets?.isActive(id) || !!this.workspaces?.isActive(id); }
|
||||
isActive(sessionId) { return [...this.active.values()].some(value => value.sessionId === sessionId)
|
||||
|| [...this.store.core.collaboration.values()].some(group => group.sessionId === sessionId && !COLLABORATION_CLOSED.has(group.status) && group.children.some(child => this.childActive(child.sessionId))); }
|
||||
projectActive(projectId, excludeGroupId = null) {
|
||||
return [...this.active.entries()].some(([id, value]) => id !== excludeGroupId && (value.projectId === projectId
|
||||
|| ['preparing', 'collecting', 'reconciling', 'merge_preparing'].includes(value.group.status) && value.group.children.some(child => child.projectId === projectId)));
|
||||
|| ['preparing', 'collecting', 'reconciling', 'merge_preparing'].includes(value.group.status) && value.group.children.some(child => child.projectId === projectId)))
|
||||
|| [...this.store.core.collaboration.values()].some(group => group.id !== excludeGroupId && group.projectId === projectId
|
||||
&& !COLLABORATION_CLOSED.has(group.status) && group.children.some(child => this.childActive(child.sessionId)));
|
||||
}
|
||||
assertIdle(session) {
|
||||
if (this.runtime.projectHasActivity(session.projectId)) throw workerConflict('项目已有执行或协作操作,请等待结束。');
|
||||
@@ -42,9 +51,9 @@ export class WorkerManager {
|
||||
assertChangeSetsResolved(this.store, session.projectId, { includePrepared: true });
|
||||
if (session.changes.some(change => ['pending', 'conflict'].includes(change.status))) throw workerConflict('请先审查当前待处理改动,再创建协作目标。');
|
||||
}
|
||||
own(session, id, fingerprint = null) {
|
||||
own(session, id, fingerprint = null, { current = true } = {}) {
|
||||
const group = this.store.owned('collaboration', id, session.id);
|
||||
if (group.taskId !== session.taskId || group.runId !== session.runId || group.projectId !== session.projectId
|
||||
if (current && (group.taskId !== session.taskId || group.runId !== session.runId) || group.projectId !== session.projectId
|
||||
|| workerHash(collaborationSeal(group)) !== group.fingerprint || fingerprint !== null && fingerprint !== group.fingerprint) throw workerConflict('父任务或原协作分工指纹已变化,请核对原记录。');
|
||||
const artifact = this.store.owned('artifact', group.planArtifactId, session.id);
|
||||
if (artifact.kind !== 'collaboration-plan' || artifact.taskId !== group.taskId || artifact.runId !== group.runId
|
||||
@@ -118,20 +127,24 @@ export class WorkerManager {
|
||||
for (const child of group.children) {
|
||||
signal.throwIfAborted();
|
||||
const existing = this.store.core.workspace.get(child.workspaceId);
|
||||
if (existing && existing.status !== 'ready') throw workerConflict('原工作区记录需要核对,不能再次执行创建。');
|
||||
if (existing && !(existing.status === 'ready' || existing.status === 'prepared' && !existing.creationAttempted && !existing.requiresVerification)) throw workerConflict('原工作区记录需要核对,不能再次执行创建。');
|
||||
if (!existing) {
|
||||
this.save(session, group, 'workspace_intent');
|
||||
const workspace = await this.workspaces.prepare(session, child.label, { id: child.workspaceId, childProjectId: child.projectId, childSessionId: child.sessionId });
|
||||
signal.throwIfAborted();
|
||||
const result = await (await this.workspaces.start(session, workspace.id, workspace.fingerprint)).promise;
|
||||
if (result.status !== 'ready') throw workerConflict(result.summary);
|
||||
} else if (existing.status === 'prepared') {
|
||||
const result = await (await this.workspaces.start(session, existing.id, existing.fingerprint)).promise;
|
||||
if (result.status !== 'ready') throw workerConflict(result.summary);
|
||||
}
|
||||
await this.initializeChild(group, child); this.save(session, group, 'workspace_registered');
|
||||
await this.faultInjector?.('workspace_registered', group, child); signal.throwIfAborted();
|
||||
}
|
||||
await this.sourceFor(group); group.status = 'ready'; group.summary = '两个独立工作区已核对,来源HEAD、索引和已有修改保持。可以开始受限子任务。';
|
||||
} catch (error) {
|
||||
const uncertain = group.children.some(child => { const workspace = this.store.core.workspace.get(child.workspaceId); return workspace && workspace.status !== 'ready'; });
|
||||
const uncertain = group.children.some(child => { const workspace = this.store.core.workspace.get(child.workspaceId); return workspace && (workspace.status !== 'ready'
|
||||
|| !this.store.core.worker.has(child.id) && this.store.sessions.get(child.sessionId)?.taskId); });
|
||||
group.status = uncertain ? 'unknown' : signal.aborted ? 'partially_prepared' : 'stale'; group.requiresVerification = uncertain;
|
||||
group.summary = this.runtime.provider.redact(error.message ?? String(error));
|
||||
}
|
||||
@@ -151,14 +164,20 @@ export class WorkerManager {
|
||||
if (this.store.core.worker.has(child.id)) { workerForSession(this.store, this.store.session(child.sessionId)); return; }
|
||||
const project = this.store.project(child.projectId), session = this.store.session(child.sessionId);
|
||||
await verifyWorkspaceProject(this.store, project);
|
||||
if (session.taskId || session.messages.length || session.changes.length) throw workerConflict('新工作区已有其他任务记录,不能覆盖其身份。');
|
||||
const config = { ...group.config, maxTaskTokens: child.budget.maxTokens }, task = createTask(session, child.goal, config);
|
||||
task.id = child.taskId; task.origin = 'isolated-worker'; task.workspaceIdentity = projectWorkspaceIdentity(project);
|
||||
task.budget.maxTokens = child.budget.maxTokens; task.workerId = child.id;
|
||||
task.constraints = ['允许修改范围:' + child.paths.join('、'), '只接受原提案应用和当前工程检查;禁止独立Git提交、任意终端及外部工具。'];
|
||||
const run = createRun(session, task, config); run.workspaceIdentity = projectWorkspaceIdentity(project);
|
||||
this.store.core.task.set(task.id, task); this.store.core.run.set(run.id, run);
|
||||
session.taskId = task.id; session.runId = run.id; session.status = 'paused'; this.store.finishRun(session, run, 'worker_prepared');
|
||||
if (session.messages.length || session.changes.length) throw workerConflict('新工作区已有其他任务记录,不能覆盖其身份。');
|
||||
const config = { ...group.config, maxTaskTokens: child.budget.maxTokens };
|
||||
let task = this.store.core.task.get(session.taskId), run = this.store.core.run.get(session.runId);
|
||||
if (session.taskId) {
|
||||
if (task?.id !== child.taskId || task.goal !== child.goal || task.workerId !== child.id || task.origin !== 'isolated-worker'
|
||||
|| run?.taskId !== task.id || run.stopReason !== 'worker_prepared' || !run.finishedAt) throw workerConflict('子任务初始化有其他或不完整记录,请保留现场核对。');
|
||||
} else {
|
||||
task = createTask(session, child.goal, config); task.id = child.taskId; task.origin = 'isolated-worker'; task.workspaceIdentity = projectWorkspaceIdentity(project);
|
||||
task.budget.maxTokens = child.budget.maxTokens; task.workerId = child.id;
|
||||
task.constraints = ['允许修改范围:' + child.paths.join('、'), '只接受原提案应用和当前工程检查;禁止独立Git提交、任意终端及外部工具。'];
|
||||
run = createRun(session, task, config); run.workspaceIdentity = projectWorkspaceIdentity(project);
|
||||
this.store.core.task.set(task.id, task); this.store.core.run.set(run.id, run);
|
||||
session.taskId = task.id; session.runId = run.id; session.status = 'paused'; this.store.finishRun(session, run, 'worker_prepared');
|
||||
}
|
||||
const { files, inspection } = await this.inventory(project), git = await this.workspaces.snapshot(project);
|
||||
if (git.head !== group.source.identity.head || git.dirty) throw workerConflict('子工作区初始HEAD或源码不符合封存来源。');
|
||||
const worker = { schemaVersion: 1, ...child, collaborationId: group.id, parentProjectId: group.projectId, parentSessionId: group.sessionId,
|
||||
@@ -271,6 +290,7 @@ export class WorkerManager {
|
||||
if (git.head !== worker.baseCommit || git.indexSha256 !== baseline.git.indexSha256 || git.reference !== baseline.git.reference
|
||||
|| git.gitDir !== baseline.git.gitDir || git.commonDir !== baseline.git.commonDir
|
||||
|| JSON.stringify(git.configuration) !== JSON.stringify(baseline.git.configuration)) throw workerConflict('子任务HEAD、索引或Git配置已变化,独立提交或暂存不能作为原成果');
|
||||
if (git.changedPathCount > 32 || git.changedPaths.some(file => !after.has(workerPathKey(file.path)))) throw workerConflict('实际Git修改包含删除、链接、排除文件或超出32文件范围,不能遗漏后合并');
|
||||
const paths = [...new Set([...before.keys(), ...after.keys()])].filter(key => before.get(key)?.sha256 !== after.get(key)?.sha256);
|
||||
if (!paths.length || paths.length > 32) throw workerConflict('可交付成果需要1到32个实际文本改动');
|
||||
let bytes = 0;
|
||||
@@ -302,7 +322,14 @@ export class WorkerManager {
|
||||
}).sort((a, b) => a.path.localeCompare(b.path));
|
||||
const check = verifiedCheck(this.store, session, inspection);
|
||||
if (!check) throw workerConflict('缺少与当前源码一致、日志完整且测试数量明确的通过记录');
|
||||
this.runtime.checkRunner().planFor(session, check.planId);
|
||||
const plan = this.runtime.checkRunner().planFor(session, check.planId);
|
||||
for (const result of check.results) if (result.kind === 'test' && result.engine !== 'python') {
|
||||
const spec = plan.checks.find(value => value.name === result.name);
|
||||
const evidence = parseTestEvidence(this.store.artifactStore.read(this.store.owned('artifact', result.logArtifactId, session.id)), spec?.framework);
|
||||
if (JSON.stringify(evidence) !== JSON.stringify(result.tests)) throw workerConflict('检查数量与原日志不一致');
|
||||
}
|
||||
const cache = new Map();
|
||||
for (const request of this.store.core.context_plan.values()) if (request.taskId === worker.taskId) verifyPlanEvidence(this.store, session, request, cache);
|
||||
const latest = await this.inventory(project);
|
||||
if (latest.inspection.sourceFingerprint !== inspection.sourceFingerprint || JSON.stringify(latest.files) !== JSON.stringify(current)) throw workerConflict('核验期间子工程发生变化');
|
||||
return { schemaVersion: 1, workerId: worker.id, collaborationId: worker.collaborationId, workerFingerprint: worker.fingerprint,
|
||||
@@ -314,6 +341,319 @@ export class WorkerManager {
|
||||
...(result.metricsArtifactId ? { metricsArtifactId: result.metricsArtifactId, metricsSha256: result.metricsSha256 } : {}),
|
||||
})) }, usage: workerUsage(this.store, worker), runIds: worker.runIds };
|
||||
}
|
||||
async verifiedResult(group, child) {
|
||||
const worker = this.store.core.worker.get(child.id), session = this.store.session(child.sessionId);
|
||||
if (!worker || worker.status !== 'completed' || worker.requiresVerification || !worker.result) throw workerConflict('子任务尚未完成原成果封存:' + child.label);
|
||||
const artifact = this.store.owned('artifact', worker.result.artifactId, session.id);
|
||||
if (artifact.kind !== 'worker-result' || artifact.taskId !== worker.taskId || artifact.runId !== worker.currentRunId || artifact.sha256 !== worker.result.sha256) throw workerConflict('子成果工件归属或原散列不一致:' + child.label);
|
||||
const original = JSON.parse(this.store.artifactStore.read(artifact));
|
||||
if (workerHash(original) !== worker.result.fingerprint || original.workerId !== worker.id || original.collaborationId !== group.id
|
||||
|| original.workerFingerprint !== worker.fingerprint || original.taskId !== worker.taskId) throw workerConflict('子成果与原分工不一致:' + child.label);
|
||||
const current = await this.output(session, worker);
|
||||
if (JSON.stringify(original) !== JSON.stringify(current)) throw workerConflict('子工程、检查、账本或原成果已变化:' + child.label);
|
||||
return { worker, artifact, output: original };
|
||||
}
|
||||
async collectionFor(session, group) {
|
||||
await this.sourceFor(group);
|
||||
const proofs = [];
|
||||
for (const child of group.children) proofs.push(await this.verifiedResult(group, child));
|
||||
const conflicts = [], targets = [], project = this.store.project(group.projectId);
|
||||
const all = proofs.flatMap(proof => proof.output.files.map(file => ({ ...file, workerId: proof.worker.id, label: proof.worker.label })));
|
||||
for (let index = 0; index < all.length; index++) {
|
||||
const file = all[index], key = workerPathKey(file.path);
|
||||
for (const other of all.slice(index + 1)) {
|
||||
const otherKey = workerPathKey(other.path);
|
||||
if (key === otherKey || key.startsWith(otherKey + '/') || otherKey.startsWith(key + '/')) conflicts.push({ path: file.path, workerIds: [file.workerId, other.workerId], reason: '两个子成果修改同一路径或文件/目录相互包含;需要明确分工后重新审查。' });
|
||||
}
|
||||
const dirty = group.source.changedPaths.find(value => [value.path, value.originalPath].filter(Boolean).some(relative => {
|
||||
const dirtyKey = workerPathKey(relative); return key === dirtyKey || key.startsWith(dirtyKey + '/') || dirtyKey.startsWith(key + '/');
|
||||
}));
|
||||
if (dirty) { conflicts.push({ path: file.path, workerIds: [file.workerId], reason: '父工程已有暂存、未提交或未跟踪修改:' + dirty.path }); continue; }
|
||||
try {
|
||||
const target = safePath(project, file.path, true), exists = fs.existsSync(target);
|
||||
if (exists !== (file.before !== null)) throw workerConflict('父文件存在状态与子HEAD不一致,不能覆盖已有内容。');
|
||||
const original = exists ? readText(project, file.path) : null, before = original ? fs.readFileSync(target) : null;
|
||||
const childBefore = file.before === null ? null : Buffer.from(file.before, 'base64'), after = Buffer.from(file.after, 'base64');
|
||||
if (workerHash(after) !== file.sha256 || childBefore && workerHash(childBefore) !== file.beforeSha256) throw workerConflict('子成果原字节不一致。');
|
||||
const normalized = value => value.replace(/^\uFEFF/, '').replaceAll('\r\n', '\n');
|
||||
if (before && normalized(before.toString('utf8')) !== normalized(childBefore.toString('utf8'))) throw workerConflict('父工程原内容与子HEAD不一致。');
|
||||
let content = new TextDecoder('utf8', { fatal: true }).decode(after);
|
||||
if (original) {
|
||||
const source = original.content;
|
||||
if (/\r\n/.test(source) && !/(?<!\r)\n/.test(source)) content = content.replace(/\r\n|\n/g, '\r\n');
|
||||
else if (!/\r\n/.test(source)) content = content.replaceAll('\r\n', '\n');
|
||||
if (original.bom) content = '\uFEFF' + content.replace(/^\uFEFF/, '');
|
||||
} else if (after.subarray(0, 3).equals(Buffer.from([0xef, 0xbb, 0xbf]))) content = '\uFEFF' + content;
|
||||
targets.push({ path: file.path, workerId: file.workerId, label: file.label, beforeSha256: original?.sha256 ?? null,
|
||||
sha256: workerHash(Buffer.from(content, 'utf8')), childSha256: file.sha256, content });
|
||||
} catch (error) { conflicts.push({ path: file.path, workerIds: [file.workerId], reason: this.runtime.provider.redact(error.message ?? String(error)) }); }
|
||||
}
|
||||
if (all.length > 32 || targets.reduce((sum, file) => sum + Buffer.byteLength(file.content, 'utf8'), 0) > 2 * 1024 * 1024) conflicts.push({ path: '(整批)', workerIds: group.children.map(child => child.id), reason: '合并超过32文件或2MB建议内容,请缩小交付范围。' });
|
||||
await this.sourceFor(group);
|
||||
return { schemaVersion: 1, collaborationId: group.id, groupFingerprint: group.fingerprint, projectId: group.projectId,
|
||||
sessionId: session.id, taskId: group.taskId, runId: group.runId, source: group.source.identity,
|
||||
workers: proofs.map(proof => ({ workerId: proof.worker.id, artifactId: proof.artifact.id, sha256: proof.artifact.sha256, fingerprint: proof.worker.fingerprint,
|
||||
checkJobId: proof.output.check.id, paths: proof.output.files.map(file => file.path) })),
|
||||
paths: all.map(file => file.path).sort(), conflicts, targets: targets.sort((a, b) => a.path.localeCompare(b.path)) };
|
||||
}
|
||||
collect(session, id, fingerprint) {
|
||||
const group = this.own(session, id, fingerprint);
|
||||
if (!['ready_to_collect', 'collected', 'conflict'].includes(group.status) || group.requiresVerification || group.merge) throw workerConflict('先完成两个子任务,已有合并审查不能重新归集。');
|
||||
return this.launch(session, group, 'collecting', async signal => {
|
||||
try {
|
||||
const collection = await this.collectionFor(session, group); signal.throwIfAborted();
|
||||
const artifact = this.store.artifactStore.write(session, this.store.owned('run', group.runId, session.id), collection, { kind: 'worker-collection' });
|
||||
this.store.core.artifact.set(artifact.id, artifact);
|
||||
group.collection = { artifactId: artifact.id, sha256: artifact.sha256, fingerprint: workerHash(collection), conflicts: collection.conflicts, paths: collection.paths };
|
||||
group.status = collection.conflicts.length ? 'conflict' : 'collected';
|
||||
group.summary = collection.conflicts.length ? '原成果已封存,存在具体路径冲突;父工程未改变。' : '原子成果、当前检查和来源均已核对,可以准备父Task的独立变更集审查。';
|
||||
} catch (error) { group.status = signal.aborted ? 'ready_to_collect' : 'stale'; group.summary = this.runtime.provider.redact(error.message ?? String(error)); }
|
||||
this.save(session, group, group.status); return group;
|
||||
});
|
||||
}
|
||||
async readCollection(session, group) {
|
||||
this.own(session, group.id, null, { current: false });
|
||||
if (!group.collection) throw workerConflict('当前协作尚未封存成果归集。');
|
||||
const artifact = this.store.owned('artifact', group.collection.artifactId, session.id);
|
||||
if (artifact.kind !== 'worker-collection' || artifact.taskId !== group.taskId || artifact.runId !== group.runId || artifact.sha256 !== group.collection.sha256) throw workerConflict('成果归集工件归属或散列不一致。');
|
||||
const record = JSON.parse(this.store.artifactStore.read(artifact));
|
||||
if (record.collaborationId !== group.id || record.groupFingerprint !== group.fingerprint || workerHash(record) !== group.collection.fingerprint) throw workerConflict('成果归集与原协作不一致。');
|
||||
return record;
|
||||
}
|
||||
async verifyCollection(session, group) {
|
||||
const original = await this.readCollection(session, group), current = await this.collectionFor(session, group);
|
||||
if (JSON.stringify(original) !== JSON.stringify(current) || current.conflicts.length) throw workerConflict('来源、原成果或检查证据已变化,不能使用旧合并审查。');
|
||||
return original;
|
||||
}
|
||||
prepareMerge(session, id, fingerprint) {
|
||||
const group = this.own(session, id, fingerprint);
|
||||
if (group.status !== 'collected' || group.merge && group.merge.status !== 'proposing' || group.requiresVerification) throw workerConflict('只有没有冲突的原成果可以准备一次父合并审查。');
|
||||
return this.launch(session, group, 'merge_preparing', async signal => {
|
||||
try {
|
||||
const collection = await this.verifyCollection(session, group); signal.throwIfAborted();
|
||||
if (session.changes.some(change => ['pending', 'conflict'].includes(change.status) && change.sourceBinding?.collaborationId !== group.id)) throw workerConflict('父Task已有未处理提案,不能混入子成果。');
|
||||
group.merge ??= { collectionSha256: group.collection.sha256, changeIds: [], status: 'proposing' };
|
||||
const recovered = this.mergeProposals(session, group, collection);
|
||||
group.merge.changeIds = recovered.map(change => change.id); this.save(session, group, 'merge_intent');
|
||||
for (const file of collection.targets) {
|
||||
signal.throwIfAborted();
|
||||
if (recovered.some(change => change.path === file.path)) continue;
|
||||
const proof = collection.workers.find(value => value.workerId === file.workerId);
|
||||
const result = proposeFile(this.store, session, { path: file.path, base_sha256: file.beforeSha256, content: file.content, reason: '合并已核验子任务「' + file.label + '」的原成果' },
|
||||
{ sourceBinding: { collaborationId: group.id, workerId: file.workerId, resultArtifactId: proof.artifactId, resultSha256: proof.sha256, collectionSha256: group.collection.sha256 } });
|
||||
if (!result.change_id) throw workerConflict('原成果没有形成新的父提案,不能补造合并成功。');
|
||||
group.merge.changeIds.push(result.change_id); this.save(session, group, 'merge_proposal');
|
||||
await this.faultInjector?.('merge_proposal', group, file);
|
||||
}
|
||||
await this.verifyCollection(session, group); signal.throwIfAborted();
|
||||
const manager = new ChangeSetManager(this.store, { projectBusy: (projectId, exceptId) => this.runtime.projectHasActivity(projectId, exceptId, { excludeGroupId: group.id }) });
|
||||
const set = await manager.prepare(session, group.merge.changeIds, { collaborationBinding: { id: group.id, fingerprint: group.fingerprint,
|
||||
collectionArtifactId: group.collection.artifactId, collectionSha256: group.collection.sha256 } });
|
||||
group.merge.changeSetId = set.id; group.merge.fingerprint = set.fingerprint; group.merge.status = 'prepared';
|
||||
group.status = 'merge_prepared'; group.summary = '父Task独立提案与原字节变更集已封存,等待明确审查应用;父工程尚未修改。';
|
||||
session.status = 'needs_review';
|
||||
} catch (error) {
|
||||
group.status = group.merge ? 'unknown' : 'stale'; group.requiresVerification = !!group.merge;
|
||||
group.summary = this.runtime.provider.redact(error.message ?? String(error));
|
||||
}
|
||||
this.save(session, group, group.status); return group;
|
||||
});
|
||||
}
|
||||
mergeProposals(session, group, collection) {
|
||||
const changes = session.changes.filter(change => change.sourceBinding?.collaborationId === group.id);
|
||||
if (new Set(changes.map(change => workerPathKey(change.path))).size !== changes.length
|
||||
|| group.merge.collectionSha256 !== group.collection.sha256
|
||||
|| group.merge.changeIds.some(id => !changes.some(change => change.id === id))) throw workerConflict('原合并提案账本不连续,不能补造或重放。');
|
||||
for (const change of changes) {
|
||||
const target = collection.targets.find(file => file.path === change.path), proof = collection.workers.find(worker => worker.workerId === target?.workerId);
|
||||
if (!target || !proof || change.taskId !== group.taskId || change.runId !== group.runId || change.status !== 'pending' || change.changeSetId
|
||||
|| change.baseSha256 !== target.beforeSha256 || change.sha256 !== target.sha256 || workerHash(change.content) !== target.sha256
|
||||
|| change.sourceBinding.workerId !== target.workerId || change.sourceBinding.resultArtifactId !== proof.artifactId
|
||||
|| change.sourceBinding.resultSha256 !== proof.sha256 || change.sourceBinding.collectionSha256 !== group.collection.sha256) throw workerConflict('原父提案与封存子成果不一致,不能续封合并。');
|
||||
}
|
||||
return collection.targets.flatMap(file => changes.filter(change => change.path === file.path));
|
||||
}
|
||||
async recoverMerge(session, group) {
|
||||
if (!group.merge) return;
|
||||
const sets = [...this.store.core.change_set.values()].filter(set => set.collaborationBinding?.id === group.id || set.id === group.merge.changeSetId);
|
||||
if (sets.length > 1) throw workerConflict('原协作存在多个父批次,不能猜测或重放。');
|
||||
if (sets.length) {
|
||||
const set = sets[0], binding = set.collaborationBinding;
|
||||
if (set.sessionId !== session.id || set.taskId !== group.taskId || binding?.fingerprint !== group.fingerprint
|
||||
|| binding.collectionArtifactId !== group.collection?.artifactId || binding.collectionSha256 !== group.collection?.sha256
|
||||
|| !['prepared', 'completed'].includes(set.status) || set.requiresVerification
|
||||
|| JSON.stringify(set.files.map(file => file.changeId)) !== JSON.stringify(group.merge.changeIds)) throw workerConflict('父批次仍需原恢复流程核对或只能保留现场结束。');
|
||||
this.runtime.changeSetManager().snapshots(session, set);
|
||||
group.merge.changeSetId = set.id; group.merge.fingerprint = set.fingerprint; group.merge.status = 'prepared';
|
||||
} else {
|
||||
const collection = await this.verifyCollection(session, group);
|
||||
group.merge.changeIds = this.mergeProposals(session, group, collection).map(change => change.id); group.merge.status = 'proposing';
|
||||
}
|
||||
}
|
||||
async verifyMerge(session, set) {
|
||||
const linked = this.groups(session).find(group => group.merge?.changeSetId === set.id);
|
||||
if (!set.collaborationBinding && !linked) return;
|
||||
const group = this.own(session, set.collaborationBinding?.id ?? linked.id, set.collaborationBinding?.fingerprint ?? linked.fingerprint);
|
||||
if (group.status !== 'merge_prepared' || group.requiresVerification || group.merge?.changeSetId !== set.id || group.merge.fingerprint !== set.fingerprint
|
||||
|| set.collaborationBinding?.collectionArtifactId !== group.collection?.artifactId || set.collaborationBinding?.collectionSha256 !== group.collection?.sha256
|
||||
|| JSON.stringify(set.files.map(file => file.changeId)) !== JSON.stringify(group.merge.changeIds)) throw workerConflict('父变更集与原协作/归集记录不一致,不能应用。');
|
||||
const collection = await this.verifyCollection(session, group);
|
||||
for (const file of set.files) {
|
||||
const target = collection.targets.find(value => value.path === file.path), change = session.changes.find(value => value.id === file.changeId);
|
||||
if (!target || target.sha256 !== file.sha256 || target.beforeSha256 !== file.baseSha256
|
||||
|| change.sourceBinding?.collaborationId !== group.id || change.sourceBinding?.workerId !== target.workerId
|
||||
|| change.sourceBinding?.collectionSha256 !== group.collection.sha256) throw workerConflict('父提案的原子成果绑定失效。');
|
||||
}
|
||||
}
|
||||
async finalize(session, id, fingerprint) {
|
||||
const group = this.own(session, id, fingerprint); this.assertIdle(session);
|
||||
if (group.status !== 'merge_prepared' || !group.merge?.changeSetId) throw workerConflict('先审查应用原父变更集,再核对交付。');
|
||||
const set = this.store.owned('change_set', group.merge.changeSetId, session.id);
|
||||
if (set.status !== 'completed' || set.requiresVerification || set.fingerprint !== group.merge.fingerprint) throw workerConflict('原父变更集没有完整审查应用结果。');
|
||||
this.runtime.changeSetManager().snapshots(session, set);
|
||||
const inspection = await inspectProject(this.store.project(group.projectId)), check = verifiedCheck(this.store, session, inspection);
|
||||
if (!check) throw workerConflict('合并后父工程需要当前源码的真实通过检查。');
|
||||
for (const file of set.files) if (readText(this.store.project(group.projectId), file.path).sha256 !== file.sha256) throw workerConflict('父合并文件已变化,请先审查实际修复与重新归集。');
|
||||
const delivery = [...this.store.core.delivery_job.values()].find(job => job.taskId === group.taskId && job.sessionId === session.id
|
||||
&& job.status === 'completed' && !job.requiresVerification && job.commit === inspection.git.head && job.sourceFingerprint === inspection.sourceFingerprint);
|
||||
if (!delivery || !delivery.reportArtifactId) throw workerConflict('父工程尚未完成当前成果的本地Git交付。');
|
||||
const report = this.store.owned('artifact', delivery.reportArtifactId, session.id); this.store.artifactStore.read(report);
|
||||
for (const child of group.children) await this.verifiedResult(group, child);
|
||||
group.delivery = { jobId: delivery.id, commit: delivery.commit, checkJobId: check.id, reportArtifactId: report.id, reportSha256: report.sha256 };
|
||||
group.status = 'completed'; group.finishedAt = at(); group.summary = '两个隔离子成果已审查合并,父工程重新检查通过并完成本地Git交付。';
|
||||
session.status = 'completed'; this.save(session, group, 'completed'); return group;
|
||||
}
|
||||
async observe(group) {
|
||||
const source = await this.workspaces.snapshot(this.store.project(group.projectId)), children = [];
|
||||
for (const child of group.children) {
|
||||
const workspace = this.store.core.workspace.get(child.workspaceId), worker = this.store.core.worker.get(child.id);
|
||||
if (alive(workspace?.ownerPid) || alive(workspace?.gitPid) || alive(worker?.ownerPid)) throw workerConflict('历史所属PID仍存活,请先人工核对进程归属;工作台不会按历史PID停止进程。');
|
||||
let inventory = null, inventoryError = null;
|
||||
if (worker) {
|
||||
this.baseline(this.store.session(child.sessionId), worker);
|
||||
try { inventory = await this.inventory(this.store.project(child.projectId)); } catch (error) { inventoryError = error.message; }
|
||||
}
|
||||
const checks = [];
|
||||
for (const job of [...this.store.core.check_job.values()].filter(value => value.taskId === child.taskId)) {
|
||||
const processes = [job.currentCheck?.process, ...(job.unreportedProcesses ?? []), ...job.results.map(result => result.process)].filter(Boolean);
|
||||
if (processes.some(processInfo => !processInfo.rootExited && alive(processInfo.pid))) throw workerConflict('原检查PID仍存活,结束状态需要人工核对;不能自动重试或按历史PID终止。');
|
||||
if (job.requiresVerification || job.status === 'unknown') for (const processInfo of processes) {
|
||||
const tree = await readOwnedProcessTree(processInfo);
|
||||
if (tree.processes.length) throw workerConflict('原检查子进程PID仍存活:' + tree.processes.map(value => value.pid).join('、') + ';不能按历史PID自动终止或解除核对。');
|
||||
}
|
||||
checks.push({ id: job.id, status: job.status, requiresVerification: !!job.requiresVerification, processes,
|
||||
results: job.results.map(result => ({ id: result.id, status: result.status, logArtifactId: result.logArtifactId,
|
||||
logSha256: this.store.core.artifact.get(result.logArtifactId)?.sha256 ?? null })) });
|
||||
}
|
||||
const session = this.store.sessions.get(child.sessionId);
|
||||
const tools = [...this.store.core.tool.values()].filter(value => value.taskId === child.taskId && value.requiresVerification).map(tool => {
|
||||
const call = session?.providerMessages.flatMap(message => message.tool_calls ?? []).find(call => call.id === tool.callId);
|
||||
const input = call?.function.arguments;
|
||||
if (typeof input !== 'string' || workerHash(input) !== tool.argsSha256) throw workerConflict('原未知工具的输入哈希不一致,请保留现场。');
|
||||
return { id: tool.id, callId: tool.callId, tool: tool.tool, state: tool.state, argsSha256: tool.argsSha256,
|
||||
input: this.runtime.provider.redact(input).slice(0, 24000), inputTruncated: input.length > 24000, resultArtifactId: tool.resultArtifactId };
|
||||
});
|
||||
const changes = (session?.changes ?? []).filter(change => change.taskId === child.taskId).map(change => {
|
||||
let sha256 = null, error = null;
|
||||
try { const target = safePath(this.store.project(child.projectId), change.path, true); sha256 = fs.existsSync(target) ? readText(this.store.project(child.projectId), change.path).sha256 : null; }
|
||||
catch (failure) { error = failure.message; }
|
||||
return { id: change.id, path: change.path, status: change.status, baseSha256: change.baseSha256, proposedSha256: change.sha256, actualSha256: sha256, error };
|
||||
});
|
||||
children.push({ id: child.id, workspaceId: child.workspaceId, workspaceStatus: workspace?.status ?? 'absent',
|
||||
workerStatus: worker?.status ?? 'absent', taskId: child.taskId, sessionId: child.sessionId, projectId: child.projectId,
|
||||
inventoryFingerprint: inventory ? workerHash(inventory.files) : null, sourceFingerprint: inventory?.inspection.sourceFingerprint ?? null, inventoryError,
|
||||
tools, checks, changes, changeSets: [...this.store.core.change_set.values()].filter(set => set.taskId === child.taskId).map(set => ({ id: set.id, status: set.status, requiresVerification: !!set.requiresVerification, observationFingerprint: set.observationFingerprint ?? null })) });
|
||||
}
|
||||
const parent = this.store.session(group.sessionId);
|
||||
return { schemaVersion: 1, kind: 'worker-recovery-observation', collaborationId: group.id, groupFingerprint: group.fingerprint,
|
||||
projectId: group.projectId, sessionId: group.sessionId, taskId: group.taskId, runId: group.runId, source: sourceIdentity(source), children,
|
||||
parentChanges: parent.changes.filter(change => change.sourceBinding?.collaborationId === group.id).map(change => ({ id: change.id, path: change.path, status: change.status, sha256: change.sha256 })),
|
||||
parentChangeSets: [...this.store.core.change_set.values()].filter(set => set.taskId === group.taskId).map(set => ({ id: set.id, status: set.status, requiresVerification: !!set.requiresVerification })),
|
||||
note: '原输入及当前现场供用户核对;未报告的检查或工具结果保持未知,不补造完成、不执行模型或旧工具。' };
|
||||
}
|
||||
reconcile(session, id, fingerprint) {
|
||||
const group = this.own(session, id, fingerprint);
|
||||
if (!group.requiresVerification || group.status !== 'unknown') throw workerConflict('只有原结果未知的协作需要现场核对。');
|
||||
return this.launch(session, group, 'reconciling', async signal => {
|
||||
this.preparingGroupId = group.id;
|
||||
try {
|
||||
for (const child of group.children) {
|
||||
const workspace = this.store.core.workspace.get(child.workspaceId);
|
||||
if (workspace?.status === 'unknown') await this.workspaces.reconcile(session, workspace.id, workspace.fingerprint);
|
||||
if (workspace?.status === 'ready' && !this.store.core.worker.has(child.id)) await this.initializeChild(group, child);
|
||||
}
|
||||
const observed = await this.observe(group); signal.throwIfAborted();
|
||||
const artifact = this.store.artifactStore.write(session, this.store.owned('run', group.runId, session.id), observed, { kind: 'worker-recovery' });
|
||||
this.store.core.artifact.set(artifact.id, artifact); group.evidence = { artifactId: artifact.id, sha256: artifact.sha256 };
|
||||
group.summary = '原输入与现场核对已封存,请阅读后明确确认;不会自动发送模型、重放工具或补造检查通过。';
|
||||
} catch (error) { group.summary = this.runtime.provider.redact(error.message ?? String(error)); }
|
||||
group.status = 'unknown'; group.requiresVerification = true; this.save(session, group, 'observed'); return group;
|
||||
});
|
||||
}
|
||||
readEvidence(session, group) {
|
||||
this.own(session, group.id, null, { current: false });
|
||||
const artifact = this.store.owned('artifact', group.evidence?.artifactId, session.id);
|
||||
if (artifact.kind !== 'worker-recovery' || artifact.taskId !== group.taskId || artifact.runId !== group.runId || artifact.sha256 !== group.evidence.sha256) throw workerConflict('原核对工件不属于当前父协作。');
|
||||
const record = JSON.parse(this.store.artifactStore.read(artifact));
|
||||
if (record.groupFingerprint !== group.fingerprint || record.collaborationId !== group.id) throw workerConflict('核对记录与原协作不一致。');
|
||||
return record;
|
||||
}
|
||||
async acknowledge(session, id, fingerprint, evidenceSha256, retain = false) {
|
||||
const group = this.own(session, id, fingerprint);
|
||||
if (group.status !== 'unknown' || !group.requiresVerification || typeof evidenceSha256 !== 'string' || evidenceSha256 !== group.evidence?.sha256) throw workerConflict('请先阅读原输入与现场核对工件,再提交原SHA256。');
|
||||
const started = this.launch(session, group, 'reconciling', async signal => {
|
||||
this.preparingGroupId = group.id;
|
||||
try {
|
||||
const original = this.readEvidence(session, group), current = await this.observe(group); signal.throwIfAborted();
|
||||
if (JSON.stringify(original) !== JSON.stringify(current)) throw workerConflict('原输入或现场在核对后发生变化,请重新核对。');
|
||||
const unresolvedSets = current.children.some(child => child.changeSets.some(set => set.requiresVerification)) || current.parentChangeSets.some(set => set.requiresVerification);
|
||||
if (unresolvedSets) throw workerConflict('原文件批次仍待核对或回退,请先在所属任务记录处理实际原字节。');
|
||||
const merged = group.merge?.changeSetId && this.store.core.change_set.get(group.merge.changeSetId)?.status === 'completed';
|
||||
if (!retain && !merged && JSON.stringify(current.source) !== JSON.stringify(group.source.identity)) throw workerConflict('父来源已经变化,只能明确保留现场后重新规划。');
|
||||
if (current.children.some(child => !['ready', 'absent', 'prepared', 'discarded', 'resolved_external'].includes(child.workspaceStatus))) throw workerConflict('原工作区仍需专门核对,不能开始子执行或结束恢复。');
|
||||
if (!retain && current.children.some(child => ['discarded', 'resolved_external'].includes(child.workspaceStatus))) throw workerConflict('原工作区没有核对为可执行,只能明确保留现场结束。');
|
||||
if (!retain) await this.recoverMerge(session, group);
|
||||
for (const child of group.children) {
|
||||
const worker = this.store.core.worker.get(child.id); if (!worker) continue;
|
||||
const childSession = this.store.session(child.sessionId);
|
||||
for (const tool of this.store.core.tool.values()) if (tool.taskId === child.taskId && tool.requiresVerification) {
|
||||
tool.originalState = tool.state; tool.state = 'resolved_external'; tool.requiresVerification = false;
|
||||
tool.review = { source: 'user', evidenceSha256, reviewedAt: at(), result: 'unknown-preserved' };
|
||||
const run = this.store.core.run.get(tool.runId);
|
||||
if (run) {
|
||||
run.checkpoint.requiresVerification = run.checkpoint.requiresVerification.filter(id => id !== tool.id);
|
||||
run.checkpoint.pendingToolIds = run.checkpoint.pendingToolIds.filter(id => id !== tool.id);
|
||||
run.checkpoint.reviewedUnknownToolIds ??= []; if (!run.checkpoint.reviewedUnknownToolIds.includes(tool.id)) run.checkpoint.reviewedUnknownToolIds.push(tool.id);
|
||||
}
|
||||
}
|
||||
for (const job of this.store.core.check_job.values()) if (job.taskId === child.taskId && (job.requiresVerification || job.status === 'unknown')) {
|
||||
job.originalStatus = job.status; job.status = 'resolved_external'; job.requiresVerification = false; job.review = { evidenceSha256, reviewedAt: at() };
|
||||
this.store.core.task.get(child.taskId).verification = { status: 'needs_checks', checkJobId: job.id };
|
||||
}
|
||||
if (worker.requiresVerification || ['unknown', 'running', 'stopping'].includes(worker.status)) {
|
||||
worker.requiresVerification = false; worker.status = retain ? 'resolved_external' : 'paused'; worker.ownerPid = null;
|
||||
worker.summary = '用户已核对原未知输入与现场;原结果保持未知,重新检查需要新计划与审批。';
|
||||
childSession.providerMessages = repairToolPairs(childSession.providerMessages, callId => this.store.savedToolResult(childSession.id, callId)).messages;
|
||||
childSession.providerMessages.push({ role: 'user', content: worker.summary + ' 核对记录SHA256:' + evidenceSha256 });
|
||||
childSession.status = childSession.changes.some(change => ['pending', 'conflict'].includes(change.status)) ? 'needs_review' : 'paused';
|
||||
this.store.putRecord('worker', childSession, worker, 'worker.reviewed_unknown');
|
||||
}
|
||||
}
|
||||
group.requiresVerification = false;
|
||||
if (retain) {
|
||||
this.retain(session, group);
|
||||
group.status = 'resolved_external'; group.finishedAt = at();
|
||||
} else group.status = merged || group.merge?.changeSetId ? 'merge_prepared'
|
||||
: group.merge ? 'collected'
|
||||
: group.children.some(child => !this.store.core.worker.has(child.id)) ? 'partially_prepared'
|
||||
: group.children.every(child => this.store.core.worker.get(child.id).status === 'completed') ? 'ready_to_collect'
|
||||
: group.children.some(child => this.store.session(child.sessionId).status === 'needs_review') ? 'awaiting_review' : group.startedAt ? 'paused' : 'ready';
|
||||
group.summary = retain ? '用户已明确保留原工作区、任务与未知结果,结束协作;没有声明交付成功。' : '原未知结果已按用户核对保留,可以显式继续;累计预算和原时限保持。';
|
||||
session.status = 'paused'; this.save(session, group, 'reviewed_unknown'); return group;
|
||||
} catch (error) { group.status = 'unknown'; group.requiresVerification = true; this.save(session, group, 'review_failed'); throw error; }
|
||||
});
|
||||
return started.promise;
|
||||
}
|
||||
async stopParent(sessionId) {
|
||||
const running = [...this.active.values()].filter(value => value.sessionId === sessionId && value.controller);
|
||||
for (const value of running) {
|
||||
@@ -323,11 +663,36 @@ export class WorkerManager {
|
||||
await Promise.allSettled(value.group.children.map(child => this.runtime.stop(child.sessionId)));
|
||||
await value.promise;
|
||||
}
|
||||
for (const group of this.store.core.collaboration.values()) if (group.sessionId === sessionId && !COLLABORATION_CLOSED.has(group.status)
|
||||
&& group.children.some(child => this.childActive(child.sessionId))) {
|
||||
await Promise.allSettled(group.children.filter(child => this.store.sessions.has(child.sessionId)).map(child => this.runtime.stop(child.sessionId)));
|
||||
group.summary = '父任务已等待所属子执行与检查结束,已完成记录保留。'; this.save(this.store.session(sessionId), group, 'stopped');
|
||||
}
|
||||
}
|
||||
async stopAll() { await Promise.all([...new Set([...this.active.values()].map(value => value.sessionId).concat([...this.store.core.collaboration.values()]
|
||||
.filter(group => !COLLABORATION_CLOSED.has(group.status) && group.children.some(child => this.childActive(child.sessionId))).map(group => group.sessionId)))].map(id => this.stopParent(id))); }
|
||||
retain(session, group) {
|
||||
const sessions = [session, ...group.children.flatMap(child => this.store.sessions.has(child.sessionId) ? [this.store.session(child.sessionId)] : [])];
|
||||
for (const child of group.children) {
|
||||
const workspace = this.store.core.workspace.get(child.workspaceId);
|
||||
if (workspace?.status === 'prepared' && !workspace.creationAttempted) this.workspaces.discard(session, workspace.id, workspace.fingerprint);
|
||||
}
|
||||
const manager = new ChangeSetManager(this.store, { projectBusy: (projectId, exceptId) => this.runtime.projectHasActivity(projectId, exceptId, { excludeGroupId: group.id }) });
|
||||
for (const owned of sessions) {
|
||||
for (const set of this.store.core.change_set.values()) if (set.sessionId === owned.id && set.taskId === owned.taskId && !set.attemptedAt && ['prepared', 'stale'].includes(set.status)) {
|
||||
manager.snapshots(owned, set); manager.discard(owned, set.id, set.fingerprint);
|
||||
}
|
||||
for (const change of owned.changes) if ((owned !== session || change.sourceBinding?.collaborationId === group.id) && ['pending', 'conflict'].includes(change.status) && !change.changeSetId) change.status = 'rejected';
|
||||
const worker = workerForProject(this.store, owned.projectId);
|
||||
if (worker && worker.status !== 'completed') { worker.status = 'resolved_external'; worker.requiresVerification = false; worker.ownerPid = null; }
|
||||
owned.status = 'paused'; this.store.save(owned, { type: 'worker.retained' });
|
||||
}
|
||||
}
|
||||
async stopAll() { await Promise.all([...new Set([...this.active.values()].map(value => value.sessionId))].map(id => this.stopParent(id))); }
|
||||
discard(session, id, fingerprint) {
|
||||
const group = this.own(session, id, fingerprint);
|
||||
if (this.active.has(group.id) || group.requiresVerification || group.merge?.changeSetId) throw workerConflict('协作仍在执行、待核对或已有合并审查,请先处理原记录。');
|
||||
if (this.runtime.projectHasActivity(group.projectId) || group.requiresVerification || group.children.some(child => this.store.core.worker.get(child.id)?.requiresVerification)
|
||||
|| [...this.store.core.change_set.values()].some(set => (set.taskId === group.taskId || group.children.some(child => child.taskId === set.taskId)) && set.requiresVerification)) throw workerConflict('协作仍在执行或原批次待核对,请先处理原记录。');
|
||||
this.retain(session, group);
|
||||
group.status = group.children.some(child => this.store.core.workspace.has(child.workspaceId)) ? 'resolved_external' : 'discarded';
|
||||
group.finishedAt = at(); group.summary = '已明确保留原工作区、任务和成果,结束本次协作;未删除、重置或声明交付完成。';
|
||||
session.status = 'paused'; this.save(session, group, 'discarded'); return group;
|
||||
|
||||
@@ -14,6 +14,7 @@ const conflict = message => Object.assign(new Error(message), { status: 409 });
|
||||
const key = value => process.platform === 'win32' ? value.toLowerCase() : value;
|
||||
const gitBaseline = inspection => ({ available: inspection.git.available, head: inspection.git.head ?? null, branch: inspection.git.branch ?? null });
|
||||
const sealed = set => ({ id: set.id, sessionId: set.sessionId, projectId: set.projectId, taskId: set.taskId, runId: set.runId,
|
||||
...(set.collaborationBinding ? { collaborationBinding: set.collaborationBinding } : {}),
|
||||
root: set.root, baseline: set.baseline, files: set.files.map(file => ({ changeId: file.changeId, path: file.path,
|
||||
baseExists: file.baseExists, baseSha256: file.baseSha256, sha256: file.sha256, baseBytes: file.baseBytes, bytes: file.bytes,
|
||||
mode: file.mode, reason: file.reason, sourceRunId: file.sourceRunId, snapshotArtifactId: file.snapshotArtifactId })) });
|
||||
@@ -70,8 +71,8 @@ function base64(value) {
|
||||
}
|
||||
|
||||
export class ChangeSetManager {
|
||||
constructor(store, { projectBusy = () => false, faultInjector = null } = {}) {
|
||||
this.store = store; this.projectBusy = projectBusy; this.faultInjector = faultInjector;
|
||||
constructor(store, { projectBusy = () => false, faultInjector = null, beforeApply = null } = {}) {
|
||||
this.store = store; this.projectBusy = projectBusy; this.faultInjector = faultInjector; this.beforeApply = beforeApply;
|
||||
this.active = new Map(); this.projects = new Map();
|
||||
}
|
||||
projectActive(id) { return this.projects.has(id); }
|
||||
@@ -81,8 +82,8 @@ export class ChangeSetManager {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
}
|
||||
save(session, set, phase) { this.store.save(session, { type: 'change_set.' + phase, detail: { changeSetId: set.id } }); }
|
||||
requireIdle(session, { source = 'user', exceptId = null, includePrepared = false } = {}) {
|
||||
assertWorkersResolved(this.store, session.projectId);
|
||||
requireIdle(session, { source = 'user', exceptId = null, includePrepared = false, recoveryTaskId = null } = {}) {
|
||||
assertWorkersResolved(this.store, session.projectId, { recoveryTaskId });
|
||||
assertMcpResolved(this.store, session.projectId);
|
||||
if (this.projectActive(session.projectId) || this.projectBusy(session.projectId, source === 'agent' ? session.id : null)) {
|
||||
throw conflict('同一项目正在执行任务、检查、提交或文件批次,请等待结束后再操作。');
|
||||
@@ -128,7 +129,7 @@ export class ChangeSetManager {
|
||||
for (let index = 0; index < set.files.length; index++) set.files[index].state = observations[index].state;
|
||||
return observations;
|
||||
}
|
||||
async prepare(session, changeIds, { source = 'user' } = {}) {
|
||||
async prepare(session, changeIds, { source = 'user', collaborationBinding = null } = {}) {
|
||||
if (!Array.isArray(changeIds) || !changeIds.length || changeIds.length > 32 || new Set(changeIds).size !== changeIds.length) throw conflict('变更集需要1到32个不同的提案ID。');
|
||||
this.requireIdle(session, { source });
|
||||
const task = this.store.owned('task', session.taskId, session.id), run = this.store.owned('run', session.runId, session.id);
|
||||
@@ -152,6 +153,7 @@ export class ChangeSetManager {
|
||||
const inspection = await inspectProject(project);
|
||||
if (session.taskId !== task.id || session.runId !== run.id) throw conflict('准备变更集期间任务已切换。');
|
||||
const set = { id: randomUUID(), sessionId: session.id, projectId: project.id, taskId: task.id, runId: run.id, root: project.root,
|
||||
...(collaborationBinding ? { collaborationBinding: structuredClone(collaborationBinding) } : {}),
|
||||
schemaVersion: 1, status: 'prepared', requiresVerification: false, source, createdAt: at(), baseline: gitBaseline(inspection), files: [],
|
||||
policy: 'reviewed-byte-snapshots', summary: '整批范围已封存,项目文件尚未改变。' };
|
||||
let budget = 0;
|
||||
@@ -191,6 +193,7 @@ export class ChangeSetManager {
|
||||
const operation = this.claim(session, set);
|
||||
try {
|
||||
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
|
||||
await this.beforeApply?.(session, set);
|
||||
const snapshots = this.snapshots(session, set, { pending: true });
|
||||
const project = this.store.project(session.projectId), inspection = await inspectProject(project);
|
||||
if (set.taskId !== session.taskId || JSON.stringify(gitBaseline(inspection)) !== JSON.stringify(set.baseline)) throw conflict('任务或Git HEAD/分支已变化,未写入任何文件。');
|
||||
@@ -308,7 +311,7 @@ export class ChangeSetManager {
|
||||
async reconcile(session, id) {
|
||||
const set = this.own(session, id);
|
||||
if (!set.attemptedAt || !['unknown', 'partial', 'completed', 'rolled_back', 'interrupted'].includes(set.status)) throw conflict('这个批次尚未执行或已结束外部改动确认,无需恢复核对。');
|
||||
this.requireIdle(session, { exceptId: set.id });
|
||||
this.requireIdle(session, { exceptId: set.id, recoveryTaskId: set.taskId });
|
||||
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
|
||||
this.snapshots(session, set);
|
||||
return this.settle(session, set, this.inspect(session, set), set.direction === 'rollback' ? 'rollback' : 'apply');
|
||||
@@ -316,7 +319,7 @@ export class ChangeSetManager {
|
||||
async rollback(session, id, fingerprint) {
|
||||
const set = this.own(session, id, fingerprint, { current: true });
|
||||
if (typeof fingerprint !== 'string' || !set.attemptedAt || !['completed', 'partial', 'unknown'].includes(set.status)) throw conflict('只有已执行且已审查指纹的批次可以回退。');
|
||||
this.requireIdle(session, { exceptId: set.id });
|
||||
this.requireIdle(session, { exceptId: set.id, recoveryTaskId: set.taskId });
|
||||
const snapshots = this.snapshots(session, set), operation = this.claim(session, set);
|
||||
try {
|
||||
await verifyWorkspaceProject(this.store, this.store.project(session.projectId));
|
||||
@@ -334,7 +337,7 @@ export class ChangeSetManager {
|
||||
acknowledge(session, id, fingerprint, observationFingerprint) {
|
||||
const set = this.own(session, id, fingerprint, { current: true });
|
||||
if (typeof fingerprint !== 'string' || set.status !== 'partial' || !set.requiresVerification || typeof observationFingerprint !== 'string') throw conflict('请先核对这个批次的当前文件,再确认保留外部改动。');
|
||||
this.requireIdle(session, { exceptId: set.id }); this.snapshots(session, set);
|
||||
this.requireIdle(session, { exceptId: set.id, recoveryTaskId: set.taskId }); this.snapshots(session, set);
|
||||
const observations = this.inspect(session, set);
|
||||
if (set.observationFingerprint !== observationFingerprint || observations.some(value => !['before', 'external'].includes(value.state))) throw conflict('当前文件变化或仍有本批次建议内容未回退,请先重新核对/回退。');
|
||||
set.status = 'resolved_external'; set.requiresVerification = false; set.finishedAt = at();
|
||||
@@ -348,7 +351,7 @@ export class ChangeSetManager {
|
||||
discard(session, id, fingerprint) {
|
||||
const set = this.own(session, id, fingerprint, { current: true });
|
||||
if (typeof fingerprint !== 'string' || !['prepared', 'stale'].includes(set.status) || set.attemptedAt) throw conflict('只有尚未执行的封存批次可以撤销。');
|
||||
this.requireIdle(session, { exceptId: set.id });
|
||||
this.requireIdle(session, { exceptId: set.id, recoveryTaskId: set.taskId });
|
||||
set.status = 'cancelled'; set.finishedAt = at(); set.summary = '已撤销未执行批次,文件提案仍可重新审查。';
|
||||
for (const file of set.files) { const change = session.changes.find(value => value.id === file.changeId); if (change?.changeSetId === set.id) delete change.changeSetId; }
|
||||
this.save(session, set, 'cancelled'); return set;
|
||||
|
||||
Reference in New Issue
Block a user