feat: connect controlled engines to owned Tasks, approvals and budget ledger

This commit is contained in:
Agent Workbench Dev
2026-10-31 05:04:55 +08:00
parent a01c41811b
commit f0eecbda47
27 changed files with 1653 additions and 33 deletions
+3 -3
View File
@@ -1,14 +1,14 @@
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { engineDigest, engineError } from '../../shared/contracts/engines.mjs';
import { engineDigest, engineError, engineCanonicalDigest } from '../../shared/contracts/engines.mjs';
import { resolveDeepSeekInstallation, assertDeepSeekInstallation, engineFileIdentity } from './deepseek-installation.mjs';
import { CONTROLLED_BRIDGE_PLUGIN, CONTROLLED_DISABLED_PLUGINS, canonicalEngineValue } from './controlled-plugin.mjs';
import { CONTROLLED_BRIDGE_PLUGIN, CONTROLLED_DISABLED_PLUGINS } from './controlled-plugin.mjs';
export const CONTROLLED_ENGINE_SCOPE = 'deepseek-workbench-engine-v1';
export const CONTROLLED_CRITICAL_FILES = Object.freeze(['packages/core/tools/lib/index.js', 'packages/core/agent/lib/index.js',
'packages/core/agent-loop/lib/index.js', 'packages/core/system-prompt/lib/index.js', 'packages/llm/llm/lib/index.js',
'packages/llm/llm-retry/lib/index.js', 'vendor/include/lib/index.js']);
export const controlledDigest = value => engineDigest(canonicalEngineValue(value));
export const controlledDigest = engineCanonicalDigest;
export function resolveControlledInstallation(appDirectory = process.cwd(), root) {
const base = resolveDeepSeekInstallation(appDirectory, root);
+1
View File
@@ -59,6 +59,7 @@ export async function apply(ctx, config) {
tools: options.tools ?? [], toolHistory: options.toolHistory ?? null,
model: options.model, maxTokens: options.maxTokens ?? config.maxOutputTokens }, options.signal);
const message = result.message;
if (result.finishReason === 'length') throw new Error('Owned model output reached its bound; incomplete tools are not executed.');
if (!message || message.role !== 'assistant' || typeof message.content !== 'string'
|| message.tool_calls !== undefined && !Array.isArray(message.tool_calls)) throw new Error('Invalid owned model result.');
let index = 0;
+37 -2
View File
@@ -6,6 +6,7 @@ import { DeepSeekAdapter } from './deepseek-adapter.mjs';
import { engineFileIdentity } from './deepseek-installation.mjs';
import { engineError } from '../../shared/contracts/engines.mjs';
import { isId } from '../../shared/contracts/execution.mjs';
import { readOwnedProcessTree } from '../owned-process-tree.mjs';
import { CONTROLLED_BRIDGE_PLUGIN } from './controlled-plugin.mjs';
import { assertControlledInstallation, controlledDigest, controlledPatch, controlledToolCatalogue, CONTROLLED_ENGINE_SCOPE } from './controlled-installation.mjs';
@@ -18,6 +19,7 @@ export class ControlledDeepSeekSession extends DeepSeekAdapter {
constructor(installation, options = {}) {
super(installation, options); this.maximumFrameBytes = options.maximumFrameBytes ?? 4 * 1024 * 1024;
this.pending = new Set(); this.closed = false;
this.processTreeReader = options.processTreeReader ?? readOwnedProcessTree;
}
async open({ directory, cwd, owner, sdkSessionId, tools, config, signal, onModel, onTool, onState = () => {}, onNotification = () => {} }) {
for (const field of ['sessionId', 'projectId', 'taskId', 'runId', 'jobId']) if (!isId(owner?.[field])) throw engineError('受控引擎所属身份无效。', 400);
@@ -59,6 +61,15 @@ export class ControlledDeepSeekSession extends DeepSeekAdapter {
{ timeoutMs: this.timeoutMs, maximumFrameBytes: this.maximumFrameBytes, cancellationMethod: null });
this.report.process = { pid: this.client.pid, connectionId: this.client.id, startedAt: this.client.startedAt, ownership: 'one-process-per-engine-run' };
onState('starting', this.report.process);
this.publishOwnership = () => onState('ownership', this.report.process);
try { await this.captureTree(); } catch (error) { this.report.process.treeObservationError = error.message; throw error; }
this.treeTimer = setInterval(() => {
if (this.closed || this.treePending) return;
this.treePending = this.captureTree().catch(error => {
this.report.process.treeObservationError = error.message; this.client.fail(error);
this.publishOwnership();
}).finally(() => { this.treePending = null; });
}, 1200); this.treeTimer.unref();
this.client.on('server-request', request => {
this.report.rejectedServerRequests.push({ id: request.id, method: request.method, parametersSha256: controlledDigest(request.params ?? {}) });
this.client.fail(engineError('原SDK反向请求未在受控契约中开放。'));
@@ -94,8 +105,19 @@ export class ControlledDeepSeekSession extends DeepSeekAdapter {
if (engineFileIdentity(input.path).sha256 !== input.sha256) throw engineError('所属叠加插件或启动输入发生变化。');
}
}
async captureTree(publish = true) {
const info = this.report.process, tree = await this.processTreeReader(info);
if (tree.supported) {
if (tree.root) info.rootCreatedAt ??= tree.root.createdAt;
const known = new Map((info.descendants ?? []).map(value => [value.pid + ':' + value.createdAt, value]));
for (const value of tree.processes) if (value.pid !== info.pid) known.set(value.pid + ':' + value.createdAt, value);
info.descendants = [...known.values()].sort((left, right) => left.pid - right.pid);
info.treeRemainingPids = tree.processes.map(value => value.pid); info.treeObservedAt = at();
}
if (publish) this.publishOwnership?.(); return tree;
}
async handleBridge(request, response, { authentication, onModel, onTool, signal }) {
if (this.closed || signal?.aborted || request.method !== 'POST' || !['/model', '/tool'].includes(request.url)
if (this.closed || this.client?.exited === true || signal?.aborted || request.method !== 'POST' || !['/model', '/tool'].includes(request.url)
|| request.headers.origin !== undefined || request.headers['content-type'] !== 'application/json'
|| !equalSecret(request.headers.authorization, authentication)) throw engineError('桥请求不属于当前受控Run。', 403);
const chunks = []; let size = 0;
@@ -153,6 +175,7 @@ export class ControlledDeepSeekSession extends DeepSeekAdapter {
if (!assistant || !Array.isArray(assistant.data?.message?.content)) throw engineError('所属活动没有完整assistant原消息。');
const ends = activity.filter(event => event?.type === 'turn/end');
this.report.activity = { receiptIndex: receipt, endEvents: ends };
if (ends.length !== 1 || ends[0].data?.reason?.kind !== 'completed') throw engineError('所属活动没有唯一的正常结束事件,原输出不能视为完成。');
this.report.finalText = assistant.data.message.content.filter(block => block.type === 'text').map(block => block.text).join('');
return this.report.finalText;
}
@@ -160,21 +183,33 @@ export class ControlledDeepSeekSession extends DeepSeekAdapter {
if (this.closePromise) return this.closePromise;
const interrupted = force || this.parentSignal?.aborted === true;
this.closed = true;
clearInterval(this.treeTimer);
this.bridgeController?.abort(new Error('所属引擎正在关闭。'));
return this.closePromise = (async () => {
let failure = null;
if (this.client) {
await this.treePending;
if (!interrupted && !this.client.exited) {
try { await this.client.request('shutdown', {}, { timeoutMs: 1000 }); } catch (error) { failure = error; }
}
try { this.report.cleanup = await this.client.close({ force: interrupted || !!failure }); } catch (error) { failure ??= error; }
try {
const tree = await this.captureTree(false);
this.report.cleanup ??= { rootExited: false };
this.report.cleanup.treeConfirmed = tree.supported ? tree.processes.length === 0 && !!this.report.process.rootCreatedAt
&& !this.report.process.treeObservationError : this.report.cleanup.rootExited;
this.report.cleanup.remainingProcesses = tree.processes;
} catch (error) {
failure ??= error; this.report.cleanup ??= { rootExited: false }; this.report.cleanup.treeConfirmed = false;
}
this.report.trace = this.client.trace; this.report.stderr = this.client.stderr;
}
if (this.guard) { this.guard.closeAllConnections(); await new Promise(resolve => this.guard.close(resolve)); }
await Promise.allSettled([...this.pending]);
if (!interrupted && this.report.launchInputs) { try { this.verifyLaunch(); } catch (error) { failure ??= error; } }
if (!interrupted && this.report.cleanup?.exit?.code !== 0) failure ??= engineError('所属引擎未正常退出。');
this.report.finishedAt = at(); this.report.status = failure || !this.report.cleanup?.rootExited ? 'unknown' : interrupted ? 'cancelled' : 'closed';
this.report.finishedAt = at(); this.report.status = this.client && (!this.report.cleanup?.rootExited || !this.report.cleanup?.treeConfirmed)
? 'unknown' : failure ? 'failed' : interrupted ? 'cancelled' : 'closed';
if (failure) this.report.error = failure.message;
return this.report;
})();
+268
View File
@@ -0,0 +1,268 @@
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { randomUUID } from 'node:crypto';
import { ControlledDeepSeekSession } from './controlled-session.mjs';
import { resolveControlledInstallation, assertControlledInstallation, controlledDigest } from './controlled-installation.mjs';
import { engineRunInput, engineRunReport, verifyEngineInput, nativeMessagesToChat } from '../../shared/contracts/engine-execution.mjs';
import { engineError } from '../../shared/contracts/engines.mjs';
import { modelSnapshot, accountedTokens, checkBudget, BudgetExceeded } from '../../shared/contracts/execution.mjs';
import { completionBody, requestMetrics } from '../../shared/contracts/model-request.mjs';
import { workerForSession } from '../../shared/contracts/workers.mjs';
const at = () => new Date().toISOString();
const deferred = () => {
let resolve, reject; const promise = new Promise((yes, no) => { resolve = yes; reject = no; });
promise.catch(() => {}); return { promise, resolve, reject };
};
export class EngineRunner {
constructor(store, provider, { appDirectory = process.cwd(), sessionFactory = installation => new ControlledDeepSeekSession(installation), directoryRoot = null } = {}) {
this.store = store; this.provider = provider; this.appDirectory = appDirectory; this.sessionFactory = sessionFactory;
this.directoryRoot = directoryRoot; this.drivers = new Map();
}
availability(root) { return resolveControlledInstallation(this.appDirectory, root); }
selected(input) {
const installation = this.availability(input.root);
if (installation.controlledStatus !== 'available' || installation.controlledSha256 !== input.controlledSha256) {
throw engineError('受控引擎与刚才检测的程序/插件身份不一致,请重新检测后制定新计划。');
}
return installation;
}
assertCurrent(session, run) {
const driver = this.drivers.get(run.id);
if (!driver || driver.session !== session || driver.run !== run) throw engineError('原引擎执行不属于当前Run。');
driver.assertInput(); driver.native?.verifyLaunch?.();
}
complete(session, run, messages, progress, planned, tools) {
let driver = this.drivers.get(run.id);
if (!driver) {
if (workerForSession(this.store, session) !== run.worker || !run.worker?.engine) throw engineError('外部引擎只能在所属受限子工程中执行。');
assertControlledInstallation(run.worker.engine);
driver = new EngineRunDriver(this, session, run, planned, tools); this.drivers.set(run.id, driver);
}
return driver.complete(messages, progress).catch(error => { driver.fail(error); throw error; });
}
async finish(session, run) {
const driver = this.drivers.get(run.id); if (!driver) return;
try { await driver.close(); } finally { this.drivers.delete(run.id); }
if (driver.job.requiresVerification) {
session.status = 'paused'; run.stopReason = 'engine_verification';
this.store.message(session, 'system', '所属外部引擎的原进程或结果待核对。请在父工程核对原输入与现场后继续。');
} else if (driver.job.status !== 'completed' && session.status === 'completed') {
session.status = 'paused'; run.stopReason = 'engine_incomplete';
this.store.message(session, 'system', '原引擎没有完成全部协议和结果核验,不能交付。');
}
return driver.job;
}
}
class EngineRunDriver {
constructor(manager, session, run, seedPlan, tools) {
this.manager = manager; this.store = manager.store; this.provider = manager.provider; this.session = session; this.run = run;
this.queue = []; this.waiter = null; this.pendingCalls = new Map(); this.finished = false;
const base = path.resolve(manager.directoryRoot ?? path.join(os.tmpdir(), 'AgentWorkbench-Engine-Runs'));
fs.mkdirSync(base, { recursive: true }); if (fs.lstatSync(base).isSymbolicLink()) throw engineError('引擎运行目录不能是链接。');
const id = randomUUID();
this.job = { schemaVersion: 1, id, sessionId: session.id, projectId: session.projectId, taskId: run.taskId, runId: run.id,
workerId: run.worker.id, collaborationId: run.worker.collaborationId, installation: structuredClone(run.worker.engine),
sdkSessionId: randomUUID(), directory: path.join(base, id), model: modelSnapshot(run.config), budget: { ...run.record.budget, maxToolCalls: run.worker.budget.maxToolCalls },
tools: structuredClone(tools), sourceContextPlanId: seedPlan.id, sourceRequestSha256: seedPlan.requestSha256,
status: 'queued', startedAt: at(), finishedAt: null, requiresVerification: false, verified: false, native: null, requests: [], calls: [] };
const artifact = this.store.artifactStore.write(session, run.record, engineRunInput(this.job), { kind: 'engine-run-input' });
this.store.core.artifact.set(artifact.id, artifact); this.job.inputArtifactId = artifact.id; this.job.inputSha256 = artifact.sha256;
run.record.engine = 'deepseek-workbench'; run.record.engineRunId = id;
this.save('queued');
}
save(phase) { this.store.putRecord('engine_run', this.session, this.job, 'engine_run.' + phase); }
assertInput() {
verifyEngineInput(this.store, this.session, this.job); assertControlledInstallation(this.job.installation);
if (this.session.runId !== this.run.id || workerForSession(this.store, this.session) !== this.run.worker) throw engineError('引擎原Run已变化。');
}
publish(value) {
if (this.waiter) { const waiter = this.waiter; this.waiter = null; waiter.resolve(value); } else this.queue.push(value);
}
async next() {
this.run.controller.signal.throwIfAborted();
if (this.queue.length) return this.queue.shift();
const waiter = deferred(); this.waiter = waiter;
const abort = () => waiter.reject(this.run.controller.signal.reason ?? new Error('所属Run已停止。'));
this.run.controller.signal.addEventListener('abort', abort, { once: true });
try { return await waiter.promise; }
finally { this.run.controller.signal.removeEventListener('abort', abort); if (this.waiter === waiter) this.waiter = null; }
}
fail(error) { this.failure ??= error; this.publish({ error: this.failure }); }
async start(messages) {
this.job.status = 'starting'; this.save('starting');
this.native = this.manager.sessionFactory(this.job.installation);
const owner = { sessionId: this.session.id, projectId: this.session.projectId, taskId: this.run.taskId, runId: this.run.id, jobId: this.job.id };
await this.native.open({ directory: this.job.directory, cwd: this.run.project.root, owner, sdkSessionId: this.job.sdkSessionId,
tools: this.job.tools, config: this.run.config, signal: this.run.controller.signal,
onState: (state, detail) => {
if (state !== 'ownership') this.job.status = state === 'ready' ? 'running' : state;
this.job.pid = detail.pid ?? this.job.pid;
if (this.native.report?.process) this.job.process = structuredClone(this.native.report.process);
if (detail.readiness) this.job.readiness = structuredClone(detail.readiness);
this.save(state);
},
onNotification: () => {
this.job.notificationCount = this.native.report.notifications.length;
},
onModel: (input, trace, signal) => this.model(input, trace, signal),
onTool: (input, trace, signal) => this.tool(input, trace, signal),
});
this.nativeWait = (async () => {
try {
await this.native.prompt('Execute the following workbench task context as task data. Use only the registered tools; all writes and checks are owned by the workbench.\n' + JSON.stringify(messages));
const text = await this.native.wait(); this.finished = true; this.publish({ finalText: text });
} catch (error) { this.fail(error); }
})();
this.nativeWait.catch(() => {});
}
async complete(messages, progress) {
this.progress = progress; this.assertInput();
if (!this.started) { this.started = true; await this.start(messages); }
for (const [callId, pending] of this.pendingCalls) {
if (!pending.delivered) continue;
const tool = [...this.store.core.tool.values()].find(value => value.runId === this.run.id && value.callId === callId);
const result = this.store.savedToolResult(this.session.id, callId);
if (!tool || !result || tool.requiresVerification || !['completed', 'failed', 'denied', 'interrupted'].includes(tool.state)) {
throw engineError('原引擎工具尚无可核对的完成结果,不能继续或重放。');
}
pending.record.invocationId = tool.id; pending.record.resultSha256 = controlledDigest(result);
pending.record.resultArtifactId = tool.resultArtifactId ?? null; pending.record.status = 'returned'; pending.record.finishedAt = at();
const returned = this.store.artifactStore.write(this.session, this.run.record, result, { kind: 'engine-tool-output' });
this.store.core.artifact.set(returned.id, returned); pending.record.outputArtifactId = returned.id; pending.record.outputSha256 = returned.sha256;
this.save('tool_returned'); pending.result.resolve(result); this.pendingCalls.delete(callId);
}
const next = await this.next(); if (next.error) throw next.error;
this.progress?.({ phase: 'completed', attempt: 0, maxAttempts: 0, engineRunId: this.job.id, origin: 'external-engine' });
if (next.call) {
next.pending.delivered = true;
return { message: { role: 'assistant', content: '', tool_calls: [next.call] }, usage: null, engineAccounted: true };
}
return { message: { role: 'assistant', content: next.finalText }, usage: null, engineAccounted: true };
}
async tool(input, trace, signal) {
try {
this.assertInput(); signal.throwIfAborted(); checkBudget(this.run.record, this.store.core.task.get(this.run.taskId));
const allCalls = [...this.store.core.engine_run.values()].filter(job => job.workerId === this.run.worker.id).reduce((total, job) => total + job.calls.length, 0);
if (allCalls >= this.run.worker.budget.maxToolCalls) throw new BudgetExceeded('worker_tool_limit', '原引擎工具次数已到所属子任务累计上限。');
const callId = this.job.id + ':' + input.callId;
if (this.pendingCalls.has(callId) || this.job.calls.some(call => call.callId === callId)) throw engineError('原引擎调用已经登记,不会再次执行。');
const record = { sourceCallId: input.callId, callId, name: input.name, argumentsSha256: controlledDigest(input.arguments),
bridgeTraceId: trace.id, status: 'received', receivedAt: at() };
const artifact = this.store.artifactStore.write(this.session, this.run.record, { owner: trace.id, ...input }, { kind: 'engine-tool-input' });
this.store.core.artifact.set(artifact.id, artifact); record.inputArtifactId = artifact.id; record.inputSha256 = artifact.sha256;
const pending = { record, result: deferred(), delivered: false }; this.job.calls.push(record); this.pendingCalls.set(callId, pending);
this.job.status = 'waiting_tool'; this.save('tool_received');
const abort = () => pending.result.reject(signal.reason ?? new Error('所属引擎工具已停止。'));
signal.addEventListener('abort', abort, { once: true });
this.publish({ call: { id: callId, type: 'function', function: { name: input.name, arguments: JSON.stringify(input.arguments) } }, pending });
try { return await pending.result.promise; } finally { signal.removeEventListener('abort', abort); }
} catch (error) { this.fail(error); throw error; }
}
async model(input, trace, signal) {
let request, dispatched = false;
if (this.modelActive) { const error = engineError('所属引擎已有模型请求,不能并发扩展封存预算。'); this.fail(error); throw error; }
this.modelActive = true;
try {
this.assertInput(); signal.throwIfAborted();
const task = this.store.core.task.get(this.run.taskId); checkBudget(this.run.record, task);
const messages = nativeMessagesToChat(input), tools = input.tools.map(tool => ({ type: 'function', function: tool }));
const config = { ...this.run.config, maxOutputTokens: input.maxTokens }, body = completionBody(config, messages, tools), metrics = requestMetrics(body);
const maximumAttempts = config.mode === 'demo' ? 1 : config.modelRouting?.maxAttempts ?? 1;
const credit = Math.min(this.run.record.budget.maxTokens - accountedTokens(this.run.record.usage), task.budget.maxTokens - accountedTokens(task.usage));
const estimate = metrics.estimatedInputTokens + 128 + config.maxOutputTokens;
const attempts = Math.min(maximumAttempts, Math.floor(credit / estimate), this.run.worker.budget.maxSteps - task.usage.requests);
if (metrics.estimatedInputTokens + 128 + config.maxOutputTokens > config.contextLimit || attempts < 1) {
throw new BudgetExceeded('engine_input_budget', '原引擎完整模型输入或累计尝试超出封存预算,未发送模型请求。');
}
const selected = config.modelRouting ? { ...config, modelRouting: { ...config.modelRouting, maxAttempts: attempts } } : config;
request = { id: randomUUID(), bridgeTraceId: trace.id, sourceSha256: controlledDigest(input), requestSha256: metrics.sha256,
requestBytes: metrics.bytes, estimatedInputTokens: metrics.estimatedInputTokens, reservedPerAttempt: estimate,
maxAttempts: attempts, accountedAttempts: 0, observedAttempts: 0, status: 'prepared', startedAt: at() };
const owner = { sessionId: this.session.id, projectId: this.session.projectId, taskId: this.run.taskId, runId: this.run.id };
const plan = { ...request, ...owner };
const artifact = this.store.artifactStore.write(this.session, this.run.record,
JSON.parse(this.provider.redact(JSON.stringify({ schemaVersion: 1, owner, native: input, body, plan }))), { kind: 'engine-model-input' });
this.store.core.artifact.set(artifact.id, artifact); request.inputArtifactId = artifact.id; request.inputSha256 = artifact.sha256;
this.job.requests.push(request);
const modelEvent = this.store.event(this.session, { kind: 'model', source: 'engine_model_bridge', engineRunId: this.job.id, engineModelRequestId: request.id,
taskId: this.job.taskId, runId: this.job.runId, label: '引擎请求模型服务', phase: 'running', startedAt: at(),
output: '原引擎完整输入已封存,模型服务由工作台代理。' });
request.eventId = modelEvent.id; request.status = 'dispatched'; request.dispatchedAt = at(); this.job.status = 'running'; this.save('model_intent');
dispatched = true;
const response = await this.provider.complete(selected, messages, this.run.project, signal, progress => {
if (progress.phase === 'running') {
const attempt = progress.attempt ?? 1;
if (!Number.isSafeInteger(attempt) || attempt < 1 || attempt > request.maxAttempts) throw engineError('模型尝试超出原引擎封存预算。');
request.observedAttempts = Math.max(request.observedAttempts, attempt);
}
if (progress.connectionId) { request.connectionId = progress.connectionId; request.actualModel = progress.model; }
modelEvent.phase = progress.phase; modelEvent.attempts = progress.attempt; modelEvent.connectionId = progress.connectionId ?? null;
modelEvent.actualModel = progress.model ?? config.model;
modelEvent.label = progress.switching ? '引擎切换模型池连接' : progress.phase === 'failed' ? '引擎模型请求失败' : progress.phase === 'completed' ? '引擎模型响应已收到' : '引擎请求模型服务';
if (['completed', 'failed'].includes(progress.phase)) modelEvent.finishedAt = at();
this.save('model_progress');
}, plan, tools, { attempts: 1 });
const count = Math.max(1, request.observedAttempts);
request.actualAttemptsFinal = count; request.responseReceivedAt = at(); request.status = 'responded';
while (request.accountedAttempts < count - 1) {
request.accountedAttempts++; this.store.recordResponse(this.session, this.run.record, null, estimate);
}
request.accountedAttempts++; request.usage = response.usage ?? null;
this.store.recordResponse(this.session, this.run.record, response.usage, estimate);
this.save('model_usage');
const output = JSON.parse(this.provider.redact(JSON.stringify(response))), saved = this.store.artifactStore.write(this.session, this.run.record, output, { kind: 'engine-model-output' });
this.store.core.artifact.set(saved.id, saved); request.responseArtifactId = saved.id; request.responseSha256 = saved.sha256;
request.status = 'completed'; request.finishedAt = at(); this.save('model_completed');
modelEvent.phase = 'completed'; modelEvent.finishedAt = at(); modelEvent.output = '原模型响应与尝试用量已保存。'; this.save('model_event_completed');
checkBudget(this.run.record, task); signal.throwIfAborted(); return output;
} catch (error) {
this.fail(error);
if (request) {
request.actualAttemptsFinal ??= request.observedAttempts || request.maxAttempts;
if (dispatched) while (request.accountedAttempts < request.actualAttemptsFinal) {
request.accountedAttempts++; this.store.recordResponse(this.session, this.run.record, null, request.reservedPerAttempt);
}
request.status = signal.aborted ? 'interrupted' : 'failed'; request.error = this.provider.redact(error.message); request.finishedAt = at(); this.save('model_failed');
const event = this.session.events.find(value => value.id === request.eventId);
if (event) { event.phase = request.status; event.finishedAt = at(); event.output = request.error; this.save('model_event_failed'); }
}
throw error;
} finally {
this.modelActive = false;
}
}
async close() {
if (this.closePromise) return this.closePromise;
return this.closePromise = this.finishClose();
}
async finishClose() {
this.job.status = 'closing'; this.save('closing');
for (const pending of this.pendingCalls.values()) pending.result.reject(new Error('所属引擎已结束,未返回结果保留原状态。'));
let native;
try { native = this.native ? await this.native.close({ force: !this.finished || !!this.failure || this.run.controller.signal.aborted }) : null; }
catch (error) { this.failure ??= error; }
if (this.nativeWait) await this.nativeWait;
this.job.native = native ? JSON.parse(this.provider.redact(JSON.stringify(native))) : null;
const launched = !!this.native?.client || !!native?.process?.pid || !!this.job.pid;
const unknown = launched && (!native?.cleanup?.rootExited || native?.cleanup?.treeConfirmed === false || native?.status === 'unknown');
this.job.requiresVerification = unknown;
this.job.status = unknown ? 'unknown' : this.finished && !this.failure && native?.status === 'closed' ? 'completed'
: this.run.controller.signal.aborted ? 'cancelled' : 'failed';
this.job.verified = this.job.status === 'completed'; this.job.finishedAt = at();
this.job.error = this.failure ? this.provider.redact(this.failure.message) : native?.error ?? null;
this.job.originalOutcome = { status: this.job.status, requiresVerification: this.job.requiresVerification, verified: this.job.verified, error: this.job.error };
try {
verifyEngineInput(this.store, this.session, this.job);
const artifact = this.store.artifactStore.write(this.session, this.run.record, engineRunReport(this.job), { kind: 'engine-run-report' });
this.store.core.artifact.set(artifact.id, artifact); this.job.reportArtifactId = artifact.id; this.job.reportSha256 = artifact.sha256;
this.save('finished');
} catch (error) {
this.job.status = 'unknown'; this.job.requiresVerification = true; this.job.verified = false;
this.job.error = '原引擎结果尚未完整封存:' + this.provider.redact(error.message); this.save('seal_failed');
}
}
}
+38 -6
View File
@@ -15,6 +15,8 @@ import { parseTestEvidence } from '../check-runner.mjs';
import { verifyPlanEvidence } from '../context/request-plan.mjs';
import { repairToolPairs } from '../../server/store.mjs';
import { readOwnedProcessTree } from '../owned-process-tree.mjs';
import { assertControlledInstallation } from '../engines/controlled-installation.mjs';
import { verifyEngineInput, originalEngineRun, assertEngineWorkerResult } from '../../shared/contracts/engine-execution.mjs';
const at = () => new Date().toISOString();
const sourceIdentity = source => Object.fromEntries(['root', 'head', 'reference', 'branch', 'gitDir', 'commonDir', 'indexSha256',
@@ -81,6 +83,7 @@ export class WorkerManager {
|| [...this.store.core.collaboration.values()].some(group => group.projectId === session.projectId && !COLLABORATION_CLOSED.has(group.status))) throw workerConflict('项目已有未收尾的父协作,请先完成或明确保留原成果。');
const initial = structuredClone(this.store.config), config = this.runtime.provider.prepareConfig ? this.runtime.provider.prepareConfig(initial, this.store) : initial;
const definition = collaborationInput(input, config), project = this.store.project(session.projectId);
const engine = definition.engine ? this.runtime.engineRunner().selected(definition.engine) : null;
const temporaryId = randomUUID(), controller = new AbortController(), active = { sessionId: session.id, projectId: project.id, group: { children: [], status: 'planning' }, controller };
active.promise = new Promise(resolve => { active.settle = resolve; });
this.active.set(temporaryId, active);
@@ -95,7 +98,7 @@ export class WorkerManager {
const group = { schemaVersion: 1, id: randomUUID(), projectId: project.id, sessionId: session.id, taskId: session.taskId, runId: run.id,
goal: definition.goal, budget: definition.budget, model: modelSnapshot(config), config: configView(config),
source: { identity: sourceIdentity(source), changedPaths: source.changedPaths, dirty: source.dirty, scope: source.scope },
children: definition.children.map(child => ({ ...child, id: randomUUID(), workspaceId: randomUUID(), projectId: randomUUID(), sessionId: randomUUID(), taskId: randomUUID() })),
children: definition.children.map(child => ({ ...child, ...(engine ? { engine: structuredClone(engine) } : {}), id: randomUUID(), workspaceId: randomUUID(), projectId: randomUUID(), sessionId: randomUUID(), taskId: randomUUID() })),
status: 'planned', requiresVerification: false, createdAt: at(), updatedAt: at(), startedAt: null, deadline: null, finishedAt: null,
summary: '父子目标、范围、模型和预算已封存,尚未创建工作区或请求模型。', collection: null, merge: null };
group.fingerprint = workerHash(collaborationSeal(group));
@@ -141,7 +144,7 @@ export class WorkerManager {
await this.initializeChild(group, child); this.save(session, group, 'workspace_registered');
await this.faultInjector?.('workspace_registered', group, child); signal.throwIfAborted();
}
await this.sourceFor(group); group.status = 'ready'; group.summary = '两个独立工作区已核对,来源HEAD、索引和已有修改保持。可以开始受限子任务。';
await this.sourceFor(group); group.status = 'ready'; group.summary = group.children.length + ' 个独立工作区已核对,来源HEAD、索引和已有修改保持。可以开始受限子任务。';
} catch (error) {
const uncertain = group.children.some(child => { const workspace = this.store.core.workspace.get(child.workspaceId); return workspace && (workspace.status !== 'ready'
|| !this.store.core.worker.has(child.id) && this.store.sessions.get(child.sessionId)?.taskId); });
@@ -195,11 +198,16 @@ export class WorkerManager {
this.store.putRecord('worker', session, worker, 'worker.ready'); return worker;
}
configFor(group, worker) {
if (worker.engine) {
assertControlledInstallation(worker.engine);
for (const job of this.store.core.engine_run.values()) if (job.workerId === worker.id) verifyEngineInput(this.store, this.store.session(worker.sessionId), job);
}
const initial = structuredClone(this.store.config), current = this.runtime.provider.prepareConfig ? this.runtime.provider.prepareConfig(initial, this.store) : initial;
if (JSON.stringify(modelSnapshot(current)) !== JSON.stringify(group.model)) throw workerConflict('模型连接或池快照已变化;不能在原协作中悄悄替换模型。');
const usage = workerUsage(this.store, worker), remaining = group.deadline - Date.now();
if (remaining <= 0) throw new BudgetExceeded('time_limit', '父协作总时限已用完,原任务与成果保留。');
if (usage.accountedTokens >= worker.budget.maxTokens || usage.steps >= worker.budget.maxSteps || usage.toolCalls >= worker.budget.maxToolCalls) throw new BudgetExceeded('worker_budget', '子任务累计预算已用完,续接不能重置预算。');
if (usage.accountedTokens >= worker.budget.maxTokens || usage.steps >= worker.budget.maxSteps || usage.toolCalls >= worker.budget.maxToolCalls
|| worker.engine && usage.requests >= worker.budget.maxSteps) throw new BudgetExceeded('worker_budget', '子任务累计预算已用完,续接不能重置预算。');
return { ...group.config, maxSteps: worker.budget.maxSteps - usage.steps, maxTaskTokens: worker.budget.maxTokens,
maxRunTokens: worker.budget.maxTokens - usage.accountedTokens, maxRunDurationMs: remaining };
}
@@ -227,7 +235,7 @@ export class WorkerManager {
group.requiresVerification = outcomes.some(outcome => outcome.status === 'rejected') || workers.some(worker => worker.requiresVerification || ['running', 'stopping'].includes(worker.status));
group.status = group.requiresVerification ? 'unknown' : workers.every(worker => worker.status === 'completed') ? 'ready_to_collect'
: workers.some(worker => this.store.session(worker.sessionId).status === 'needs_review') ? 'awaiting_review' : 'paused';
group.summary = group.status === 'ready_to_collect' ? '两个子任务均有实际改动和当前通过的检查,等待父任务核对原成果。'
group.summary = group.status === 'ready_to_collect' ? group.children.length + ' 个子任务均有实际改动和当前通过的检查,等待父任务核对原成果。'
: group.status === 'awaiting_review' ? '子任务提案已保存,请分别审查应用后继续;完成状态以实际检查为准。' : '子任务已停止或需要处理,已完成结果保留。';
} catch (error) { group.status = 'paused'; group.summary = this.runtime.provider.redact(error.message ?? String(error)); }
session.status = group.status === 'awaiting_review' ? 'needs_review' : 'paused'; this.save(session, group, group.status); return group;
@@ -263,7 +271,7 @@ export class WorkerManager {
return worker;
}
uncertain(worker) {
return ['tool', 'check_job', 'change_set', 'delivery_job'].flatMap(kind => [...this.store.core[kind].values()]
return ['tool', 'check_job', 'change_set', 'delivery_job', 'engine_run'].flatMap(kind => [...this.store.core[kind].values()]
.filter(value => value.taskId === worker.taskId && (value.requiresVerification || value.status === 'unknown' || value.state === 'unknown'))
.map(value => ({ kind, id: value.id })));
}
@@ -283,6 +291,7 @@ export class WorkerManager {
const baseline = this.baseline(session, worker), project = this.store.project(worker.projectId);
await verifyWorkspaceProject(this.store, project);
if (this.uncertain(worker).length) throw workerConflict('子任务仍有结果未知的工具、检查或文件批次');
assertEngineWorkerResult(this.store, session, worker);
if (session.changes.some(change => ['pending', 'conflict'].includes(change.status))) throw workerConflict('子任务仍有未审查改动');
const before = new Map(baseline.files.map(file => [workerPathKey(file.path), file]));
const { inspection, files: current } = await this.inventory(project), after = new Map(current.map(file => [workerPathKey(file.path), file]));
@@ -546,6 +555,24 @@ export class WorkerManager {
logSha256: this.store.core.artifact.get(result.logArtifactId)?.sha256 ?? null })) });
}
const session = this.store.sessions.get(child.sessionId);
const engines = [];
for (const job of this.store.core.engine_run.values()) if (job.taskId === child.taskId) {
verifyEngineInput(this.store, session, job);
if (job.reportArtifactId) originalEngineRun(this.store, session, job);
const processInfo = job.process ?? job.native?.process;
let tree = null;
if (job.requiresVerification || job.status === 'unknown') {
if (processInfo) {
tree = await readOwnedProcessTree(processInfo);
if (!tree.supported || tree.processes.length) throw workerConflict('原引擎进程树仍存活或无法核对,请先人工核对原创建身份;不会按历史PID停止。');
}
}
engines.push({ id: job.id, status: job.status, requiresVerification: !!job.requiresVerification,
inputArtifactId: job.inputArtifactId, inputSha256: job.inputSha256, reportArtifactId: job.reportArtifactId ?? null,
reportSha256: job.reportSha256 ?? null, process: processInfo ?? null, remainingProcesses: tree?.processes ?? null,
missingProcessIdentity: !processInfo && job.status === 'unknown', requests: job.requests, calls: job.calls,
originalOutcome: job.originalOutcome ?? null, error: job.error ?? null });
}
const tools = [...this.store.core.tool.values()].filter(value => value.taskId === child.taskId && value.requiresVerification).map(tool => {
const call = session?.providerMessages.flatMap(message => message.tool_calls ?? []).find(call => call.id === tool.callId);
const input = call?.function.arguments;
@@ -562,7 +589,7 @@ export class WorkerManager {
children.push({ id: child.id, workspaceId: child.workspaceId, workspaceStatus: workspace?.status ?? 'absent',
workerStatus: worker?.status ?? 'absent', taskId: child.taskId, sessionId: child.sessionId, projectId: child.projectId,
inventoryFingerprint: inventory ? workerHash(inventory.files) : null, sourceFingerprint: inventory?.inspection.sourceFingerprint ?? null, inventoryError,
tools, checks, changes, changeSets: [...this.store.core.change_set.values()].filter(set => set.taskId === child.taskId).map(set => ({ id: set.id, status: set.status, requiresVerification: !!set.requiresVerification, observationFingerprint: set.observationFingerprint ?? null })) });
tools, checks, changes, ...(engines.length ? { engines } : {}), changeSets: [...this.store.core.change_set.values()].filter(set => set.taskId === child.taskId).map(set => ({ id: set.id, status: set.status, requiresVerification: !!set.requiresVerification, observationFingerprint: set.observationFingerprint ?? null })) });
}
const parent = this.store.session(group.sessionId);
return { schemaVersion: 1, kind: 'worker-recovery-observation', collaborationId: group.id, groupFingerprint: group.fingerprint,
@@ -612,10 +639,15 @@ export class WorkerManager {
if (!retain && !merged && JSON.stringify(current.source) !== JSON.stringify(group.source.identity)) throw workerConflict('父来源已经变化,只能明确保留现场后重新规划。');
if (current.children.some(child => !['ready', 'absent', 'prepared', 'discarded', 'resolved_external'].includes(child.workspaceStatus))) throw workerConflict('原工作区仍需专门核对,不能开始子执行或结束恢复。');
if (!retain && current.children.some(child => ['discarded', 'resolved_external'].includes(child.workspaceStatus))) throw workerConflict('原工作区没有核对为可执行,只能明确保留现场结束。');
if (!retain && current.children.some(child => child.engines?.some(job => job.missingProcessIdentity))) throw workerConflict('引擎原进程创建身份没有封存,只能人工处理后明确保留现场结束。');
if (!retain) await this.recoverMerge(session, group);
for (const child of group.children) {
const worker = this.store.core.worker.get(child.id); if (!worker) continue;
const childSession = this.store.session(child.sessionId);
for (const job of this.store.core.engine_run.values()) if (job.taskId === child.taskId && job.requiresVerification) {
job.review = { source: 'user', evidenceSha256, reviewedAt: at(), result: 'unknown-preserved' };
job.status = 'resolved_external'; job.requiresVerification = false;
}
for (const tool of this.store.core.tool.values()) if (tool.taskId === child.taskId && tool.requiresVerification) {
tool.originalState = tool.state; tool.state = 'resolved_external'; tool.requiresVerification = false;
tool.review = { source: 'user', evidenceSha256, reviewedAt: at(), result: 'unknown-preserved' };