feat(vision): bind task screenshots, route vision and resume budget pauses

This commit is contained in:
Agent Workbench Dev
2026-11-01 00:48:38 +08:00
parent 0d1ab743f1
commit 96a7ed19c7
79 changed files with 1721 additions and 36 deletions
+22 -3
View File
@@ -48,12 +48,12 @@ function json(response, value, status = 200) {
'X-Content-Type-Options': 'nosniff' });
response.end(JSON.stringify(value));
}
async function body(request) {
async function body(request, maximum=2*1024*1024) {
if (!String(request.headers['content-type'] ?? '').startsWith('application/json')) throw Object.assign(new Error('请求需要 JSON。'), { status: 415 });
const chunks = []; let length = 0;
for await (const chunk of request) {
length += chunk.length;
if (length > 2 * 1024 * 1024) throw Object.assign(new Error('请求超过 2 MB。'), { status: 413 });
if (length > maximum) throw Object.assign(new Error('请求超过允许大小。'), { status: 413 });
chunks.push(chunk);
}
let input;
@@ -269,6 +269,23 @@ return async (request, response) => {
if (sessionMatch) {
const id = sessionMatch[1], action = sessionMatch[2] ?? '';
const session = store.session(id);
if (action === 'input-attachments') {
if (method === 'GET') return json(response,runtime.inputAttachments().list(session));
if (workerForProject(store,session.projectId)) throw workerConflict('截图附件请在父任务中添加。');
if (method === 'POST') return json(response,runtime.inputAttachments().upload(session,await body(request,3*1024*1024)),201);
}
const inputImage = action.match(/^input-attachments\/([a-f0-9-]+)(?:\/(image|original))?$/);
if (inputImage) {
const manager=runtime.inputAttachments();
if(method==='DELETE'&&!inputImage[2]) { await body(request);return json(response,manager.remove(session,inputImage[1])); }
if(method==='GET'&&inputImage[2]) {
const {row,artifact,bytes}=manager.read(session,inputImage[1],{original:inputImage[2]==='original'});
const download=inputImage[2]==='original'||url.searchParams.get('download')==='1';
response.writeHead(200,{'Content-Type':artifact.mediaType,'Cache-Control':'no-store','X-Content-Type-Options':'nosniff','X-Content-SHA256':artifact.sha256,
...(download?{'Content-Disposition':'attachment; filename="workbench-input-'+row.id+(artifact.mediaType==='image/png'?'.png':'.jpg')+'"'}:{})});
response.end(bytes);return;
}
}
if (method === 'POST' && workerForProject(store, session.projectId)
&& /^(?:mcp|engines|research|workspaces|delivery|browser|task-plans|task-proofs|task-checkpoints|collaborations)(?:\/|-|$)/.test(action)) throw workerConflict('受限子任务只开放原目标的读取、范围内改动和已审批检查;此操作请返回父项目。');
if (method === 'GET' && action === 'collaborations') return json(response, runtime.workerManager().view(session));
@@ -574,7 +591,7 @@ return async (request, response) => {
}
if (method === 'POST' && action === 'messages') {
const input = await body(request);
runtime.start(id, input.content);
runtime.start(id, input.content,false,{attachmentIds:input.attachmentIds??[]});
return json(response, store.publicSession(session), 202);
}
if (method === 'POST' && action === 'resume') {
@@ -582,6 +599,8 @@ return async (request, response) => {
runtime.resume(id, input.content);
return json(response, store.publicSession(session), 202);
}
if(method==='POST'&&action==='resume-preview')return json(response,runtime.budgetResumes().prepare(session,await body(request)),201);
if(method==='POST'&&action==='resume-budget'){runtime.resumeBudget(id,await body(request));return json(response,store.publicSession(session),202);}
if (method === 'POST' && action === 'continue-reviewed') {
await body(request); runtime.continueReviewed(id);
return json(response, store.publicSession(session), 202);
+10 -1
View File
@@ -9,6 +9,7 @@ import { CompanionTLS } from '../core/companion/tls-authority.mjs';
import { fields, companionError, companionHash, approvalBinding, operationInput } from '../shared/contracts/companion.mjs';
import { isId } from '../shared/contracts/execution.mjs';
import { mobileSession, sessionBrief, reportKinds } from '../core/companion/projection.mjs';
import { InputAttachments } from '../core/context/input-attachments.mjs';
import { hostAuthority } from '../core/desktop/host-authority.mjs';
export function companionGateway(context) {
@@ -179,7 +180,7 @@ export class CompanionGateway {
async handle(request, response) {
response.setHeader('Cache-Control', 'no-store'); response.setHeader('X-Content-Type-Options', 'nosniff');
response.setHeader('Referrer-Policy', 'no-referrer'); response.setHeader('X-Frame-Options', 'DENY');
response.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'");
response.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data: blob:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'");
try {
this.registry.assertHost();
if (!this.server?.listening || !this.origin || request.headers.host !== new URL(this.origin).host) throw companionError('手机网关地址已变化,请重新连接。', 421);
@@ -220,6 +221,14 @@ export class CompanionGateway {
if (limit < 1 || limit > 200) throw companionError('手机事件每页需要1至200条。');
return reply(response, this.events(device, after, limit));
}
const attachment=route.match(/^\/mobile-api\/sessions\/([a-f0-9-]+)\/input-attachments\/([a-f0-9-]+)\/(image|original)$/);
if(attachment&&method==='GET'){
const session=this.session(device,attachment[1]),manager=new InputAttachments(this.store),row=manager.get(session,attachment[2]);
if(row.taskId!==session.taskId||row.status!=='attached'||!(this.store.core.task.get(session.taskId)?.attachmentIds??[]).includes(row.id))throw companionError('当前任务没有这张截图。',404);
const {artifact,bytes}=manager.read(session,row.id,{original:attachment[3]==='original'});
response.writeHead(200,{'Content-Type':artifact.mediaType,'Content-Length':bytes.length,'X-Content-SHA256':artifact.sha256,
...(attachment[3]==='original'?{'Content-Disposition':'attachment; filename="workbench-input-'+row.id+(row.mediaType==='image/png'?'.png':'.jpg')+'"'}:{})});response.end(bytes);return;
}
const match = route.match(/^\/mobile-api\/sessions\/([a-f0-9-]+)(?:\/(approve|stop|changes\/([a-f0-9-]+)|reports\/([a-f0-9-]+)))?$/);
if (match) {
const session = this.session(device, match[1]);
+6
View File
@@ -34,6 +34,8 @@ export function installUpgrade(context, { appDirectory, updateServiceInfo = true
const oldStoreProperties = { ...store };
const oldRuntimeProperties = { appDirectory: runtime.appDirectory, workers: runtime.workers, controlledEngines: runtime.controlledEngines, checks: runtime.checks, deliveries: runtime.deliveries, codeIndex: runtime.codeIndex, changeSets: runtime.changeSets, contextPlans: runtime.contextPlans, workspaces: runtime.workspaces, research: runtime.research, engines: runtime.engines, browser: runtime.browser, taskPlans: runtime.taskPlans, projectMemories: runtime.projectMemories, taskCheckpoints: runtime.taskCheckpoints, skills: runtime.skills, mcp: runtime.mcp };
const oldUpdates = store.listeners('update');
const oldInputs = runtime.inputs;
const oldResumes = runtime.resumes;
const closeStorage = () => store.close();
if (oldUpdates.length > 1) throw new Error('Unexpected Store observers; preserve their ownership before upgrading.');
const handler = createRequestHandler({ appDirectory: root, server, store, provider, runtime, token, clients, started,
@@ -49,6 +51,8 @@ export function installUpgrade(context, { appDirectory, updateServiceInfo = true
}
runtime.appDirectory = root; runtime.workers = null; runtime.controlledEngines = null; runtime.checks = null; runtime.deliveries = null; runtime.codeIndex = null; runtime.changeSets = null; runtime.contextPlans = null; runtime.workspaces = null; runtime.research = null; runtime.engines = null; runtime.browser = null; runtime.taskPlans = null; runtime.projectMemories = null; runtime.taskCheckpoints = null; runtime.skills = null; runtime.mcp = null;
delete server.workbenchCompanion;
runtime.inputs = null;
runtime.resumes = null;
store.ensureStorage();
// A SQLite host can retain an artifact object from an earlier bundle.
// Reopen the same blobs through the new API without mutating that object;
@@ -80,6 +84,8 @@ export function installUpgrade(context, { appDirectory, updateServiceInfo = true
for (const key of Object.keys(store)) if (!Object.hasOwn(oldStoreProperties, key)) delete store[key];
Object.assign(store, oldStoreProperties);
Object.assign(runtime, oldRuntimeProperties);
if (oldInputs === undefined) delete runtime.inputs; else runtime.inputs = oldInputs;
if (oldResumes === undefined) delete runtime.resumes; else runtime.resumes = oldResumes;
Object.setPrototypeOf(store, oldPrototypes[0]); Object.setPrototypeOf(provider, oldPrototypes[1]); Object.setPrototypeOf(runtime, oldPrototypes[2]);
if (oldModelPrototypes.hub) Object.setPrototypeOf(provider.hub, oldModelPrototypes.hub);
if (oldModelPrototypes.importer) Object.setPrototypeOf(provider.hub.importer, oldModelPrototypes.importer);
+1
View File
@@ -210,6 +210,7 @@ export class Provider {
async complete(config, messages, project, signal, onProgress, requestPlan = null, tools = toolDefinitions, requestPolicy = {}) {
const body = completionBody(config, messages, tools);
assertPlannedRequest(body, requestPlan);
if (config.inputImages?.length && (!config.modelRouting || !this.hub || config.mode==='demo')) throw new Error('图片请求必须通过模型中心的视觉能力选路,尚未发送。');
if (config.mode === 'demo') return this.demo(messages, project);
this.validate(config);
const execute = async (selected, key, input, progress) => {
+21 -5
View File
@@ -14,6 +14,8 @@ import { projectEnvironment } from '../core/process-environment.mjs';
import { GitDelivery } from '../core/git-delivery.mjs';
import { ProjectIndex } from '../core/context/project-index.mjs';
import { RequestPlanner, verifyPlanEvidence } from '../core/context/request-plan.mjs';
import { InputAttachments } from '../core/context/input-attachments.mjs';
import { BudgetResume } from '../core/context/budget-resume.mjs';
import { artifactPage, filePage, readMessageBudget, recordArtifactRead, recordBrowserRead, clearReadStreak, recordSessionInspection, readMetadataReserve, BROWSER_READ_HINT } from '../core/context/read-budget.mjs';
import { ChangeSetManager } from '../core/workspaces/change-sets.mjs';
import { assertChangeSetsResolved } from '../shared/contracts/change-sets.mjs';
@@ -119,6 +121,9 @@ export class Runtime {
redact: value => this.provider.redact(value), projectBusy: projectId => this.projectHasActivity(projectId) });
}
requestPlanner() { return this.contextPlans ??= new RequestPlanner(this.store); }
inputAttachments() { return this.inputs ??= new InputAttachments(this.store,{busy:projectId=>this.projectHasActivity(projectId)}); }
budgetResumes(){return this.resumes??=new BudgetResume(this.store,this.provider,this);}
resumeBudget(id,input){const session=this.store.session(id),plan=this.budgetResumes().verify(session,input);return this.start(id,'',true,{resumePlan:plan});}
engineRunner() { return this.controlledEngines ??= new EngineRunner(this.store, this.provider, { appDirectory: this.appDirectory }); }
memoryManager() { return this.projectMemories ??= new ProjectMemoryManager(this.store); }
checkpointManager() { return this.taskCheckpoints ??= new TaskCheckpointManager(this.store, {
@@ -159,7 +164,7 @@ export class Runtime {
|| this.store.core.collaboration.get(worker.collaborationId)?.status !== 'running') throw workerConflict('子任务只能由所属父协作启动。');
return this.start(session.id, worker.goal, true, { workerAuthorization: workerAuthority, worker, workerConfig: config });
}
start(id, content, resume = false, { instructionSource = null, workerAuthorization = null, worker = null, workerConfig = null } = {}) {
start(id, content, resume = false, { instructionSource = null, workerAuthorization = null, worker = null, workerConfig = null, attachmentIds=[],resumePlan=null } = {}) {
const session = this.store.session(id);
assertWorkersResolved(this.store, session.projectId);
const boundWorker = workerForSession(this.store, session);
@@ -174,19 +179,24 @@ export class Runtime {
if ([...this.store.core.delivery_job.values()].some(job => job.projectId === session.projectId && job.requiresVerification)) throw Object.assign(new Error('项目有结果未知的Git交付,请先在所属任务中核对提交结果。'), { status: 409 });
if (!resume && (typeof content !== 'string' || !content.trim() || content.length > 20000)) throw new Error('任务内容不能为空,也不能超过 20,000 字符。');
const project = this.store.project(session.projectId);
const attachments=this.inputAttachments().validateSelection(session,attachmentIds);
if(attachments.length&&this.store.core.task.get(session.taskId)?.attachmentIds?.length&&!['completed','idle'].includes(this.store.core.task.get(session.taskId).status))throw Object.assign(new Error('当前任务已有截图,请新建任务再添加。'),{status:409});
safePath(project);
const controller = new AbortController();
const initialConfig = structuredClone(workerConfig ?? this.store.config);
const config = workerConfig ? initialConfig : this.provider.prepareConfig ? this.provider.prepareConfig(initialConfig, this.store) : initialConfig;
const initialConfig = structuredClone(resumePlan?.config ?? workerConfig ?? this.store.config);
const config = resumePlan || workerConfig ? initialConfig : this.provider.prepareConfig ? this.provider.prepareConfig(initialConfig, this.store) : initialConfig;
if (config.mode !== 'demo') this.provider.validate(config);
const record = this.store.beginRun(session, content, resume, config, { instructionSource });
if (attachments.length && config.mode==='demo') throw Object.assign(new Error('演示执行器不会读取图片,请先选择支持视觉的模型连接。'),{status:409});
if (attachments.length && !config.modelRouting) throw Object.assign(new Error('请在模型中心添加并声明支持视觉的连接,再发送图片。'),{status:409});
const record = this.store.beginRun(session, content, resume, config, { instructionSource, inputAttachments:attachments,resumePlan });
const run = { controller, id: record.id, taskId: record.taskId, project, record, config, ...(worker ? { worker } : {}) };
this.active.set(id, run);
session.status = 'running'; session.recoveryNotice = null; session.executionMode = config.mode;
session.runId = run.id;
if (session.title === '新任务') session.title = String(content).replace(/\s+/g, ' ').slice(0, 28);
if (!resume || content) {
this.store.message(session, 'user', String(content));
const userMessage=this.store.message(session, 'user', String(content));
if(attachments.length)userMessage.attachmentIds=attachments.map(row=>row.boundAttachmentId);
session.providerMessages.push({ role: 'user', content: String(content) });
}
if (resume) this.store.message(session, 'system', '从已保存的执行记录继续,已完成的工具结果保留。');
@@ -231,6 +241,8 @@ export class Runtime {
checkBudget(run.record, this.store.core.task.get(run.taskId));
const stepRecord = this.store.beginStep(session, run.record);
const task = this.store.core.task.get(run.taskId);
config.inputImages=this.inputAttachments().images(session,task);
if(config.inputImages.length&&config.mode==='demo')throw new Error('固定演示执行器不会读取图片;请连接并声明支持视觉的模型。');
const taskPlan = task.currentTaskPlanId ? await this.taskPlanner().inspect(session, { candidates: false }) : null;
const state = session.changes.filter(change => change.taskId === task.id || ['pending', 'conflict'].includes(change.status))
.map(change => ({ path: change.path, status: change.status, id: change.id, changeSetId: change.changeSetId ?? null }));
@@ -408,6 +420,10 @@ export class Runtime {
const budget = error instanceof BudgetExceeded ? error : signal.reason instanceof BudgetExceeded ? signal.reason : null;
if (signal.aborted || budget) {
run.stopReason = budget?.reason ?? 'cancelled';
if(budget)run.record.budgetPause={schema:1,reason:budget.reason,createdAt:now(),taskId:run.taskId,runId:run.id,
...(budget.contextBudget??{}),runUsage:structuredClone(run.record.usage),taskUsage:structuredClone(this.store.core.task.get(run.taskId).usage),
runBudget:structuredClone(run.record.budget),taskBudget:structuredClone(this.store.core.task.get(run.taskId).budget),
note:'本地预留估算与服务实际用量分开记录;图片预留不是厂商计费或容量保证。'};
for (const event of session.events.filter(item => item.kind === 'model' && item.runId === run.id && ['running', 'retrying'].includes(item.phase))) {
event.phase = 'interrupted'; event.finishedAt = now(); event.output = '模型请求已停止,已完成的工具结果保留。';
}
+19 -1
View File
@@ -450,8 +450,15 @@ export class Store extends EventEmitter {
return session;
}
beginRun(session, content, resume, config, { instructionSource = null, forceNewTask = false } = {}) {
beginRun(session, content, resume, config, { instructionSource = null, forceNewTask = false, inputAttachments = [],resumePlan=null } = {}) {
let task = this.core.task.get(session.taskId);
if(resumePlan&&(!resume||!task||resumePlan.taskId!==task.id||this.owned('resume_plan',resumePlan.id,session.id)!==resumePlan||resumePlan.status!=='prepared'))throw new Error('续接预览不属于原任务。');
// Bind inputs in the same session transaction as run.started.
const inputs = inputAttachments.map(row => {
if (this.owned('input_attachment',row.id,session.id) !== row || row.status !== 'draft') throw new Error('附件选择已变化。');
return {row,artifacts:[row.originalArtifactId,row.sentArtifactId].map(id=>this.owned('artifact',id,session.id))};
});
if (inputs.length>1 || inputs.length && !forceNewTask && task?.attachmentIds?.length && (resume || !['completed','idle'].includes(task.status))) throw new Error('当前任务已有截图,请新建任务再添加。');
const previousRunId = session.runId ?? null;
if (forceNewTask || !task || !resume && ['completed', 'idle'].includes(task.status)) {
const previous = task?.id ?? null;
@@ -466,8 +473,18 @@ export class Store extends EventEmitter {
task.workspaceIdentity ??= structuredClone(workspaceIdentity);
if (JSON.stringify(task.workspaceIdentity) !== JSON.stringify(workspaceIdentity)) throw Object.assign(new Error('任务的工作区身份已变化,不能将原任务移动到另一个工作区。'), { status: 409 });
const run = createRun(session, task, config, resume ? previousRunId : null);
if(resumePlan){task.budget.maxTokens=config.maxTaskTokens;run.resumePlanId=resumePlan.id;resumePlan.status='consumed';resumePlan.newRunId=run.id;}
run.workspaceIdentity = structuredClone(workspaceIdentity);
this.core.run.set(run.id, run); task.activeRunId = run.id;
for (const {row,artifacts} of inputs) {
const copies=artifacts.map(artifact=>({...artifact,id:randomUUID(),taskId:task.id,runId:run.id}));
for(const artifact of copies)this.core.artifact.set(artifact.id,artifact);
const bound={...row,id:randomUUID(),taskId:task.id,runId:run.id,status:'attached',sourceDraftId:row.id,
originalArtifactId:copies[0].id,sentArtifactId:copies[1].id};
this.core.input_attachment.set(bound.id,bound);
row.status='consumed';row.boundAttachmentId=bound.id;
}
if (inputs.length) task.attachmentIds=inputs.map(({row})=>row.boundAttachmentId);
session.taskId = task.id; session.runId = run.id; session.status = 'running';
this.save(session, { type: 'run.started', detail: { resumeOf: run.resumeOf } });
return run;
@@ -682,6 +699,7 @@ export class Store extends EventEmitter {
publicSession(session) {
const { providerMessages, ...visible } = session;
visible.inputAttachments=[...this.core.input_attachment.values()].filter(row=>row.sessionId===session.id&&['draft','attached'].includes(row.status));
visible.execution = this.executionSnapshot(session);
visible.lastFailure = lastSessionFailure(session);
if (session.status === 'failed' && !visible.failure && visible.lastFailure) {