feat(vision): bind task screenshots, route vision and resume budget pauses
This commit is contained in:
+22
-3
@@ -48,12 +48,12 @@ function json(response, value, status = 200) {
|
||||
'X-Content-Type-Options': 'nosniff' });
|
||||
response.end(JSON.stringify(value));
|
||||
}
|
||||
async function body(request) {
|
||||
async function body(request, maximum=2*1024*1024) {
|
||||
if (!String(request.headers['content-type'] ?? '').startsWith('application/json')) throw Object.assign(new Error('请求需要 JSON。'), { status: 415 });
|
||||
const chunks = []; let length = 0;
|
||||
for await (const chunk of request) {
|
||||
length += chunk.length;
|
||||
if (length > 2 * 1024 * 1024) throw Object.assign(new Error('请求超过 2 MB。'), { status: 413 });
|
||||
if (length > maximum) throw Object.assign(new Error('请求超过允许大小。'), { status: 413 });
|
||||
chunks.push(chunk);
|
||||
}
|
||||
let input;
|
||||
@@ -269,6 +269,23 @@ return async (request, response) => {
|
||||
if (sessionMatch) {
|
||||
const id = sessionMatch[1], action = sessionMatch[2] ?? '';
|
||||
const session = store.session(id);
|
||||
if (action === 'input-attachments') {
|
||||
if (method === 'GET') return json(response,runtime.inputAttachments().list(session));
|
||||
if (workerForProject(store,session.projectId)) throw workerConflict('截图附件请在父任务中添加。');
|
||||
if (method === 'POST') return json(response,runtime.inputAttachments().upload(session,await body(request,3*1024*1024)),201);
|
||||
}
|
||||
const inputImage = action.match(/^input-attachments\/([a-f0-9-]+)(?:\/(image|original))?$/);
|
||||
if (inputImage) {
|
||||
const manager=runtime.inputAttachments();
|
||||
if(method==='DELETE'&&!inputImage[2]) { await body(request);return json(response,manager.remove(session,inputImage[1])); }
|
||||
if(method==='GET'&&inputImage[2]) {
|
||||
const {row,artifact,bytes}=manager.read(session,inputImage[1],{original:inputImage[2]==='original'});
|
||||
const download=inputImage[2]==='original'||url.searchParams.get('download')==='1';
|
||||
response.writeHead(200,{'Content-Type':artifact.mediaType,'Cache-Control':'no-store','X-Content-Type-Options':'nosniff','X-Content-SHA256':artifact.sha256,
|
||||
...(download?{'Content-Disposition':'attachment; filename="workbench-input-'+row.id+(artifact.mediaType==='image/png'?'.png':'.jpg')+'"'}:{})});
|
||||
response.end(bytes);return;
|
||||
}
|
||||
}
|
||||
if (method === 'POST' && workerForProject(store, session.projectId)
|
||||
&& /^(?:mcp|engines|research|workspaces|delivery|browser|task-plans|task-proofs|task-checkpoints|collaborations)(?:\/|-|$)/.test(action)) throw workerConflict('受限子任务只开放原目标的读取、范围内改动和已审批检查;此操作请返回父项目。');
|
||||
if (method === 'GET' && action === 'collaborations') return json(response, runtime.workerManager().view(session));
|
||||
@@ -574,7 +591,7 @@ return async (request, response) => {
|
||||
}
|
||||
if (method === 'POST' && action === 'messages') {
|
||||
const input = await body(request);
|
||||
runtime.start(id, input.content);
|
||||
runtime.start(id, input.content,false,{attachmentIds:input.attachmentIds??[]});
|
||||
return json(response, store.publicSession(session), 202);
|
||||
}
|
||||
if (method === 'POST' && action === 'resume') {
|
||||
@@ -582,6 +599,8 @@ return async (request, response) => {
|
||||
runtime.resume(id, input.content);
|
||||
return json(response, store.publicSession(session), 202);
|
||||
}
|
||||
if(method==='POST'&&action==='resume-preview')return json(response,runtime.budgetResumes().prepare(session,await body(request)),201);
|
||||
if(method==='POST'&&action==='resume-budget'){runtime.resumeBudget(id,await body(request));return json(response,store.publicSession(session),202);}
|
||||
if (method === 'POST' && action === 'continue-reviewed') {
|
||||
await body(request); runtime.continueReviewed(id);
|
||||
return json(response, store.publicSession(session), 202);
|
||||
|
||||
@@ -9,6 +9,7 @@ import { CompanionTLS } from '../core/companion/tls-authority.mjs';
|
||||
import { fields, companionError, companionHash, approvalBinding, operationInput } from '../shared/contracts/companion.mjs';
|
||||
import { isId } from '../shared/contracts/execution.mjs';
|
||||
import { mobileSession, sessionBrief, reportKinds } from '../core/companion/projection.mjs';
|
||||
import { InputAttachments } from '../core/context/input-attachments.mjs';
|
||||
import { hostAuthority } from '../core/desktop/host-authority.mjs';
|
||||
|
||||
export function companionGateway(context) {
|
||||
@@ -179,7 +180,7 @@ export class CompanionGateway {
|
||||
async handle(request, response) {
|
||||
response.setHeader('Cache-Control', 'no-store'); response.setHeader('X-Content-Type-Options', 'nosniff');
|
||||
response.setHeader('Referrer-Policy', 'no-referrer'); response.setHeader('X-Frame-Options', 'DENY');
|
||||
response.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'");
|
||||
response.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data: blob:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'");
|
||||
try {
|
||||
this.registry.assertHost();
|
||||
if (!this.server?.listening || !this.origin || request.headers.host !== new URL(this.origin).host) throw companionError('手机网关地址已变化,请重新连接。', 421);
|
||||
@@ -220,6 +221,14 @@ export class CompanionGateway {
|
||||
if (limit < 1 || limit > 200) throw companionError('手机事件每页需要1至200条。');
|
||||
return reply(response, this.events(device, after, limit));
|
||||
}
|
||||
const attachment=route.match(/^\/mobile-api\/sessions\/([a-f0-9-]+)\/input-attachments\/([a-f0-9-]+)\/(image|original)$/);
|
||||
if(attachment&&method==='GET'){
|
||||
const session=this.session(device,attachment[1]),manager=new InputAttachments(this.store),row=manager.get(session,attachment[2]);
|
||||
if(row.taskId!==session.taskId||row.status!=='attached'||!(this.store.core.task.get(session.taskId)?.attachmentIds??[]).includes(row.id))throw companionError('当前任务没有这张截图。',404);
|
||||
const {artifact,bytes}=manager.read(session,row.id,{original:attachment[3]==='original'});
|
||||
response.writeHead(200,{'Content-Type':artifact.mediaType,'Content-Length':bytes.length,'X-Content-SHA256':artifact.sha256,
|
||||
...(attachment[3]==='original'?{'Content-Disposition':'attachment; filename="workbench-input-'+row.id+(row.mediaType==='image/png'?'.png':'.jpg')+'"'}:{})});response.end(bytes);return;
|
||||
}
|
||||
const match = route.match(/^\/mobile-api\/sessions\/([a-f0-9-]+)(?:\/(approve|stop|changes\/([a-f0-9-]+)|reports\/([a-f0-9-]+)))?$/);
|
||||
if (match) {
|
||||
const session = this.session(device, match[1]);
|
||||
|
||||
@@ -34,6 +34,8 @@ export function installUpgrade(context, { appDirectory, updateServiceInfo = true
|
||||
const oldStoreProperties = { ...store };
|
||||
const oldRuntimeProperties = { appDirectory: runtime.appDirectory, workers: runtime.workers, controlledEngines: runtime.controlledEngines, checks: runtime.checks, deliveries: runtime.deliveries, codeIndex: runtime.codeIndex, changeSets: runtime.changeSets, contextPlans: runtime.contextPlans, workspaces: runtime.workspaces, research: runtime.research, engines: runtime.engines, browser: runtime.browser, taskPlans: runtime.taskPlans, projectMemories: runtime.projectMemories, taskCheckpoints: runtime.taskCheckpoints, skills: runtime.skills, mcp: runtime.mcp };
|
||||
const oldUpdates = store.listeners('update');
|
||||
const oldInputs = runtime.inputs;
|
||||
const oldResumes = runtime.resumes;
|
||||
const closeStorage = () => store.close();
|
||||
if (oldUpdates.length > 1) throw new Error('Unexpected Store observers; preserve their ownership before upgrading.');
|
||||
const handler = createRequestHandler({ appDirectory: root, server, store, provider, runtime, token, clients, started,
|
||||
@@ -49,6 +51,8 @@ export function installUpgrade(context, { appDirectory, updateServiceInfo = true
|
||||
}
|
||||
runtime.appDirectory = root; runtime.workers = null; runtime.controlledEngines = null; runtime.checks = null; runtime.deliveries = null; runtime.codeIndex = null; runtime.changeSets = null; runtime.contextPlans = null; runtime.workspaces = null; runtime.research = null; runtime.engines = null; runtime.browser = null; runtime.taskPlans = null; runtime.projectMemories = null; runtime.taskCheckpoints = null; runtime.skills = null; runtime.mcp = null;
|
||||
delete server.workbenchCompanion;
|
||||
runtime.inputs = null;
|
||||
runtime.resumes = null;
|
||||
store.ensureStorage();
|
||||
// A SQLite host can retain an artifact object from an earlier bundle.
|
||||
// Reopen the same blobs through the new API without mutating that object;
|
||||
@@ -80,6 +84,8 @@ export function installUpgrade(context, { appDirectory, updateServiceInfo = true
|
||||
for (const key of Object.keys(store)) if (!Object.hasOwn(oldStoreProperties, key)) delete store[key];
|
||||
Object.assign(store, oldStoreProperties);
|
||||
Object.assign(runtime, oldRuntimeProperties);
|
||||
if (oldInputs === undefined) delete runtime.inputs; else runtime.inputs = oldInputs;
|
||||
if (oldResumes === undefined) delete runtime.resumes; else runtime.resumes = oldResumes;
|
||||
Object.setPrototypeOf(store, oldPrototypes[0]); Object.setPrototypeOf(provider, oldPrototypes[1]); Object.setPrototypeOf(runtime, oldPrototypes[2]);
|
||||
if (oldModelPrototypes.hub) Object.setPrototypeOf(provider.hub, oldModelPrototypes.hub);
|
||||
if (oldModelPrototypes.importer) Object.setPrototypeOf(provider.hub.importer, oldModelPrototypes.importer);
|
||||
|
||||
@@ -210,6 +210,7 @@ export class Provider {
|
||||
async complete(config, messages, project, signal, onProgress, requestPlan = null, tools = toolDefinitions, requestPolicy = {}) {
|
||||
const body = completionBody(config, messages, tools);
|
||||
assertPlannedRequest(body, requestPlan);
|
||||
if (config.inputImages?.length && (!config.modelRouting || !this.hub || config.mode==='demo')) throw new Error('图片请求必须通过模型中心的视觉能力选路,尚未发送。');
|
||||
if (config.mode === 'demo') return this.demo(messages, project);
|
||||
this.validate(config);
|
||||
const execute = async (selected, key, input, progress) => {
|
||||
|
||||
+21
-5
@@ -14,6 +14,8 @@ import { projectEnvironment } from '../core/process-environment.mjs';
|
||||
import { GitDelivery } from '../core/git-delivery.mjs';
|
||||
import { ProjectIndex } from '../core/context/project-index.mjs';
|
||||
import { RequestPlanner, verifyPlanEvidence } from '../core/context/request-plan.mjs';
|
||||
import { InputAttachments } from '../core/context/input-attachments.mjs';
|
||||
import { BudgetResume } from '../core/context/budget-resume.mjs';
|
||||
import { artifactPage, filePage, readMessageBudget, recordArtifactRead, recordBrowserRead, clearReadStreak, recordSessionInspection, readMetadataReserve, BROWSER_READ_HINT } from '../core/context/read-budget.mjs';
|
||||
import { ChangeSetManager } from '../core/workspaces/change-sets.mjs';
|
||||
import { assertChangeSetsResolved } from '../shared/contracts/change-sets.mjs';
|
||||
@@ -119,6 +121,9 @@ export class Runtime {
|
||||
redact: value => this.provider.redact(value), projectBusy: projectId => this.projectHasActivity(projectId) });
|
||||
}
|
||||
requestPlanner() { return this.contextPlans ??= new RequestPlanner(this.store); }
|
||||
inputAttachments() { return this.inputs ??= new InputAttachments(this.store,{busy:projectId=>this.projectHasActivity(projectId)}); }
|
||||
budgetResumes(){return this.resumes??=new BudgetResume(this.store,this.provider,this);}
|
||||
resumeBudget(id,input){const session=this.store.session(id),plan=this.budgetResumes().verify(session,input);return this.start(id,'',true,{resumePlan:plan});}
|
||||
engineRunner() { return this.controlledEngines ??= new EngineRunner(this.store, this.provider, { appDirectory: this.appDirectory }); }
|
||||
memoryManager() { return this.projectMemories ??= new ProjectMemoryManager(this.store); }
|
||||
checkpointManager() { return this.taskCheckpoints ??= new TaskCheckpointManager(this.store, {
|
||||
@@ -159,7 +164,7 @@ export class Runtime {
|
||||
|| this.store.core.collaboration.get(worker.collaborationId)?.status !== 'running') throw workerConflict('子任务只能由所属父协作启动。');
|
||||
return this.start(session.id, worker.goal, true, { workerAuthorization: workerAuthority, worker, workerConfig: config });
|
||||
}
|
||||
start(id, content, resume = false, { instructionSource = null, workerAuthorization = null, worker = null, workerConfig = null } = {}) {
|
||||
start(id, content, resume = false, { instructionSource = null, workerAuthorization = null, worker = null, workerConfig = null, attachmentIds=[],resumePlan=null } = {}) {
|
||||
const session = this.store.session(id);
|
||||
assertWorkersResolved(this.store, session.projectId);
|
||||
const boundWorker = workerForSession(this.store, session);
|
||||
@@ -174,19 +179,24 @@ export class Runtime {
|
||||
if ([...this.store.core.delivery_job.values()].some(job => job.projectId === session.projectId && job.requiresVerification)) throw Object.assign(new Error('项目有结果未知的Git交付,请先在所属任务中核对提交结果。'), { status: 409 });
|
||||
if (!resume && (typeof content !== 'string' || !content.trim() || content.length > 20000)) throw new Error('任务内容不能为空,也不能超过 20,000 字符。');
|
||||
const project = this.store.project(session.projectId);
|
||||
const attachments=this.inputAttachments().validateSelection(session,attachmentIds);
|
||||
if(attachments.length&&this.store.core.task.get(session.taskId)?.attachmentIds?.length&&!['completed','idle'].includes(this.store.core.task.get(session.taskId).status))throw Object.assign(new Error('当前任务已有截图,请新建任务再添加。'),{status:409});
|
||||
safePath(project);
|
||||
const controller = new AbortController();
|
||||
const initialConfig = structuredClone(workerConfig ?? this.store.config);
|
||||
const config = workerConfig ? initialConfig : this.provider.prepareConfig ? this.provider.prepareConfig(initialConfig, this.store) : initialConfig;
|
||||
const initialConfig = structuredClone(resumePlan?.config ?? workerConfig ?? this.store.config);
|
||||
const config = resumePlan || workerConfig ? initialConfig : this.provider.prepareConfig ? this.provider.prepareConfig(initialConfig, this.store) : initialConfig;
|
||||
if (config.mode !== 'demo') this.provider.validate(config);
|
||||
const record = this.store.beginRun(session, content, resume, config, { instructionSource });
|
||||
if (attachments.length && config.mode==='demo') throw Object.assign(new Error('演示执行器不会读取图片,请先选择支持视觉的模型连接。'),{status:409});
|
||||
if (attachments.length && !config.modelRouting) throw Object.assign(new Error('请在模型中心添加并声明支持视觉的连接,再发送图片。'),{status:409});
|
||||
const record = this.store.beginRun(session, content, resume, config, { instructionSource, inputAttachments:attachments,resumePlan });
|
||||
const run = { controller, id: record.id, taskId: record.taskId, project, record, config, ...(worker ? { worker } : {}) };
|
||||
this.active.set(id, run);
|
||||
session.status = 'running'; session.recoveryNotice = null; session.executionMode = config.mode;
|
||||
session.runId = run.id;
|
||||
if (session.title === '新任务') session.title = String(content).replace(/\s+/g, ' ').slice(0, 28);
|
||||
if (!resume || content) {
|
||||
this.store.message(session, 'user', String(content));
|
||||
const userMessage=this.store.message(session, 'user', String(content));
|
||||
if(attachments.length)userMessage.attachmentIds=attachments.map(row=>row.boundAttachmentId);
|
||||
session.providerMessages.push({ role: 'user', content: String(content) });
|
||||
}
|
||||
if (resume) this.store.message(session, 'system', '从已保存的执行记录继续,已完成的工具结果保留。');
|
||||
@@ -231,6 +241,8 @@ export class Runtime {
|
||||
checkBudget(run.record, this.store.core.task.get(run.taskId));
|
||||
const stepRecord = this.store.beginStep(session, run.record);
|
||||
const task = this.store.core.task.get(run.taskId);
|
||||
config.inputImages=this.inputAttachments().images(session,task);
|
||||
if(config.inputImages.length&&config.mode==='demo')throw new Error('固定演示执行器不会读取图片;请连接并声明支持视觉的模型。');
|
||||
const taskPlan = task.currentTaskPlanId ? await this.taskPlanner().inspect(session, { candidates: false }) : null;
|
||||
const state = session.changes.filter(change => change.taskId === task.id || ['pending', 'conflict'].includes(change.status))
|
||||
.map(change => ({ path: change.path, status: change.status, id: change.id, changeSetId: change.changeSetId ?? null }));
|
||||
@@ -408,6 +420,10 @@ export class Runtime {
|
||||
const budget = error instanceof BudgetExceeded ? error : signal.reason instanceof BudgetExceeded ? signal.reason : null;
|
||||
if (signal.aborted || budget) {
|
||||
run.stopReason = budget?.reason ?? 'cancelled';
|
||||
if(budget)run.record.budgetPause={schema:1,reason:budget.reason,createdAt:now(),taskId:run.taskId,runId:run.id,
|
||||
...(budget.contextBudget??{}),runUsage:structuredClone(run.record.usage),taskUsage:structuredClone(this.store.core.task.get(run.taskId).usage),
|
||||
runBudget:structuredClone(run.record.budget),taskBudget:structuredClone(this.store.core.task.get(run.taskId).budget),
|
||||
note:'本地预留估算与服务实际用量分开记录;图片预留不是厂商计费或容量保证。'};
|
||||
for (const event of session.events.filter(item => item.kind === 'model' && item.runId === run.id && ['running', 'retrying'].includes(item.phase))) {
|
||||
event.phase = 'interrupted'; event.finishedAt = now(); event.output = '模型请求已停止,已完成的工具结果保留。';
|
||||
}
|
||||
|
||||
+19
-1
@@ -450,8 +450,15 @@ export class Store extends EventEmitter {
|
||||
return session;
|
||||
}
|
||||
|
||||
beginRun(session, content, resume, config, { instructionSource = null, forceNewTask = false } = {}) {
|
||||
beginRun(session, content, resume, config, { instructionSource = null, forceNewTask = false, inputAttachments = [],resumePlan=null } = {}) {
|
||||
let task = this.core.task.get(session.taskId);
|
||||
if(resumePlan&&(!resume||!task||resumePlan.taskId!==task.id||this.owned('resume_plan',resumePlan.id,session.id)!==resumePlan||resumePlan.status!=='prepared'))throw new Error('续接预览不属于原任务。');
|
||||
// Bind inputs in the same session transaction as run.started.
|
||||
const inputs = inputAttachments.map(row => {
|
||||
if (this.owned('input_attachment',row.id,session.id) !== row || row.status !== 'draft') throw new Error('附件选择已变化。');
|
||||
return {row,artifacts:[row.originalArtifactId,row.sentArtifactId].map(id=>this.owned('artifact',id,session.id))};
|
||||
});
|
||||
if (inputs.length>1 || inputs.length && !forceNewTask && task?.attachmentIds?.length && (resume || !['completed','idle'].includes(task.status))) throw new Error('当前任务已有截图,请新建任务再添加。');
|
||||
const previousRunId = session.runId ?? null;
|
||||
if (forceNewTask || !task || !resume && ['completed', 'idle'].includes(task.status)) {
|
||||
const previous = task?.id ?? null;
|
||||
@@ -466,8 +473,18 @@ export class Store extends EventEmitter {
|
||||
task.workspaceIdentity ??= structuredClone(workspaceIdentity);
|
||||
if (JSON.stringify(task.workspaceIdentity) !== JSON.stringify(workspaceIdentity)) throw Object.assign(new Error('任务的工作区身份已变化,不能将原任务移动到另一个工作区。'), { status: 409 });
|
||||
const run = createRun(session, task, config, resume ? previousRunId : null);
|
||||
if(resumePlan){task.budget.maxTokens=config.maxTaskTokens;run.resumePlanId=resumePlan.id;resumePlan.status='consumed';resumePlan.newRunId=run.id;}
|
||||
run.workspaceIdentity = structuredClone(workspaceIdentity);
|
||||
this.core.run.set(run.id, run); task.activeRunId = run.id;
|
||||
for (const {row,artifacts} of inputs) {
|
||||
const copies=artifacts.map(artifact=>({...artifact,id:randomUUID(),taskId:task.id,runId:run.id}));
|
||||
for(const artifact of copies)this.core.artifact.set(artifact.id,artifact);
|
||||
const bound={...row,id:randomUUID(),taskId:task.id,runId:run.id,status:'attached',sourceDraftId:row.id,
|
||||
originalArtifactId:copies[0].id,sentArtifactId:copies[1].id};
|
||||
this.core.input_attachment.set(bound.id,bound);
|
||||
row.status='consumed';row.boundAttachmentId=bound.id;
|
||||
}
|
||||
if (inputs.length) task.attachmentIds=inputs.map(({row})=>row.boundAttachmentId);
|
||||
session.taskId = task.id; session.runId = run.id; session.status = 'running';
|
||||
this.save(session, { type: 'run.started', detail: { resumeOf: run.resumeOf } });
|
||||
return run;
|
||||
@@ -682,6 +699,7 @@ export class Store extends EventEmitter {
|
||||
|
||||
publicSession(session) {
|
||||
const { providerMessages, ...visible } = session;
|
||||
visible.inputAttachments=[...this.core.input_attachment.values()].filter(row=>row.sessionId===session.id&&['draft','attached'].includes(row.status));
|
||||
visible.execution = this.executionSnapshot(session);
|
||||
visible.lastFailure = lastSessionFailure(session);
|
||||
if (session.status === 'failed' && !visible.failure && visible.lastFailure) {
|
||||
|
||||
Reference in New Issue
Block a user