feat(vision): bind task screenshots, route vision and resume budget pauses

This commit is contained in:
Agent Workbench Dev
2026-11-01 00:48:38 +08:00
parent 0d1ab743f1
commit 96a7ed19c7
79 changed files with 1721 additions and 36 deletions
+21 -5
View File
@@ -14,6 +14,8 @@ import { projectEnvironment } from '../core/process-environment.mjs';
import { GitDelivery } from '../core/git-delivery.mjs';
import { ProjectIndex } from '../core/context/project-index.mjs';
import { RequestPlanner, verifyPlanEvidence } from '../core/context/request-plan.mjs';
import { InputAttachments } from '../core/context/input-attachments.mjs';
import { BudgetResume } from '../core/context/budget-resume.mjs';
import { artifactPage, filePage, readMessageBudget, recordArtifactRead, recordBrowserRead, clearReadStreak, recordSessionInspection, readMetadataReserve, BROWSER_READ_HINT } from '../core/context/read-budget.mjs';
import { ChangeSetManager } from '../core/workspaces/change-sets.mjs';
import { assertChangeSetsResolved } from '../shared/contracts/change-sets.mjs';
@@ -119,6 +121,9 @@ export class Runtime {
redact: value => this.provider.redact(value), projectBusy: projectId => this.projectHasActivity(projectId) });
}
requestPlanner() { return this.contextPlans ??= new RequestPlanner(this.store); }
inputAttachments() { return this.inputs ??= new InputAttachments(this.store,{busy:projectId=>this.projectHasActivity(projectId)}); }
budgetResumes(){return this.resumes??=new BudgetResume(this.store,this.provider,this);}
resumeBudget(id,input){const session=this.store.session(id),plan=this.budgetResumes().verify(session,input);return this.start(id,'',true,{resumePlan:plan});}
engineRunner() { return this.controlledEngines ??= new EngineRunner(this.store, this.provider, { appDirectory: this.appDirectory }); }
memoryManager() { return this.projectMemories ??= new ProjectMemoryManager(this.store); }
checkpointManager() { return this.taskCheckpoints ??= new TaskCheckpointManager(this.store, {
@@ -159,7 +164,7 @@ export class Runtime {
|| this.store.core.collaboration.get(worker.collaborationId)?.status !== 'running') throw workerConflict('子任务只能由所属父协作启动。');
return this.start(session.id, worker.goal, true, { workerAuthorization: workerAuthority, worker, workerConfig: config });
}
start(id, content, resume = false, { instructionSource = null, workerAuthorization = null, worker = null, workerConfig = null } = {}) {
start(id, content, resume = false, { instructionSource = null, workerAuthorization = null, worker = null, workerConfig = null, attachmentIds=[],resumePlan=null } = {}) {
const session = this.store.session(id);
assertWorkersResolved(this.store, session.projectId);
const boundWorker = workerForSession(this.store, session);
@@ -174,19 +179,24 @@ export class Runtime {
if ([...this.store.core.delivery_job.values()].some(job => job.projectId === session.projectId && job.requiresVerification)) throw Object.assign(new Error('项目有结果未知的Git交付,请先在所属任务中核对提交结果。'), { status: 409 });
if (!resume && (typeof content !== 'string' || !content.trim() || content.length > 20000)) throw new Error('任务内容不能为空,也不能超过 20,000 字符。');
const project = this.store.project(session.projectId);
const attachments=this.inputAttachments().validateSelection(session,attachmentIds);
if(attachments.length&&this.store.core.task.get(session.taskId)?.attachmentIds?.length&&!['completed','idle'].includes(this.store.core.task.get(session.taskId).status))throw Object.assign(new Error('当前任务已有截图,请新建任务再添加。'),{status:409});
safePath(project);
const controller = new AbortController();
const initialConfig = structuredClone(workerConfig ?? this.store.config);
const config = workerConfig ? initialConfig : this.provider.prepareConfig ? this.provider.prepareConfig(initialConfig, this.store) : initialConfig;
const initialConfig = structuredClone(resumePlan?.config ?? workerConfig ?? this.store.config);
const config = resumePlan || workerConfig ? initialConfig : this.provider.prepareConfig ? this.provider.prepareConfig(initialConfig, this.store) : initialConfig;
if (config.mode !== 'demo') this.provider.validate(config);
const record = this.store.beginRun(session, content, resume, config, { instructionSource });
if (attachments.length && config.mode==='demo') throw Object.assign(new Error('演示执行器不会读取图片,请先选择支持视觉的模型连接。'),{status:409});
if (attachments.length && !config.modelRouting) throw Object.assign(new Error('请在模型中心添加并声明支持视觉的连接,再发送图片。'),{status:409});
const record = this.store.beginRun(session, content, resume, config, { instructionSource, inputAttachments:attachments,resumePlan });
const run = { controller, id: record.id, taskId: record.taskId, project, record, config, ...(worker ? { worker } : {}) };
this.active.set(id, run);
session.status = 'running'; session.recoveryNotice = null; session.executionMode = config.mode;
session.runId = run.id;
if (session.title === '新任务') session.title = String(content).replace(/\s+/g, ' ').slice(0, 28);
if (!resume || content) {
this.store.message(session, 'user', String(content));
const userMessage=this.store.message(session, 'user', String(content));
if(attachments.length)userMessage.attachmentIds=attachments.map(row=>row.boundAttachmentId);
session.providerMessages.push({ role: 'user', content: String(content) });
}
if (resume) this.store.message(session, 'system', '从已保存的执行记录继续,已完成的工具结果保留。');
@@ -231,6 +241,8 @@ export class Runtime {
checkBudget(run.record, this.store.core.task.get(run.taskId));
const stepRecord = this.store.beginStep(session, run.record);
const task = this.store.core.task.get(run.taskId);
config.inputImages=this.inputAttachments().images(session,task);
if(config.inputImages.length&&config.mode==='demo')throw new Error('固定演示执行器不会读取图片;请连接并声明支持视觉的模型。');
const taskPlan = task.currentTaskPlanId ? await this.taskPlanner().inspect(session, { candidates: false }) : null;
const state = session.changes.filter(change => change.taskId === task.id || ['pending', 'conflict'].includes(change.status))
.map(change => ({ path: change.path, status: change.status, id: change.id, changeSetId: change.changeSetId ?? null }));
@@ -408,6 +420,10 @@ export class Runtime {
const budget = error instanceof BudgetExceeded ? error : signal.reason instanceof BudgetExceeded ? signal.reason : null;
if (signal.aborted || budget) {
run.stopReason = budget?.reason ?? 'cancelled';
if(budget)run.record.budgetPause={schema:1,reason:budget.reason,createdAt:now(),taskId:run.taskId,runId:run.id,
...(budget.contextBudget??{}),runUsage:structuredClone(run.record.usage),taskUsage:structuredClone(this.store.core.task.get(run.taskId).usage),
runBudget:structuredClone(run.record.budget),taskBudget:structuredClone(this.store.core.task.get(run.taskId).budget),
note:'本地预留估算与服务实际用量分开记录;图片预留不是厂商计费或容量保证。'};
for (const event of session.events.filter(item => item.kind === 'model' && item.runId === run.id && ['running', 'retrying'].includes(item.phase))) {
event.phase = 'interrupted'; event.finishedAt = now(); event.output = '模型请求已停止,已完成的工具结果保留。';
}