fix: bind observed models and calibrate every model dispatch budget

This commit is contained in:
Agent Workbench Dev
2026-11-02 20:50:42 +08:00
parent 30c318bf20
commit bb45dabac4
23 changed files with 526 additions and 29 deletions
+42
View File
@@ -0,0 +1,42 @@
// This reports observed Task consumption, never the model selected in bootstrap.
// Public execution views may contain only the most recent Runs; gaps stay explicit.
export function observedTaskModels(state,taskId=state?.execution?.task?.id){
const execution=state?.execution??{},runs=new Map(),attempts=new Map(),issues=[];
for(const run of [...(execution.runs??[]),execution.run].filter(Boolean)){
if(taskId&&run.taskId===taskId&&typeof run.id==='string')runs.set(run.id,run);
}
const task=execution.task?.id===taskId?execution.task:null;
const validCount=value=>Number.isSafeInteger(value)&&value>=0;
let declared=task&&validCount(task.usage?.requests)?task.usage.requests:null;
if(declared===null&&runs.size&&[...runs.values()].every(run=>validCount(run.usage?.requests)))
declared=[...runs.values()].reduce((total,run)=>total+run.usage.requests,0);
for(const run of runs.values())for(const receipt of run.consumption??[]){
if(typeof receipt.attemptId!=='string'||!receipt.attemptId){issues.push({kind:'missing_attempt_id',runId:run.id});continue;}
const attempt={runId:run.id,attemptId:receipt.attemptId,requestId:receipt.requestId??null,
model:typeof receipt.model==='string'&&receipt.model.trim()?receipt.model:null,
protocol:typeof receipt.protocol==='string'&&receipt.protocol.trim()?receipt.protocol:null,
connectionId:receipt.connectionId??null,outcome:receipt.outcome??null,
usageStatus:receipt.usageStatus??'unknown',usage:receipt.usage??null};
const previous=attempts.get(attempt.attemptId);
if(previous){
if(JSON.stringify(previous)!==JSON.stringify(attempt))issues.push({kind:'conflicting_attempt',attemptId:attempt.attemptId,runIds:[previous.runId,run.id]});
continue;
}
attempts.set(attempt.attemptId,attempt);
if(!attempt.model||!attempt.protocol)issues.push({kind:'missing_route_identity',attemptId:attempt.attemptId});
if(attempt.usageStatus==='reported'&&!attempt.usage)issues.push({kind:'missing_reported_usage',attemptId:attempt.attemptId});
}
const observed=[...attempts.values()],models=[...new Set(observed.map(value=>value.model).filter(Boolean))],routes=new Map();
for(const attempt of observed){
const identity={connectionId:attempt.connectionId,model:attempt.model,protocol:attempt.protocol},key=JSON.stringify(identity);
if(!routes.has(key))routes.set(key,{...identity,requests:0,reportedUsageRequests:0,unknownUsageRequests:0});
const route=routes.get(key);route.requests++;
if(attempt.usageStatus==='reported'&&attempt.usage)route.reportedUsageRequests++;else route.unknownUsageRequests++;
}
if(!taskId)issues.push({kind:'missing_task_identity'});
if(declared!==null&&declared!==observed.length)issues.push({kind:'request_coverage_gap',recordedRequests:declared,observedAttempts:observed.length});
const status=issues.length?'incomplete':!observed.length?declared===0?'not_observed':'unavailable':models.length>1?'mixed':'observed';
return {schemaVersion:1,source:'task_run_consumption',taskId:taskId??null,status,
model:['observed','mixed'].includes(status)&&models.length===1?models[0]:null,models,
observedAttempts:observed.length,recordedRequests:declared,routes:[...routes.values()],attempts:observed,issues};
}
+3 -3
View File
@@ -52,7 +52,7 @@ const frozen = inputs(), mark = message => { cases.push(message); console.log('P
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
const startedAt = Date.now(), label = sampleName;
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
fs.mkdirSync(sample); let runtime, group, result, desktopQa, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
fs.mkdirSync(sample); let runtime, group, result, desktopQa, model, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
try {
const baseline = contextMode ? 'docs/live-model-purge-1.24-evidence/source-baselines/r1-off' : kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
@@ -74,7 +74,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
};
if (kind === 'react') install(f.projectRoot, 'parent');
let phase = 'new';
const model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
assertToolPairs(input.messages);
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
@@ -237,7 +237,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
writable, reviews, modelRequests, ...(desktopQa ? { desktop: { events: desktopQa.events, screenshots: desktopQa.screenshots, scope: desktopQa.scope, errors: desktopQa.errors } } : {}), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
return result;
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, ownedFixtureErrors:model?.errors??[], engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
finally {
await desktopQa?.close();
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+14 -4
View File
@@ -10,6 +10,7 @@ import { createPythonDeliveryProject, pythonProtected } from '../tests/fixtures/
import { deliveryGit, writeDeliveryFile } from '../tests/fixtures/delivery-project.mjs';
import { captureWorkspaceSource, assertWorkspaceSource } from '../tests/fixtures/workspace-project.mjs';
import { verifyLiveReactClient } from './live-react-client.mjs';
import { observedTaskModels } from './live-model-observation.mjs';
if (process.argv.includes('--help')) {
console.log('node scripts/verify-live-engineering.mjs --case react|python [--url http://127.0.0.1:4318] — fresh owned Git source/workspace, configured model and bounded review/check/delivery acceptance'); process.exit(0);
@@ -149,7 +150,7 @@ try {
fs.writeFileSync(path.join(root, 'delivery-report.json'), reportText); fs.writeFileSync(path.join(root, 'delivery-report.md'), markdown);
const browser = kind === 'react' ? await verifyLiveReactClient(childProject.root, path.join(root, 'screenshots')) : null;
fs.writeFileSync(path.join(root, 'session.json'), JSON.stringify(await api(route + '/export'), null, 2) + '\n');
result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, model: bootstrap.config.model, root,
result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, root,
taskId, sessionId: childSession.id, projectId: childProject.id, sourceProjectId: parentProject.id, workspaceId: workspace.id, workspaceRoot: childProject.root,
sourceCommit: workspace.source.head, commit: completedDelivery.commit, tests, checks: passing.results, usage: state.execution.task.usage,
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), applications, decisions, continuations, browser,
@@ -158,11 +159,20 @@ try {
credentialsHandling: 'The original Provider executes requests in its own process; no model key is read, copied or written.' };
succeeded = true;
} catch (error) {
result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root, model: bootstrap.config.model,
result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root,
taskId, sessionId: childSession?.id, projectId: childProject?.id, workspaceId: workspace?.id, error: error.stack,
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), usage: state?.execution?.task?.usage, state: state?.status,
stopReason: state?.execution?.run?.stopReason, applications, decisions, continuations, checks: state?.execution?.checks }; process.exitCode = 1;
} finally { if (!succeeded && route) await api(route + '/stop', {}).catch(() => {}); }
} finally {
if (!succeeded && route) await api(route + '/stop', {}).catch(() => {});
let refreshError=null;
if(route){try{state=await api(route);taskId??=state.execution?.task?.id;}catch(error){refreshError=error.message;}}
const observation=observedTaskModels(state,taskId);
if(refreshError){observation.status='incomplete';observation.model=null;observation.issues.push({kind:'final_state_refresh_failed',message:refreshError});}
result.model=observation.model;result.modelObservation=observation;result.plannedModel=bootstrap.config.model;
if(state?.execution?.task?.id===taskId)result.usage=state.execution.task.usage;
}
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n');
fs.writeFileSync(path.resolve('docs', 'live-' + kind + '-engineering-validation.json'), JSON.stringify(result, null, 2) + '\n');
console.log(JSON.stringify({ success: result.success, kind, root, taskId, commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));
console.log(JSON.stringify({ success: result.success, kind, root, taskId, model:result.model,modelObservation:result.modelObservation.status,
commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));
+61
View File
@@ -0,0 +1,61 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {createHash} from 'node:crypto';
import {execFileSync} from 'node:child_process';
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
import {observedTaskModels} from './live-model-observation.mjs';
// Offline only: no Provider, Runtime, credentials, model request or formal data.
const root=path.resolve('docs/live-budget-c-evidence'),sha=value=>createHash('sha256').update(value).digest('hex');
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
const raw=fs.readFileSync(inside('manifest.json')),manifest=JSON.parse(raw),read=relative=>JSON.parse(fs.readFileSync(inside(relative),'utf8'));
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.equal(manifest.formalServiceChanged,false);
const entries=new Map();let bytes=0;
for(const entry of manifest.entries){assert.ok(!entries.has(entry.path));entries.set(entry.path,entry);const data=fs.readFileSync(inside(entry.path));
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;}
assert.equal(entries.size,manifest.count);assert.equal(bytes,manifest.bytes);
for(const input of manifest.sourceInputs){
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$|^package(?:-lock)?\.json$/);assert.ok(!input.path.split('/').includes('..'));
const blob=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
const data=input.snapshot?fs.readFileSync(inside(input.snapshot)):blob;
assert.equal(sha(blob),input.gitBlobSha256);assert.equal(sha(data),input.sha256);assert.equal(data.length,input.bytes);
assert.equal(data.toString('utf8').replaceAll('\r\n','\n'),blob.toString('utf8').replaceAll('\r\n','\n'));
}
const receipt=name=>fs.readFileSync(inside(manifest.receipts.find(value=>value.name===name).file),'utf8');
assert.match(receipt('final-related'),/tests 149\b/);assert.match(receipt('final-related'),/pass 149\b/);assert.match(receipt('final-related'),/fail 0\b/);
assert.match(receipt('final-build'),/Server bundle built\./);
const ui=read(manifest.ui.result),session=read(manifest.ui.session);
assert.equal(ui.passed,true);assert.equal(ui.cases.length,2);assert.equal(ui.ownedHttpRequests,1);assert.deepEqual(ui.errors,[]);
assert.equal(session.status,'paused');assert.equal(session.execution.run.stopReason,'token_limit');
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.reportedTokens,55017);
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.exceededTokens,20017);
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.unknownRequests,0);
assert.equal(session.execution.run.budgetPause.modelFailure.httpStatus,503);
assert.equal(observedTaskModels(session).model,'owned-budget-ui');
let originalRequests=0,artifactRecords=0;
for(const sample of manifest.samples){
const result=read(sample.result),records=read(sample.records),sessions=new Map(read(sample.sessions).map(value=>[value.id,value])),core={};
for(const row of records){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
artifactRecords+=core.artifact.size;const artifacts=new Map(sample.artifacts.map(value=>[value.id,value]));
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId);return value;},
artifactStore:{read(artifact){const data=fs.readFileSync(inside(artifacts.get(artifact.id).file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
assert.equal(result.success,sample.passed);assert.equal(result.realModelRequests,0);
if(!sample.passed){assert.ok(result.error);continue;}
assert.equal(result.metrics.contextLimit,16000);assert.equal(result.parentCheck.results[0].tests.passed,9);
assert.ok(result.checks.some(job=>job.status==='failed'&&job.results[0].tests.failed===9));assert.equal(result.writable.length,3);
for(const job of core.engine_run.values()){
const owner=sessions.get(job.sessionId);originalEngineRun(store,owner,job);
assert.equal(job.status,'completed');assert.equal(job.native.cleanup.treeConfirmed,true);assert.equal(job.native.cleanup.rootExited,true);
for(const request of job.requests){
verifyEngineModelInput(store,owner,job,request);assert.equal(request.reservationVersion,1);originalRequests++;
assert.ok(request.tokenReservation.routes.every(route=>route.inputRatio>=1));
}
}
}
assert.deepEqual(manifest.samples.filter(value=>value.passed).map(value=>value.name).sort(),['chat','responses']);
for(const name of ['a-offline','b-offline','pilot-offline'])assert.equal(JSON.parse(receipt(name)).passed,true);
console.log(JSON.stringify({passed:true,sourceCommit:manifest.sourceCommit,manifestSha256:sha(raw),files:entries.size,bytes,
sourceInputs:manifest.sourceInputs.length,artifactRecords,originalRequests,realModelRequests:0,
scope:'Offline raw bytes/source, known usage reservation, actual Task model identity and owned native engineering evidence; original failures retained, real model and original Purge not measured.'}));
+56
View File
@@ -0,0 +1,56 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {randomUUID} from 'node:crypto';
import {chromium,expect} from 'playwright/test';
import {fixture,compatibleFixture,launchWorkbench,until} from '../tests/helpers.mjs';
import {observedTaskModels} from './live-model-observation.mjs';
const directory=path.resolve('.qa','model-budget-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),screenshots=[],cases=[],errors=[];
let host,browser,report,state;
try{
const model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'Owned UI budget failure'},
usage:{prompt_tokens:55000,completion_tokens:17,total_tokens:55017}}}));
f.store.close();host=await launchWorkbench(t,f.data);assert.notEqual(new URL(host.url).port,'4318');
await host.api('/config',{...host.bootstrap.config,contextLimit:16000,maxOutputTokens:512,maxRunTokens:35000,maxTaskTokens:1000000});
const connection=await host.api('/model-center/connections',{name:'自有预算验收模型',model:'owned-budget-ui',baseUrl:model.baseUrl,requiresKey:false});
await host.api('/model-center/select',{type:'connection',id:connection.id});
const route='/sessions/'+f.session.id;
await host.api(route+'/messages',{content:'读取README,不修改文件,不运行检查。'});
await until(async()=>{state=await host.api(route);return !['running','awaiting_approval','stopping'].includes(state.status);},15000);
assert.equal(state.status,'paused',JSON.stringify({status:state.status,failure:state.failure?.message,stopReason:state.execution.run?.stopReason,usage:state.execution.task?.usage,budget:state.execution.run?.budget}));
assert.equal(state.execution.run.stopReason,'token_limit');assert.equal(model.requests.length,1);
assert.equal(observedTaskModels(state).model,'owned-budget-ui');
browser=await chromium.launch({channel:'msedge',headless:true});
for(const width of [1440,390]){
const page=await browser.newPage({viewport:{width,height:1000}});page.on('pageerror',error=>errors.push(error.message));
await page.goto(host.url);await page.waitForLoadState('networkidle');
const project=page.getByLabel('当前项目',{exact:true});
if(!await project.isVisible())await page.getByRole('button',{name:'展开项目栏',exact:true}).click();
await project.selectOption(f.project.id);
const panel=page.getByLabel('任务预算诊断',{exact:true});await expect(panel).toBeVisible();
for(const text of ['本次运行上游报告用量','55,017','35,000','20,017','用量未知请求 / 本地预留','0 / 0','单个在途响应可能超过预估'])await expect(panel).toContainText(text);
await panel.getByRole('button',{name:'核对预算并续接原任务',exact:true}).click();
const dialog=page.getByRole('dialog',{name:'续接原任务',exact:true});
await dialog.getByLabel('新Run累计tokens',{exact:true}).fill('100000');
await dialog.getByRole('button',{name:'准备续接预览',exact:true}).click();await expect(dialog.getByRole('status')).toContainText('将创建原Task的新Run');
assert.equal(model.requests.length,1,'A budget preview cannot dispatch another model request.');
const box=await dialog.boundingBox();assert.ok(box.x>=0&&box.x+box.width<=width+1);
await dialog.getByRole('button',{name:'关闭窗口',exact:true}).click();await page.reload();await expect(panel).toContainText('20,017');
const current=await host.api(route);assert.equal(current.execution.task.id,state.execution.task.id);assert.equal(current.execution.task.usage.requests,1);
assert.equal(current.execution.run.consumption.length,1);assert.equal(model.requests.length,1);
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+1),'No viewport overflow');
await panel.scrollIntoViewIfNeeded();const screenshot=path.join(directory,'reported-overrun-'+width+'.png');
await page.screenshot({path:screenshot,fullPage:true});screenshots.push(screenshot);
fs.writeFileSync(path.join(directory,'reported-overrun-'+width+'.html'),await page.content());
cases.push(width+': actual reported overrun persists, zero unknown use stays distinct, and preview does not dispatch or replace the original Task');await page.close();
}
assert.deepEqual(errors,[]);fs.writeFileSync(path.join(directory,'session.json'),JSON.stringify(await host.api(route),null,2));
report={passed:true,cases,screenshots,errors,realModelRequests:0,ownedHttpRequests:model.requests.length,
modelObservation:observedTaskModels(state),scope:'Actual compiled host and owned HTTP reported overrun, persistence and preview UI; real-model autonomy/Purge benefit remain separate.'};
}catch(error){if(state)fs.writeFileSync(path.join(directory,'failed-state.json'),JSON.stringify(state,null,2));report={passed:false,cases,screenshots,errors,error:error.stack,realModelRequests:0};process.exitCode=1;}
finally{
await browser?.close();await host?.close();for(const dispose of disposers.reverse())await dispose();
fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));console.log(JSON.stringify({...report,directory}));
}