fix: bind observed models and calibrate every model dispatch budget
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
// This reports observed Task consumption, never the model selected in bootstrap.
|
||||
// Public execution views may contain only the most recent Runs; gaps stay explicit.
|
||||
export function observedTaskModels(state,taskId=state?.execution?.task?.id){
|
||||
const execution=state?.execution??{},runs=new Map(),attempts=new Map(),issues=[];
|
||||
for(const run of [...(execution.runs??[]),execution.run].filter(Boolean)){
|
||||
if(taskId&&run.taskId===taskId&&typeof run.id==='string')runs.set(run.id,run);
|
||||
}
|
||||
const task=execution.task?.id===taskId?execution.task:null;
|
||||
const validCount=value=>Number.isSafeInteger(value)&&value>=0;
|
||||
let declared=task&&validCount(task.usage?.requests)?task.usage.requests:null;
|
||||
if(declared===null&&runs.size&&[...runs.values()].every(run=>validCount(run.usage?.requests)))
|
||||
declared=[...runs.values()].reduce((total,run)=>total+run.usage.requests,0);
|
||||
for(const run of runs.values())for(const receipt of run.consumption??[]){
|
||||
if(typeof receipt.attemptId!=='string'||!receipt.attemptId){issues.push({kind:'missing_attempt_id',runId:run.id});continue;}
|
||||
const attempt={runId:run.id,attemptId:receipt.attemptId,requestId:receipt.requestId??null,
|
||||
model:typeof receipt.model==='string'&&receipt.model.trim()?receipt.model:null,
|
||||
protocol:typeof receipt.protocol==='string'&&receipt.protocol.trim()?receipt.protocol:null,
|
||||
connectionId:receipt.connectionId??null,outcome:receipt.outcome??null,
|
||||
usageStatus:receipt.usageStatus??'unknown',usage:receipt.usage??null};
|
||||
const previous=attempts.get(attempt.attemptId);
|
||||
if(previous){
|
||||
if(JSON.stringify(previous)!==JSON.stringify(attempt))issues.push({kind:'conflicting_attempt',attemptId:attempt.attemptId,runIds:[previous.runId,run.id]});
|
||||
continue;
|
||||
}
|
||||
attempts.set(attempt.attemptId,attempt);
|
||||
if(!attempt.model||!attempt.protocol)issues.push({kind:'missing_route_identity',attemptId:attempt.attemptId});
|
||||
if(attempt.usageStatus==='reported'&&!attempt.usage)issues.push({kind:'missing_reported_usage',attemptId:attempt.attemptId});
|
||||
}
|
||||
const observed=[...attempts.values()],models=[...new Set(observed.map(value=>value.model).filter(Boolean))],routes=new Map();
|
||||
for(const attempt of observed){
|
||||
const identity={connectionId:attempt.connectionId,model:attempt.model,protocol:attempt.protocol},key=JSON.stringify(identity);
|
||||
if(!routes.has(key))routes.set(key,{...identity,requests:0,reportedUsageRequests:0,unknownUsageRequests:0});
|
||||
const route=routes.get(key);route.requests++;
|
||||
if(attempt.usageStatus==='reported'&&attempt.usage)route.reportedUsageRequests++;else route.unknownUsageRequests++;
|
||||
}
|
||||
if(!taskId)issues.push({kind:'missing_task_identity'});
|
||||
if(declared!==null&&declared!==observed.length)issues.push({kind:'request_coverage_gap',recordedRequests:declared,observedAttempts:observed.length});
|
||||
const status=issues.length?'incomplete':!observed.length?declared===0?'not_observed':'unavailable':models.length>1?'mixed':'observed';
|
||||
return {schemaVersion:1,source:'task_run_consumption',taskId:taskId??null,status,
|
||||
model:['observed','mixed'].includes(status)&&models.length===1?models[0]:null,models,
|
||||
observedAttempts:observed.length,recordedRequests:declared,routes:[...routes.values()],attempts:observed,issues};
|
||||
}
|
||||
@@ -52,7 +52,7 @@ const frozen = inputs(), mark = message => { cases.push(message); console.log('P
|
||||
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
|
||||
const startedAt = Date.now(), label = sampleName;
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
|
||||
fs.mkdirSync(sample); let runtime, group, result, desktopQa, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
|
||||
fs.mkdirSync(sample); let runtime, group, result, desktopQa, model, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
|
||||
try {
|
||||
const baseline = contextMode ? 'docs/live-model-purge-1.24-evidence/source-baselines/r1-off' : kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
|
||||
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
@@ -74,7 +74,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
};
|
||||
if (kind === 'react') install(f.projectRoot, 'parent');
|
||||
let phase = 'new';
|
||||
const model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
|
||||
model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
|
||||
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
|
||||
assertToolPairs(input.messages);
|
||||
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
|
||||
@@ -237,7 +237,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
|
||||
writable, reviews, modelRequests, ...(desktopQa ? { desktop: { events: desktopQa.events, screenshots: desktopQa.screenshots, scope: desktopQa.scope, errors: desktopQa.errors } } : {}), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
||||
return result;
|
||||
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, ownedFixtureErrors:model?.errors??[], engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
await desktopQa?.close();
|
||||
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
|
||||
@@ -10,6 +10,7 @@ import { createPythonDeliveryProject, pythonProtected } from '../tests/fixtures/
|
||||
import { deliveryGit, writeDeliveryFile } from '../tests/fixtures/delivery-project.mjs';
|
||||
import { captureWorkspaceSource, assertWorkspaceSource } from '../tests/fixtures/workspace-project.mjs';
|
||||
import { verifyLiveReactClient } from './live-react-client.mjs';
|
||||
import { observedTaskModels } from './live-model-observation.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-live-engineering.mjs --case react|python [--url http://127.0.0.1:4318] — fresh owned Git source/workspace, configured model and bounded review/check/delivery acceptance'); process.exit(0);
|
||||
@@ -149,7 +150,7 @@ try {
|
||||
fs.writeFileSync(path.join(root, 'delivery-report.json'), reportText); fs.writeFileSync(path.join(root, 'delivery-report.md'), markdown);
|
||||
const browser = kind === 'react' ? await verifyLiveReactClient(childProject.root, path.join(root, 'screenshots')) : null;
|
||||
fs.writeFileSync(path.join(root, 'session.json'), JSON.stringify(await api(route + '/export'), null, 2) + '\n');
|
||||
result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, model: bootstrap.config.model, root,
|
||||
result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, root,
|
||||
taskId, sessionId: childSession.id, projectId: childProject.id, sourceProjectId: parentProject.id, workspaceId: workspace.id, workspaceRoot: childProject.root,
|
||||
sourceCommit: workspace.source.head, commit: completedDelivery.commit, tests, checks: passing.results, usage: state.execution.task.usage,
|
||||
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), applications, decisions, continuations, browser,
|
||||
@@ -158,11 +159,20 @@ try {
|
||||
credentialsHandling: 'The original Provider executes requests in its own process; no model key is read, copied or written.' };
|
||||
succeeded = true;
|
||||
} catch (error) {
|
||||
result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root, model: bootstrap.config.model,
|
||||
result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root,
|
||||
taskId, sessionId: childSession?.id, projectId: childProject?.id, workspaceId: workspace?.id, error: error.stack,
|
||||
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), usage: state?.execution?.task?.usage, state: state?.status,
|
||||
stopReason: state?.execution?.run?.stopReason, applications, decisions, continuations, checks: state?.execution?.checks }; process.exitCode = 1;
|
||||
} finally { if (!succeeded && route) await api(route + '/stop', {}).catch(() => {}); }
|
||||
} finally {
|
||||
if (!succeeded && route) await api(route + '/stop', {}).catch(() => {});
|
||||
let refreshError=null;
|
||||
if(route){try{state=await api(route);taskId??=state.execution?.task?.id;}catch(error){refreshError=error.message;}}
|
||||
const observation=observedTaskModels(state,taskId);
|
||||
if(refreshError){observation.status='incomplete';observation.model=null;observation.issues.push({kind:'final_state_refresh_failed',message:refreshError});}
|
||||
result.model=observation.model;result.modelObservation=observation;result.plannedModel=bootstrap.config.model;
|
||||
if(state?.execution?.task?.id===taskId)result.usage=state.execution.task.usage;
|
||||
}
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(path.resolve('docs', 'live-' + kind + '-engineering-validation.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, kind, root, taskId, commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));
|
||||
console.log(JSON.stringify({ success: result.success, kind, root, taskId, model:result.model,modelObservation:result.modelObservation.status,
|
||||
commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {execFileSync} from 'node:child_process';
|
||||
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
|
||||
import {observedTaskModels} from './live-model-observation.mjs';
|
||||
|
||||
// Offline only: no Provider, Runtime, credentials, model request or formal data.
|
||||
const root=path.resolve('docs/live-budget-c-evidence'),sha=value=>createHash('sha256').update(value).digest('hex');
|
||||
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
|
||||
const raw=fs.readFileSync(inside('manifest.json')),manifest=JSON.parse(raw),read=relative=>JSON.parse(fs.readFileSync(inside(relative),'utf8'));
|
||||
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.equal(manifest.formalServiceChanged,false);
|
||||
const entries=new Map();let bytes=0;
|
||||
for(const entry of manifest.entries){assert.ok(!entries.has(entry.path));entries.set(entry.path,entry);const data=fs.readFileSync(inside(entry.path));
|
||||
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;}
|
||||
assert.equal(entries.size,manifest.count);assert.equal(bytes,manifest.bytes);
|
||||
for(const input of manifest.sourceInputs){
|
||||
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$|^package(?:-lock)?\.json$/);assert.ok(!input.path.split('/').includes('..'));
|
||||
const blob=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
|
||||
const data=input.snapshot?fs.readFileSync(inside(input.snapshot)):blob;
|
||||
assert.equal(sha(blob),input.gitBlobSha256);assert.equal(sha(data),input.sha256);assert.equal(data.length,input.bytes);
|
||||
assert.equal(data.toString('utf8').replaceAll('\r\n','\n'),blob.toString('utf8').replaceAll('\r\n','\n'));
|
||||
}
|
||||
const receipt=name=>fs.readFileSync(inside(manifest.receipts.find(value=>value.name===name).file),'utf8');
|
||||
assert.match(receipt('final-related'),/tests 149\b/);assert.match(receipt('final-related'),/pass 149\b/);assert.match(receipt('final-related'),/fail 0\b/);
|
||||
assert.match(receipt('final-build'),/Server bundle built\./);
|
||||
const ui=read(manifest.ui.result),session=read(manifest.ui.session);
|
||||
assert.equal(ui.passed,true);assert.equal(ui.cases.length,2);assert.equal(ui.ownedHttpRequests,1);assert.deepEqual(ui.errors,[]);
|
||||
assert.equal(session.status,'paused');assert.equal(session.execution.run.stopReason,'token_limit');
|
||||
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.reportedTokens,55017);
|
||||
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.exceededTokens,20017);
|
||||
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.unknownRequests,0);
|
||||
assert.equal(session.execution.run.budgetPause.modelFailure.httpStatus,503);
|
||||
assert.equal(observedTaskModels(session).model,'owned-budget-ui');
|
||||
let originalRequests=0,artifactRecords=0;
|
||||
for(const sample of manifest.samples){
|
||||
const result=read(sample.result),records=read(sample.records),sessions=new Map(read(sample.sessions).map(value=>[value.id,value])),core={};
|
||||
for(const row of records){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
|
||||
artifactRecords+=core.artifact.size;const artifacts=new Map(sample.artifacts.map(value=>[value.id,value]));
|
||||
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId);return value;},
|
||||
artifactStore:{read(artifact){const data=fs.readFileSync(inside(artifacts.get(artifact.id).file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
|
||||
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
|
||||
assert.equal(result.success,sample.passed);assert.equal(result.realModelRequests,0);
|
||||
if(!sample.passed){assert.ok(result.error);continue;}
|
||||
assert.equal(result.metrics.contextLimit,16000);assert.equal(result.parentCheck.results[0].tests.passed,9);
|
||||
assert.ok(result.checks.some(job=>job.status==='failed'&&job.results[0].tests.failed===9));assert.equal(result.writable.length,3);
|
||||
for(const job of core.engine_run.values()){
|
||||
const owner=sessions.get(job.sessionId);originalEngineRun(store,owner,job);
|
||||
assert.equal(job.status,'completed');assert.equal(job.native.cleanup.treeConfirmed,true);assert.equal(job.native.cleanup.rootExited,true);
|
||||
for(const request of job.requests){
|
||||
verifyEngineModelInput(store,owner,job,request);assert.equal(request.reservationVersion,1);originalRequests++;
|
||||
assert.ok(request.tokenReservation.routes.every(route=>route.inputRatio>=1));
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.deepEqual(manifest.samples.filter(value=>value.passed).map(value=>value.name).sort(),['chat','responses']);
|
||||
for(const name of ['a-offline','b-offline','pilot-offline'])assert.equal(JSON.parse(receipt(name)).passed,true);
|
||||
console.log(JSON.stringify({passed:true,sourceCommit:manifest.sourceCommit,manifestSha256:sha(raw),files:entries.size,bytes,
|
||||
sourceInputs:manifest.sourceInputs.length,artifactRecords,originalRequests,realModelRequests:0,
|
||||
scope:'Offline raw bytes/source, known usage reservation, actual Task model identity and owned native engineering evidence; original failures retained, real model and original Purge not measured.'}));
|
||||
@@ -0,0 +1,56 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {randomUUID} from 'node:crypto';
|
||||
import {chromium,expect} from 'playwright/test';
|
||||
import {fixture,compatibleFixture,launchWorkbench,until} from '../tests/helpers.mjs';
|
||||
import {observedTaskModels} from './live-model-observation.mjs';
|
||||
|
||||
const directory=path.resolve('.qa','model-budget-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),screenshots=[],cases=[],errors=[];
|
||||
let host,browser,report,state;
|
||||
try{
|
||||
const model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'Owned UI budget failure'},
|
||||
usage:{prompt_tokens:55000,completion_tokens:17,total_tokens:55017}}}));
|
||||
f.store.close();host=await launchWorkbench(t,f.data);assert.notEqual(new URL(host.url).port,'4318');
|
||||
await host.api('/config',{...host.bootstrap.config,contextLimit:16000,maxOutputTokens:512,maxRunTokens:35000,maxTaskTokens:1000000});
|
||||
const connection=await host.api('/model-center/connections',{name:'自有预算验收模型',model:'owned-budget-ui',baseUrl:model.baseUrl,requiresKey:false});
|
||||
await host.api('/model-center/select',{type:'connection',id:connection.id});
|
||||
const route='/sessions/'+f.session.id;
|
||||
await host.api(route+'/messages',{content:'读取README,不修改文件,不运行检查。'});
|
||||
await until(async()=>{state=await host.api(route);return !['running','awaiting_approval','stopping'].includes(state.status);},15000);
|
||||
assert.equal(state.status,'paused',JSON.stringify({status:state.status,failure:state.failure?.message,stopReason:state.execution.run?.stopReason,usage:state.execution.task?.usage,budget:state.execution.run?.budget}));
|
||||
assert.equal(state.execution.run.stopReason,'token_limit');assert.equal(model.requests.length,1);
|
||||
assert.equal(observedTaskModels(state).model,'owned-budget-ui');
|
||||
browser=await chromium.launch({channel:'msedge',headless:true});
|
||||
for(const width of [1440,390]){
|
||||
const page=await browser.newPage({viewport:{width,height:1000}});page.on('pageerror',error=>errors.push(error.message));
|
||||
await page.goto(host.url);await page.waitForLoadState('networkidle');
|
||||
const project=page.getByLabel('当前项目',{exact:true});
|
||||
if(!await project.isVisible())await page.getByRole('button',{name:'展开项目栏',exact:true}).click();
|
||||
await project.selectOption(f.project.id);
|
||||
const panel=page.getByLabel('任务预算诊断',{exact:true});await expect(panel).toBeVisible();
|
||||
for(const text of ['本次运行上游报告用量','55,017','35,000','20,017','用量未知请求 / 本地预留','0 / 0','单个在途响应可能超过预估'])await expect(panel).toContainText(text);
|
||||
await panel.getByRole('button',{name:'核对预算并续接原任务',exact:true}).click();
|
||||
const dialog=page.getByRole('dialog',{name:'续接原任务',exact:true});
|
||||
await dialog.getByLabel('新Run累计tokens',{exact:true}).fill('100000');
|
||||
await dialog.getByRole('button',{name:'准备续接预览',exact:true}).click();await expect(dialog.getByRole('status')).toContainText('将创建原Task的新Run');
|
||||
assert.equal(model.requests.length,1,'A budget preview cannot dispatch another model request.');
|
||||
const box=await dialog.boundingBox();assert.ok(box.x>=0&&box.x+box.width<=width+1);
|
||||
await dialog.getByRole('button',{name:'关闭窗口',exact:true}).click();await page.reload();await expect(panel).toContainText('20,017');
|
||||
const current=await host.api(route);assert.equal(current.execution.task.id,state.execution.task.id);assert.equal(current.execution.task.usage.requests,1);
|
||||
assert.equal(current.execution.run.consumption.length,1);assert.equal(model.requests.length,1);
|
||||
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+1),'No viewport overflow');
|
||||
await panel.scrollIntoViewIfNeeded();const screenshot=path.join(directory,'reported-overrun-'+width+'.png');
|
||||
await page.screenshot({path:screenshot,fullPage:true});screenshots.push(screenshot);
|
||||
fs.writeFileSync(path.join(directory,'reported-overrun-'+width+'.html'),await page.content());
|
||||
cases.push(width+': actual reported overrun persists, zero unknown use stays distinct, and preview does not dispatch or replace the original Task');await page.close();
|
||||
}
|
||||
assert.deepEqual(errors,[]);fs.writeFileSync(path.join(directory,'session.json'),JSON.stringify(await host.api(route),null,2));
|
||||
report={passed:true,cases,screenshots,errors,realModelRequests:0,ownedHttpRequests:model.requests.length,
|
||||
modelObservation:observedTaskModels(state),scope:'Actual compiled host and owned HTTP reported overrun, persistence and preview UI; real-model autonomy/Purge benefit remain separate.'};
|
||||
}catch(error){if(state)fs.writeFileSync(path.join(directory,'failed-state.json'),JSON.stringify(state,null,2));report={passed:false,cases,screenshots,errors,error:error.stack,realModelRequests:0};process.exitCode=1;}
|
||||
finally{
|
||||
await browser?.close();await host?.close();for(const dispose of disposers.reverse())await dispose();
|
||||
fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));console.log(JSON.stringify({...report,directory}));
|
||||
}
|
||||
Reference in New Issue
Block a user