From 5801f9cabbdc2ab6f18428fc6fe5c90e3ce1eeee Mon Sep 17 00:00:00 2001 From: Agent Workbench Dev Date: Wed, 4 Nov 2026 16:59:47 +0800 Subject: [PATCH] fix(budget): set aside a single isolated input-ratio outlier in dispatch reservation observed-route-max-v2 excludes at most one receipt whose reported/local input ratio is >=1.5x every other receipt on the route, with >=3 receipts and a later normal one. Excluded receipts stay in the ledger and are listed in excludedSources with their position; repeated highs, the latest receipt and short histories are never excluded. Sealed v1 reservations still verify byte-for-byte. Regression replays the original R evidence: v1 29623 > remaining 17429, v2 15285. --- src/shared/contracts/model-reservation.mjs | 104 +++++++++++++++------ tests/model-reservation.test.mjs | 49 +++++++++- 2 files changed, 122 insertions(+), 31 deletions(-) diff --git a/src/shared/contracts/model-reservation.mjs b/src/shared/contracts/model-reservation.mjs index 6704e979..999697dd 100644 --- a/src/shared/contracts/model-reservation.mjs +++ b/src/shared/contracts/model-reservation.mjs @@ -34,49 +34,93 @@ export function checkModelDispatchBudget(store,session,run,config,body,overheadT } } -// A conservative reservation for the next dispatch, not corrected historical -// billing or a promise about an unseen upstream response. Never lowers estimates. -export function modelReservation(config,body,receipts=[],overheadTokens=0){ +export const RESERVATION_METHOD='observed-route-max-v2'; +export const ISOLATED_OUTLIER_FACTOR=1.5,ISOLATED_OUTLIER_MIN_RECEIPTS=3; +const members=config=>config.modelRouting?.members??[{id:null,model:config.model,protocol:config.protocol??'chat-completions',baseUrl:config.baseUrl}]; +function routeSources(member,receipts){ + const seen=new Set(),sources=[]; + for(const receipt of receipts){ + const estimate=receipt.localEstimate,usage=receipt.usage; + if(receipt.model!==member.model||(receipt.protocol??'chat-completions')!==(member.protocol??'chat-completions') + ||(receipt.connectionId??null)!==(member.id??null)||endpoint(receipt.endpoint)!==endpoint(member.baseUrl) + ||receipt.usageStatus!=='reported'||!usage||estimate?.schemaVersion!==1||!count(estimate.inputTokens)||estimate.inputTokens<1 + ||!receipt.attemptId||seen.has(receipt.attemptId))continue; + if(!count(usage.total_tokens)&&!(count(usage.prompt_tokens)&&count(usage.completion_tokens)))continue; + seen.add(receipt.attemptId); + const total=count(usage.total_tokens)?usage.total_tokens:usage.prompt_tokens+usage.completion_tokens; + sources.push({runId:receipt.runId,attemptId:receipt.attemptId,localInputTokens:estimate.inputTokens, + reportedInputTokens:count(usage.prompt_tokens)?usage.prompt_tokens:null,reportedOutputTokens:count(usage.completion_tokens)?usage.completion_tokens:null,totalTokens:total}); + } + return sources; +} +function calibrate(local,sources,overheadTokens){ + let inputRatio=1,observedOutputTokens=local.outputTokens,observedTotalFloor=0; + for(const source of sources){ + if(source.reportedInputTokens!==null)inputRatio=Math.max(inputRatio,source.reportedInputTokens/source.localInputTokens); + if(source.reportedOutputTokens!==null)observedOutputTokens=Math.max(observedOutputTokens,source.reportedOutputTokens); + observedTotalFloor=Math.max(observedTotalFloor,source.totalTokens); + } + return {inputRatio,observedOutputTokens,observedTotalFloor, + reservedTokens:bounded(Math.max(local.inputTokens*inputRatio+observedOutputTokens,observedTotalFloor)+overheadTokens)}; +} +const ratioOf=source=>source.reportedInputTokens===null?null:source.reportedInputTokens/source.localInputTokens; +// One receipt may be set aside only when it is the single input-ratio outlier, +// at least ISOLATED_OUTLIER_FACTOR above every other receipt, with at least one +// later normal receipt on the same route. A repeated high ratio is never excluded. +export function isolatedOutlier(sources){ + if(sources.lengthvalue===null))return null; + const top=ratios.indexOf(Math.max(...ratios)),rest=ratios.filter((_,index)=>index!==top),next=Math.max(1,...rest); + if(ratios[top]{ - const wire=actualRequest(body,member),local=localModelEstimate(wire),seen=new Set(),sources=[]; - let inputRatio=1,observedOutputTokens=local.outputTokens,observedTotalFloor=0; + const routes=members(config).map(member=>{ + const local=localModelEstimate(actualRequest(body,member)); if(!count(local.inputTokens)||!count(local.outputTokens))throw new Error('模型请求输入或输出预留无效。'); - for(const receipt of receipts){ - const estimate=receipt.localEstimate,usage=receipt.usage; - if(receipt.model!==member.model||(receipt.protocol??'chat-completions')!==(member.protocol??'chat-completions') - ||(receipt.connectionId??null)!==(member.id??null)||endpoint(receipt.endpoint)!==endpoint(member.baseUrl) - ||receipt.usageStatus!=='reported'||!usage||estimate?.schemaVersion!==1||!count(estimate.inputTokens)||estimate.inputTokens<1 - ||!receipt.attemptId||seen.has(receipt.attemptId))continue; - if(!count(usage.total_tokens)&&!(count(usage.prompt_tokens)&&count(usage.completion_tokens)))continue; - seen.add(receipt.attemptId); - if(count(usage.prompt_tokens))inputRatio=Math.max(inputRatio,usage.prompt_tokens/estimate.inputTokens); - if(count(usage.completion_tokens))observedOutputTokens=Math.max(observedOutputTokens,usage.completion_tokens); - const total=count(usage.total_tokens)?usage.total_tokens:usage.prompt_tokens+usage.completion_tokens; - observedTotalFloor=Math.max(observedTotalFloor,total); - sources.push({runId:receipt.runId,attemptId:receipt.attemptId,localInputTokens:estimate.inputTokens, - reportedInputTokens:count(usage.prompt_tokens)?usage.prompt_tokens:null,reportedOutputTokens:count(usage.completion_tokens)?usage.completion_tokens:null,totalTokens:total}); - } - const reservedTokens=bounded(Math.max(local.inputTokens*inputRatio+observedOutputTokens,observedTotalFloor)+overheadTokens); - return {connectionId:member.id??null,model:member.model,protocol:member.protocol??'chat-completions',endpoint:member.baseUrl, - localInputTokens:local.inputTokens,outputReservation:local.outputTokens,inputRatio,observedOutputTokens,observedTotalFloor,reservedTokens,sources}; + const all=routeSources(member,receipts),outlier=method===RESERVATION_METHOD?isolatedOutlier(all):null; + const sources=outlier?all.filter((_,index)=>index!==outlier.index):all; + const route={connectionId:member.id??null,model:member.model,protocol:member.protocol??'chat-completions',endpoint:member.baseUrl, + localInputTokens:local.inputTokens,outputReservation:local.outputTokens,...calibrate(local,sources,overheadTokens),sources}; + if(method===RESERVATION_METHOD)route.excludedSources=outlier?[{...all[outlier.index],sourceIndex:outlier.index,reason:'isolated_input_ratio_outlier', + inputRatio:outlier.ratio,nextHighestRatio:outlier.nextRatio,factor:ISOLATED_OUTLIER_FACTOR}]:[]; + return route; }); if(!routes.length)throw new Error('模型预留没有实际候选线路。'); - return {schemaVersion:1,method:'observed-route-max-v1',overheadTokens,reservedTokens:Math.max(...routes.map(route=>route.reservedTokens)),routes}; + return {schemaVersion:1,method,overheadTokens,reservedTokens:Math.max(...routes.map(route=>route.reservedTokens)),routes}; +} + +// A conservative reservation for the next dispatch, not corrected historical +// billing or a promise about an unseen upstream response. It never lowers the +// local estimate; an excluded outlier stays in the ledger and in excludedSources. +export function modelReservation(config,body,receipts=[],overheadTokens=0){ + return reservationRoutes(config,body,receipts,overheadTokens,RESERVATION_METHOD); +} +// Original v1 calibration, kept only so sealed historical reservations verify. +export function legacyModelReservation(config,body,receipts=[],overheadTokens=0){ + return reservationRoutes(config,body,receipts,overheadTokens,'observed-route-max-v1'); } export function verifyModelReservation(store,session,taskId,config,body,reservation){ - if(reservation?.schemaVersion!==1||reservation.method!=='observed-route-max-v1'||!Array.isArray(reservation.routes))throw new Error('模型用量预留版本或线路缺失。'); - const receipts=new Map(); - for(const route of reservation.routes)for(const source of route.sources??[]){ + const compute=reservation?.method===RESERVATION_METHOD?modelReservation:reservation?.method==='observed-route-max-v1'?legacyModelReservation:null; + if(reservation?.schemaVersion!==1||!compute||!Array.isArray(reservation.routes))throw new Error('模型用量预留版本或线路缺失。'); + const receipts=new Map(),ordered=route=>{ + const list=[...(route.sources??[])]; + for(const value of route.excludedSources??[]){ + if(!Number.isSafeInteger(value.sourceIndex)||value.sourceIndex<0||value.sourceIndex>list.length)throw new Error('模型预算预留的排除来源位置无效。'); + list.splice(value.sourceIndex,0,value); + } + return list; + }; + for(const route of reservation.routes)for(const source of ordered(route)){ const run=store.owned('run',source.runId,session.id); if(run.taskId!==taskId)throw new Error('模型预算校准来源不属于当前Task。'); const receipt=run.consumption?.find(value=>value.attemptId===source.attemptId); if(!receipt)throw new Error('模型预算校准缺少原消费记录。'); receipts.set(source.attemptId,{...receipt,runId:run.id}); } - const expected=modelReservation(config,body,[...receipts.values()],reservation.overheadTokens); + const expected=compute(config,body,[...receipts.values()],reservation.overheadTokens); if(JSON.stringify(expected)!==JSON.stringify(reservation))throw new Error('模型预算预留与原估算/用量/实际线路不一致。'); return reservation; } diff --git a/tests/model-reservation.test.mjs b/tests/model-reservation.test.mjs index a20fe54b..745df216 100644 --- a/tests/model-reservation.test.mjs +++ b/tests/model-reservation.test.mjs @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import {modelReservation,localModelEstimate,verifyModelReservation} from '../src/shared/contracts/model-reservation.mjs'; +import fs from 'node:fs'; +import {modelReservation,legacyModelReservation,localModelEstimate,verifyModelReservation} from '../src/shared/contracts/model-reservation.mjs'; import {completionBody} from '../src/shared/contracts/model-request.mjs'; import {Provider} from '../src/server/provider.mjs'; import {Runtime} from '../src/server/runtime.mjs'; @@ -112,3 +113,49 @@ test('native bridge seals calibrated refusal with original consumption and prese assert.equal(refusal.tokenReservation.routes[0].sources[0].attemptId,run.consumption[0].attemptId); assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.requests,1);assert.equal(run.usage.unknownRequests,0); }); + +// Original R evidence: six reported receipts, the fourth reports 27164 input for a +// 12168 local estimate while a same-shape fifth reports 8619. The sealed v1 pause +// must still verify byte-for-byte; v2 sets aside only that single isolated outlier. +const rDir=new URL('../docs/live-delivery-r-evidence/run/',import.meta.url); +const rJson=name=>JSON.parse(fs.readFileSync(new URL(name,rDir),'utf8')); +function rEvidence(){ + const proof=rJson('analysis/budget-proof.json'),route=proof.pause.tokenReservation.routes[0]; + const find=value=>{if(value&&typeof value==='object'){if(Array.isArray(value.consumption)&&value.consumption.length===6)return value.consumption; + for(const child of Object.values(value)){const found=find(child);if(found)return found;}}return null;}; + const receipts=find(rJson('node/session.json')).map(receipt=>({...receipt,runId:proof.pause.runId})); + const config={mode:'compatible',model:route.model,protocol:route.protocol,baseUrl:route.endpoint,maxOutputTokens:2048,contextLimit:16000, + modelRouting:{members:[{id:route.connectionId,model:route.model,protocol:route.protocol,baseUrl:route.endpoint}]}}; + return {proof,route,receipts,config,body:rJson('node/captures/704d88f7-fbef-4507-9597-22cf79481961-request.json')}; +} +const rStore=(proof,receipts)=>{const run={id:proof.pause.runId,taskId:proof.pause.taskId,sessionId:'r-session',consumption:receipts}; + return {session:{id:'r-session'},store:{owned(kind,id,sessionId){assert.equal(id,run.id);assert.equal(sessionId,run.sessionId);return run;}}};}; + +test('original R pause stays verifiable under v1 while v2 excludes only the single isolated outlier and keeps the ledger intact',()=>{ + const {proof,route,receipts,config,body}=rEvidence(),original=structuredClone(receipts),{store,session}=rStore(proof,receipts); + const legacy=legacyModelReservation(config,body,receipts); + assert.equal(legacy.reservedTokens,29623);assert.deepEqual(legacy.routes[0],route); + assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,proof.pause.tokenReservation),proof.pause.tokenReservation); + const current=modelReservation(config,body,receipts); + assert.equal(current.method,'observed-route-max-v2');assert.equal(current.reservedTokens,15285); + assert.ok(current.reservedTokens<=proof.pause.remainingTokens);assert.ok(current.reservedTokens>=12352+2048); + const [excluded]=current.routes[0].excludedSources;assert.equal(current.routes[0].excludedSources.length,1); + assert.equal(excluded.attemptId,'23226d1c-9ad6-4daa-81f5-1b44f0d57ecc');assert.equal(excluded.reason,'isolated_input_ratio_outlier'); + assert.equal(current.routes[0].sources.length,5);assert.equal(current.routes[0].observedTotalFloor,13107); + assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,current),current); + const forged=structuredClone(current);forged.routes[0].excludedSources=[];forged.routes[0].sources.push(excluded); + assert.throws(()=>verifyModelReservation(store,session,proof.pause.taskId,config,body,forged),/不一致/); + assert.deepEqual(receipts,original);assert.equal(proof.pause.runUsage.totalTokens,72571); +}); + +test('v2 never excludes a repeated high ratio, the latest receipt, or a short history',()=>{ + const {receipts,config,body}=rEvidence(),high=receipts[3]; + const repeated=[...receipts,{...high,attemptId:'r-repeat'}]; + assert.equal(modelReservation(config,body,repeated).routes[0].excludedSources.length,0); + assert.equal(modelReservation(config,body,repeated).reservedTokens,legacyModelReservation(config,body,repeated).reservedTokens); + const latest=receipts.slice(0,4); + assert.equal(modelReservation(config,body,latest).routes[0].excludedSources.length,0); + assert.equal(modelReservation(config,body,latest).reservedTokens,29623); + const short=[receipts[0],receipts[3]]; + assert.equal(modelReservation(config,body,short).routes[0].excludedSources.length,0); +});