Files
Agent-Workbench/tests/model-reservation.test.mjs
2026-11-07 22:50:39 +08:00

192 lines
16 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import {modelReservation,legacyModelReservation,legacyV2ModelReservation,localModelEstimate,verifyModelReservation} from '../src/shared/contracts/model-reservation.mjs';
import {completionBody} from '../src/shared/contracts/model-request.mjs';
import {Provider} from '../src/server/provider.mjs';
import {Runtime} from '../src/server/runtime.mjs';
import {fixture,compatibleFixture,completion,call} from './helpers.mjs';
import {controlledWorker} from './fixtures/controlled-worker.mjs';
import {verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
import {replayFixtureJson} from '../scripts/verify-replay-fixtures.mjs';
const config={mode:'compatible',model:'owned-A',protocol:'chat-completions',baseUrl:'http://127.0.0.1:1/v1',maxOutputTokens:512,contextLimit:16000};
const body=completionBody(config,[{role:'user',content:'Owned bounded input.'}],[]);
const observed=(extra={})=>({attemptId:'owned-attempt',runId:'owned-run',model:config.model,protocol:config.protocol,endpoint:config.baseUrl,
connectionId:null,outcome:'completed',usageStatus:'reported',localEstimate:localModelEstimate(body),
usage:{prompt_tokens:4000,completion_tokens:800,total_tokens:4800},...extra});
test('reservation uses actual known usage from the same route, never lowers estimates or rewrites original receipts',()=>{
const receipts=[observed(),observed({attemptId:'unknown',usageStatus:'unknown',usage:{total_tokens:9999999}}),
observed({attemptId:'other',model:'other-model',usage:{total_tokens:9999999}}),observed({attemptId:'legacy',localEstimate:undefined,usage:{total_tokens:9999999}})];
const original=structuredClone(receipts),reservation=modelReservation(config,body,receipts,128);
assert.equal(reservation.reservedTokens,4928);assert.equal(reservation.routes[0].sources.length,1);
assert.ok(reservation.routes[0].inputRatio>1);assert.deepEqual(receipts,original);
const small=modelReservation(config,body,[observed({usage:{prompt_tokens:1,completion_tokens:1,total_tokens:2}})]);
assert.equal(small.reservedTokens,localModelEstimate(body).inputTokens+512);
const pool={...config,modelRouting:{members:[{id:'A',model:config.model,protocol:config.protocol,baseUrl:config.baseUrl},
{id:'B',model:'owned-B',protocol:config.protocol,baseUrl:'http://127.0.0.1:2/v1'}]}};
const mixed=modelReservation(pool,completionBody(pool,[{role:'user',content:'Owned bounded input.'}],[]),[observed({connectionId:'A'})]);
assert.equal(mixed.routes[0].sources.length,1);assert.equal(mixed.routes[1].sources.length,0);assert.equal(mixed.routes[1].inputRatio,1);
});
test('reservation verification binds original Task/Run consumption and rejects changed usage or foreign provenance',()=>{
const receipt=observed(),run={id:'owned-run',taskId:'owned-task',sessionId:'owned-session',consumption:[receipt]},session={id:'owned-session'};
const store={core:{run:new Map([[run.id,run]])},owned(kind,id,sessionId){assert.equal(kind,'run');assert.equal(id,run.id);assert.equal(sessionId,run.sessionId);return run;}};
const reservation=modelReservation(config,body,[receipt],128);
assert.deepEqual(verifyModelReservation(store,session,'owned-task',config,body,reservation),reservation);
const altered=structuredClone(reservation);altered.routes[0].inputRatio=1;
assert.throws(()=>verifyModelReservation(store,session,'owned-task',config,body,altered),/不一致/);
assert.throws(()=>verifyModelReservation(store,session,'other-task',config,body,reservation),/当前Task/);
run.consumption[0]={...receipt,usage:{...receipt.usage,total_tokens:4900}};
assert.throws(()=>verifyModelReservation(store,session,'owned-task',config,body,reservation),/不一致/);
});
test('known high input calibrates the next request and stops before dispatch while preserving the completed read',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({...completion([call('owned-read','read_file',{path:'README.md'})]),
usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}}));
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000});
const provider=new Provider(),runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll());
runtime.start(f.session.id,'Read README. Do not modify files or run checks.');await runtime.active.get(f.session.id).promise;
const run=f.store.core.run.get(f.session.runId),first=structuredClone(run.consumption[0]);
assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit');
assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.unknownRequests,0);
assert.equal(f.session.events.filter(event=>event.tool==='read_file'&&event.phase==='completed').length,1);
assert.ok(run.budgetPause.tokenReservation.reservedTokens>run.budgetPause.remainingTokens);
assert.equal(run.budgetPause.tokenReservation.routes[0].sources[0].attemptId,first.attemptId);
assert.deepEqual(run.consumption[0],first);assert.ok(first.localEstimate.requestSha256);
});
for(const mode of ['legacy','pool','connection'])test('one reported overrun stops '+mode+' requests and keeps actual use and excess',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'owned failed request'},usage:{prompt_tokens:55000,completion_tokens:17,total_tokens:55017}}}));
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:35000,maxTaskTokens:1000000});
const provider=new Provider({retryPolicy:{delays:[1,1]}});
if(mode!=='legacy'){const hub=provider.modelHub(f.store),one=hub.upsert({name:'owned first',model:config.model,baseUrl:model.baseUrl,requiresKey:false});
if(mode==='connection')hub.select({type:'connection',id:one.id});
else{
const two=hub.upsert({name:'owned unused second',model:'owned-B',baseUrl:model.baseUrl,requiresKey:false});
const pool=hub.pool({name:'owned overrun',members:[{connectionId:one.id},{connectionId:two.id}]});hub.select({type:'pool',id:pool.id});}}
const runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll());
runtime.start(f.session.id,'Read README without changing files.');await runtime.active.get(f.session.id).promise;
const run=f.store.core.run.get(f.session.runId);
assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit');
assert.equal(run.consumption.length,1);assert.equal(run.usage.totalTokens,55017);assert.equal(run.usage.unknownRequests,0);
assert.equal(run.budgetPause.tokenAccounting.run.exceededTokens,20017);assert.equal(f.store.core.tool.size,0);
if(mode!=='legacy')assert.equal(provider.modelHub(f.store).view().attempts.length,1);
if(mode==='connection')assert.equal(run.budgetPause.modelFailure.httpStatus,503);
});
for(const mode of ['legacy','pool'])test('reported usage below the limit prevents an unaffordable '+mode+' retry before another dispatch',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'owned response below maximum'},
usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}}}));
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000});
const provider=new Provider({retryPolicy:{delays:[1,1]}});
if(mode==='pool'){
const hub=provider.modelHub(f.store),one=hub.upsert({name:'owned charged route',model:config.model,baseUrl:model.baseUrl,requiresKey:false}),
two=hub.upsert({name:'owned unused route',model:'owned-B',baseUrl:model.baseUrl,requiresKey:false});
const pool=hub.pool({name:'owned remaining budget',members:[{connectionId:one.id},{connectionId:two.id}]});hub.select({type:'pool',id:pool.id});
}
const runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll());
runtime.start(f.session.id,'Read README without changing files.');await runtime.active.get(f.session.id).promise;
const run=f.store.core.run.get(f.session.runId);
assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit');
assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.unknownRequests,0);assert.equal(run.consumption.length,1);
assert.equal(run.budgetPause.remainingTokens,24900);assert.ok(run.budgetPause.tokenReservation.reservedTokens>24900);
assert.equal(run.budgetPause.modelFailure.httpStatus,503);assert.equal(run.budgetPause.tokenAccounting.run.exceededTokens,0);
if(mode==='pool')assert.equal(provider.modelHub(f.store).view().attempts.length,1);
});
test('native bridge seals calibrated refusal with original consumption and preserves versioned request verification',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({...completion([call('native-owned-read','read_file',{path:'README.md'})]),
usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}})),context=controlledWorker(f);
context.provider.complete=Provider.prototype.complete.bind(context.provider);
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000});
context.definition.budget.maxTokens=70000;
const group=await context.manager.plan(f.session,context.definition);
await (await context.manager.prepare(f.session,group.id,group.fingerprint)).promise;assert.equal(group.status,'ready',group.summary);
await (await context.manager.start(f.session,group.id,group.fingerprint)).promise;
const job=[...f.store.core.engine_run.values()].find(value=>value.collaborationId===group.id),session=f.store.session(job.sessionId),run=f.store.core.run.get(job.runId);
assert.equal(model.requests.length,1);assert.equal(group.status,'paused');assert.equal(run.stopReason,'engine_input_budget',JSON.stringify({budget:run.budget,usage:run.usage,
task:f.store.core.task.get(run.taskId).usage,requests:job.requests.map(value=>({accounted:value.accountedAttempts,observed:value.observedAttempts,max:value.maxAttempts,estimate:value.reservedPerAttempt,status:value.status})),rejections:job.inputRejections?.map(value=>({remaining:value.remainingTokens,reserved:value.tokenReservation?.reservedTokens})),error:job.error}));
assert.equal(job.calls.length,1);assert.equal(job.calls[0].status,'returned');assert.equal(job.requests[0].reservationVersion,1);
assert.doesNotThrow(()=>verifyEngineModelInput(f.store,session,job,job.requests[0]));
const refusal=job.inputRejections[0];assert.ok(refusal.tokenReservation.reservedTokens>refusal.remainingTokens);
assert.equal(refusal.tokenReservation.routes[0].sources[0].attemptId,run.consumption[0].attemptId);
assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.requests,1);assert.equal(run.usage.unknownRequests,0);
});
// Original R evidence: six reported receipts, the fourth reports 27164 input for a
// 12168 local estimate while a same-shape fifth reports 8619. The sealed v1 pause
// must still verify byte-for-byte; v2 sets aside only that single isolated outlier.
function rEvidence(){
const {proof,receipts:originalReceipts,body}=replayFixtureJson('model-reservation/r.json'),route=proof.pause.tokenReservation.routes[0];
const receipts=originalReceipts.map(receipt=>({...receipt,runId:proof.pause.runId}));
const config={mode:'compatible',model:route.model,protocol:route.protocol,baseUrl:route.endpoint,maxOutputTokens:2048,contextLimit:16000,
modelRouting:{members:[{id:route.connectionId,model:route.model,protocol:route.protocol,baseUrl:route.endpoint}]}};
return {proof,route,receipts,config,body};
}
const rStore=(proof,receipts)=>{const run={id:proof.pause.runId,taskId:proof.pause.taskId,sessionId:'r-session',consumption:receipts};
return {session:{id:'r-session'},store:{core:{run:new Map([[run.id,run]])},owned(kind,id,sessionId){assert.equal(id,run.id);assert.equal(sessionId,run.sessionId);return run;}}};};
test('original R v1/v2 stay verifiable while v3 excludes the isolated input outlier and keeps the ledger intact',()=>{
const {proof,route,receipts,config,body}=rEvidence(),original=structuredClone(receipts),{store,session}=rStore(proof,receipts);
const legacy=legacyModelReservation(config,body,receipts);
assert.equal(legacy.reservedTokens,29623);assert.deepEqual(legacy.routes[0],route);
assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,proof.pause.tokenReservation),proof.pause.tokenReservation);
const current=modelReservation(config,body,receipts);
assert.equal(current.method,'observed-route-max-v3');assert.equal(current.reservedTokens,15285);
assert.ok(current.reservedTokens<=proof.pause.remainingTokens);assert.ok(current.reservedTokens>=12352+2048);
const [excluded]=current.routes[0].excludedSources;assert.equal(current.routes[0].excludedSources.length,1);
assert.equal(excluded.attemptId,'23226d1c-9ad6-4daa-81f5-1b44f0d57ecc');assert.equal(excluded.reason,'isolated_input_ratio_outlier');
assert.equal(current.routes[0].sources.length,5);assert.equal(current.routes[0].observedTotalFloor,13107);
assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,current),current);
const v2=legacyV2ModelReservation(config,body,receipts);
assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,v2),v2);
const forged=structuredClone(current);forged.routes[0].excludedSources=[];forged.routes[0].sources.push(excluded);
assert.throws(()=>verifyModelReservation(store,session,proof.pause.taskId,config,body,forged),/不一致/);
assert.deepEqual(receipts,original);assert.equal(proof.pause.runUsage.totalTokens,72571);
});
test('v3 never excludes a repeated high ratio, the latest receipt, or a short history',()=>{
const {receipts,config,body}=rEvidence(),high=receipts[3];
const repeated=[...receipts,{...high,attemptId:'r-repeat'}];
assert.equal(modelReservation(config,body,repeated).routes[0].excludedSources.length,0);
assert.equal(modelReservation(config,body,repeated).reservedTokens,legacyModelReservation(config,body,repeated).reservedTokens);
const latest=receipts.slice(0,4);
assert.equal(modelReservation(config,body,latest).routes[0].excludedSources.length,0);
assert.equal(modelReservation(config,body,latest).reservedTokens,29623);
const short=[receipts[0],receipts[3]];
assert.equal(modelReservation(config,body,short).routes[0].excludedSources.length,0);
});
test('reservation proofs reject reordered latest high receipts, altered history and duplicates while old snapshots survive later consumption',()=>{
const {proof,receipts,config,body}=rEvidence(),latest=[receipts[0],receipts[1],receipts[3]],{store,session}=rStore(proof,latest);
const verify=value=>verifyModelReservation(store,session,proof.pause.taskId,config,body,value);
const honest=modelReservation(config,body,latest);
assert.equal(honest.routes[0].excludedSources.length,0);
for(const compute of [modelReservation,legacyV2ModelReservation]){
const reordered=compute(config,body,[latest[0],latest[2],latest[1]]);
assert.equal(reordered.routes[0].excludedSources[0].attemptId,latest[2].attemptId);
assert.throws(()=>verify(reordered),/不一致/);
}
const altered=structuredClone(honest);altered.receiptHistory.reverse();assert.throws(()=>verify(altered),/不一致/);
const duplicate=structuredClone(honest);duplicate.routes[0].sources.push(duplicate.routes[0].sources[0]);assert.throws(()=>verify(duplicate),/不一致/);
latest.push({...receipts[2],attemptId:'later-receipt'});
assert.deepEqual(verify(honest),honest,'A later response must not rewrite an earlier reservation.');
const run=store.core.run.values().next().value;
run.consumption[0]={...run.consumption[0],usage:{prompt_tokens:1,completion_tokens:1,total_tokens:2}};
assert.throws(()=>verify(honest),/不一致/);
});
test('input-ratio exclusion preserves independently reported output maxima without rewriting historical v2',()=>{
const {receipts,config,body,proof}=rEvidence(),high={...receipts[3],usage:{prompt_tokens:27164,completion_tokens:9000,total_tokens:36164}};
const ledger=[receipts[0],high,receipts[1]],original=structuredClone(ledger),current=modelReservation(config,body,ledger);
assert.equal(current.routes[0].excludedSources[0].attemptId,high.attemptId);
assert.equal(current.routes[0].observedOutputTokens,9000);
assert.ok(current.reservedTokens>=current.routes[0].localInputTokens+9000);
assert.equal(legacyV2ModelReservation(config,body,ledger).routes[0].observedOutputTokens,2048);
const {store,session}=rStore(proof,ledger);
assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,current),current);
assert.deepEqual(ledger,original);
});