Files
Agent-Workbench/docs/task-agreement-1.25-evidence/source-bytes/tests/model-reservation.test.mjs
T

115 lines
10 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import {modelReservation,localModelEstimate,verifyModelReservation} from '../src/shared/contracts/model-reservation.mjs';
import {completionBody} from '../src/shared/contracts/model-request.mjs';
import {Provider} from '../src/server/provider.mjs';
import {Runtime} from '../src/server/runtime.mjs';
import {fixture,compatibleFixture,completion,call} from './helpers.mjs';
import {controlledWorker} from './fixtures/controlled-worker.mjs';
import {verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
const config={mode:'compatible',model:'owned-A',protocol:'chat-completions',baseUrl:'http://127.0.0.1:1/v1',maxOutputTokens:512,contextLimit:16000};
const body=completionBody(config,[{role:'user',content:'Owned bounded input.'}],[]);
const observed=(extra={})=>({attemptId:'owned-attempt',runId:'owned-run',model:config.model,protocol:config.protocol,endpoint:config.baseUrl,
connectionId:null,outcome:'completed',usageStatus:'reported',localEstimate:localModelEstimate(body),
usage:{prompt_tokens:4000,completion_tokens:800,total_tokens:4800},...extra});
test('reservation uses actual known usage from the same route, never lowers estimates or rewrites original receipts',()=>{
const receipts=[observed(),observed({attemptId:'unknown',usageStatus:'unknown',usage:{total_tokens:9999999}}),
observed({attemptId:'other',model:'other-model',usage:{total_tokens:9999999}}),observed({attemptId:'legacy',localEstimate:undefined,usage:{total_tokens:9999999}})];
const original=structuredClone(receipts),reservation=modelReservation(config,body,receipts,128);
assert.equal(reservation.reservedTokens,4928);assert.equal(reservation.routes[0].sources.length,1);
assert.ok(reservation.routes[0].inputRatio>1);assert.deepEqual(receipts,original);
const small=modelReservation(config,body,[observed({usage:{prompt_tokens:1,completion_tokens:1,total_tokens:2}})]);
assert.equal(small.reservedTokens,localModelEstimate(body).inputTokens+512);
const pool={...config,modelRouting:{members:[{id:'A',model:config.model,protocol:config.protocol,baseUrl:config.baseUrl},
{id:'B',model:'owned-B',protocol:config.protocol,baseUrl:'http://127.0.0.1:2/v1'}]}};
const mixed=modelReservation(pool,completionBody(pool,[{role:'user',content:'Owned bounded input.'}],[]),[observed({connectionId:'A'})]);
assert.equal(mixed.routes[0].sources.length,1);assert.equal(mixed.routes[1].sources.length,0);assert.equal(mixed.routes[1].inputRatio,1);
});
test('reservation verification binds original Task/Run consumption and rejects changed usage or foreign provenance',()=>{
const receipt=observed(),run={id:'owned-run',taskId:'owned-task',sessionId:'owned-session',consumption:[receipt]},session={id:'owned-session'};
const store={owned(kind,id,sessionId){assert.equal(kind,'run');assert.equal(id,run.id);assert.equal(sessionId,run.sessionId);return run;}};
const reservation=modelReservation(config,body,[receipt],128);
assert.deepEqual(verifyModelReservation(store,session,'owned-task',config,body,reservation),reservation);
const altered=structuredClone(reservation);altered.routes[0].inputRatio=1;
assert.throws(()=>verifyModelReservation(store,session,'owned-task',config,body,altered),/不一致/);
assert.throws(()=>verifyModelReservation(store,session,'other-task',config,body,reservation),/当前Task/);
run.consumption[0]={...receipt,usage:{...receipt.usage,total_tokens:4900}};
assert.throws(()=>verifyModelReservation(store,session,'owned-task',config,body,reservation),/不一致/);
});
test('known high input calibrates the next request and stops before dispatch while preserving the completed read',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({...completion([call('owned-read','read_file',{path:'README.md'})]),
usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}}));
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000});
const provider=new Provider(),runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll());
runtime.start(f.session.id,'Read README. Do not modify files or run checks.');await runtime.active.get(f.session.id).promise;
const run=f.store.core.run.get(f.session.runId),first=structuredClone(run.consumption[0]);
assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit');
assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.unknownRequests,0);
assert.equal(f.session.events.filter(event=>event.tool==='read_file'&&event.phase==='completed').length,1);
assert.ok(run.budgetPause.tokenReservation.reservedTokens>run.budgetPause.remainingTokens);
assert.equal(run.budgetPause.tokenReservation.routes[0].sources[0].attemptId,first.attemptId);
assert.deepEqual(run.consumption[0],first);assert.ok(first.localEstimate.requestSha256);
});
for(const mode of ['legacy','pool','connection'])test('one reported overrun stops '+mode+' requests and keeps actual use and excess',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'owned failed request'},usage:{prompt_tokens:55000,completion_tokens:17,total_tokens:55017}}}));
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:35000,maxTaskTokens:1000000});
const provider=new Provider({retryPolicy:{delays:[1,1]}});
if(mode!=='legacy'){const hub=provider.modelHub(f.store),one=hub.upsert({name:'owned first',model:config.model,baseUrl:model.baseUrl,requiresKey:false});
if(mode==='connection')hub.select({type:'connection',id:one.id});
else{
const two=hub.upsert({name:'owned unused second',model:'owned-B',baseUrl:model.baseUrl,requiresKey:false});
const pool=hub.pool({name:'owned overrun',members:[{connectionId:one.id},{connectionId:two.id}]});hub.select({type:'pool',id:pool.id});}}
const runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll());
runtime.start(f.session.id,'Read README without changing files.');await runtime.active.get(f.session.id).promise;
const run=f.store.core.run.get(f.session.runId);
assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit');
assert.equal(run.consumption.length,1);assert.equal(run.usage.totalTokens,55017);assert.equal(run.usage.unknownRequests,0);
assert.equal(run.budgetPause.tokenAccounting.run.exceededTokens,20017);assert.equal(f.store.core.tool.size,0);
if(mode!=='legacy')assert.equal(provider.modelHub(f.store).view().attempts.length,1);
if(mode==='connection')assert.equal(run.budgetPause.modelFailure.httpStatus,503);
});
for(const mode of ['legacy','pool'])test('reported usage below the limit prevents an unaffordable '+mode+' retry before another dispatch',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'owned response below maximum'},
usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}}}));
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000});
const provider=new Provider({retryPolicy:{delays:[1,1]}});
if(mode==='pool'){
const hub=provider.modelHub(f.store),one=hub.upsert({name:'owned charged route',model:config.model,baseUrl:model.baseUrl,requiresKey:false}),
two=hub.upsert({name:'owned unused route',model:'owned-B',baseUrl:model.baseUrl,requiresKey:false});
const pool=hub.pool({name:'owned remaining budget',members:[{connectionId:one.id},{connectionId:two.id}]});hub.select({type:'pool',id:pool.id});
}
const runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll());
runtime.start(f.session.id,'Read README without changing files.');await runtime.active.get(f.session.id).promise;
const run=f.store.core.run.get(f.session.runId);
assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit');
assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.unknownRequests,0);assert.equal(run.consumption.length,1);
assert.equal(run.budgetPause.remainingTokens,24900);assert.ok(run.budgetPause.tokenReservation.reservedTokens>24900);
assert.equal(run.budgetPause.modelFailure.httpStatus,503);assert.equal(run.budgetPause.tokenAccounting.run.exceededTokens,0);
if(mode==='pool')assert.equal(provider.modelHub(f.store).view().attempts.length,1);
});
test('native bridge seals calibrated refusal with original consumption and preserves versioned request verification',async t=>{
const f=fixture(t),model=await compatibleFixture(t,()=>({...completion([call('native-owned-read','read_file',{path:'README.md'})]),
usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}})),context=controlledWorker(f);
context.provider.complete=Provider.prototype.complete.bind(context.provider);
f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000});
context.definition.budget.maxTokens=70000;
const group=await context.manager.plan(f.session,context.definition);
await (await context.manager.prepare(f.session,group.id,group.fingerprint)).promise;assert.equal(group.status,'ready',group.summary);
await (await context.manager.start(f.session,group.id,group.fingerprint)).promise;
const job=[...f.store.core.engine_run.values()].find(value=>value.collaborationId===group.id),session=f.store.session(job.sessionId),run=f.store.core.run.get(job.runId);
assert.equal(model.requests.length,1);assert.equal(group.status,'paused');assert.equal(run.stopReason,'engine_input_budget',JSON.stringify({budget:run.budget,usage:run.usage,
task:f.store.core.task.get(run.taskId).usage,requests:job.requests.map(value=>({accounted:value.accountedAttempts,observed:value.observedAttempts,max:value.maxAttempts,estimate:value.reservedPerAttempt,status:value.status})),rejections:job.inputRejections?.map(value=>({remaining:value.remainingTokens,reserved:value.tokenReservation?.reservedTokens})),error:job.error}));
assert.equal(job.calls.length,1);assert.equal(job.calls[0].status,'returned');assert.equal(job.requests[0].reservationVersion,1);
assert.doesNotThrow(()=>verifyEngineModelInput(f.store,session,job,job.requests[0]));
const refusal=job.inputRejections[0];assert.ok(refusal.tokenReservation.reservedTokens>refusal.remainingTokens);
assert.equal(refusal.tokenReservation.routes[0].sources[0].attemptId,run.consumption[0].attemptId);
assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.requests,1);assert.equal(run.usage.unknownRequests,0);
});