import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import {modelReservation,legacyModelReservation,legacyV2ModelReservation,localModelEstimate,verifyModelReservation} from '../src/shared/contracts/model-reservation.mjs'; import {completionBody} from '../src/shared/contracts/model-request.mjs'; import {Provider} from '../src/server/provider.mjs'; import {Runtime} from '../src/server/runtime.mjs'; import {fixture,compatibleFixture,completion,call} from './helpers.mjs'; import {controlledWorker} from './fixtures/controlled-worker.mjs'; import {verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs'; const config={mode:'compatible',model:'owned-A',protocol:'chat-completions',baseUrl:'http://127.0.0.1:1/v1',maxOutputTokens:512,contextLimit:16000}; const body=completionBody(config,[{role:'user',content:'Owned bounded input.'}],[]); const observed=(extra={})=>({attemptId:'owned-attempt',runId:'owned-run',model:config.model,protocol:config.protocol,endpoint:config.baseUrl, connectionId:null,outcome:'completed',usageStatus:'reported',localEstimate:localModelEstimate(body), usage:{prompt_tokens:4000,completion_tokens:800,total_tokens:4800},...extra}); test('reservation uses actual known usage from the same route, never lowers estimates or rewrites original receipts',()=>{ const receipts=[observed(),observed({attemptId:'unknown',usageStatus:'unknown',usage:{total_tokens:9999999}}), observed({attemptId:'other',model:'other-model',usage:{total_tokens:9999999}}),observed({attemptId:'legacy',localEstimate:undefined,usage:{total_tokens:9999999}})]; const original=structuredClone(receipts),reservation=modelReservation(config,body,receipts,128); assert.equal(reservation.reservedTokens,4928);assert.equal(reservation.routes[0].sources.length,1); assert.ok(reservation.routes[0].inputRatio>1);assert.deepEqual(receipts,original); const small=modelReservation(config,body,[observed({usage:{prompt_tokens:1,completion_tokens:1,total_tokens:2}})]); assert.equal(small.reservedTokens,localModelEstimate(body).inputTokens+512); const pool={...config,modelRouting:{members:[{id:'A',model:config.model,protocol:config.protocol,baseUrl:config.baseUrl}, {id:'B',model:'owned-B',protocol:config.protocol,baseUrl:'http://127.0.0.1:2/v1'}]}}; const mixed=modelReservation(pool,completionBody(pool,[{role:'user',content:'Owned bounded input.'}],[]),[observed({connectionId:'A'})]); assert.equal(mixed.routes[0].sources.length,1);assert.equal(mixed.routes[1].sources.length,0);assert.equal(mixed.routes[1].inputRatio,1); }); test('reservation verification binds original Task/Run consumption and rejects changed usage or foreign provenance',()=>{ const receipt=observed(),run={id:'owned-run',taskId:'owned-task',sessionId:'owned-session',consumption:[receipt]},session={id:'owned-session'}; const store={core:{run:new Map([[run.id,run]])},owned(kind,id,sessionId){assert.equal(kind,'run');assert.equal(id,run.id);assert.equal(sessionId,run.sessionId);return run;}}; const reservation=modelReservation(config,body,[receipt],128); assert.deepEqual(verifyModelReservation(store,session,'owned-task',config,body,reservation),reservation); const altered=structuredClone(reservation);altered.routes[0].inputRatio=1; assert.throws(()=>verifyModelReservation(store,session,'owned-task',config,body,altered),/不一致/); assert.throws(()=>verifyModelReservation(store,session,'other-task',config,body,reservation),/当前Task/); run.consumption[0]={...receipt,usage:{...receipt.usage,total_tokens:4900}}; assert.throws(()=>verifyModelReservation(store,session,'owned-task',config,body,reservation),/不一致/); }); test('known high input calibrates the next request and stops before dispatch while preserving the completed read',async t=>{ const f=fixture(t),model=await compatibleFixture(t,()=>({...completion([call('owned-read','read_file',{path:'README.md'})]), usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}})); f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000}); const provider=new Provider(),runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll()); runtime.start(f.session.id,'Read README. Do not modify files or run checks.');await runtime.active.get(f.session.id).promise; const run=f.store.core.run.get(f.session.runId),first=structuredClone(run.consumption[0]); assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit'); assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.unknownRequests,0); assert.equal(f.session.events.filter(event=>event.tool==='read_file'&&event.phase==='completed').length,1); assert.ok(run.budgetPause.tokenReservation.reservedTokens>run.budgetPause.remainingTokens); assert.equal(run.budgetPause.tokenReservation.routes[0].sources[0].attemptId,first.attemptId); assert.deepEqual(run.consumption[0],first);assert.ok(first.localEstimate.requestSha256); }); for(const mode of ['legacy','pool','connection'])test('one reported overrun stops '+mode+' requests and keeps actual use and excess',async t=>{ const f=fixture(t),model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'owned failed request'},usage:{prompt_tokens:55000,completion_tokens:17,total_tokens:55017}}})); f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:35000,maxTaskTokens:1000000}); const provider=new Provider({retryPolicy:{delays:[1,1]}}); if(mode!=='legacy'){const hub=provider.modelHub(f.store),one=hub.upsert({name:'owned first',model:config.model,baseUrl:model.baseUrl,requiresKey:false}); if(mode==='connection')hub.select({type:'connection',id:one.id}); else{ const two=hub.upsert({name:'owned unused second',model:'owned-B',baseUrl:model.baseUrl,requiresKey:false}); const pool=hub.pool({name:'owned overrun',members:[{connectionId:one.id},{connectionId:two.id}]});hub.select({type:'pool',id:pool.id});}} const runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll()); runtime.start(f.session.id,'Read README without changing files.');await runtime.active.get(f.session.id).promise; const run=f.store.core.run.get(f.session.runId); assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit'); assert.equal(run.consumption.length,1);assert.equal(run.usage.totalTokens,55017);assert.equal(run.usage.unknownRequests,0); assert.equal(run.budgetPause.tokenAccounting.run.exceededTokens,20017);assert.equal(f.store.core.tool.size,0); if(mode!=='legacy')assert.equal(provider.modelHub(f.store).view().attempts.length,1); if(mode==='connection')assert.equal(run.budgetPause.modelFailure.httpStatus,503); }); for(const mode of ['legacy','pool'])test('reported usage below the limit prevents an unaffordable '+mode+' retry before another dispatch',async t=>{ const f=fixture(t),model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'owned response below maximum'}, usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}}})); f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000}); const provider=new Provider({retryPolicy:{delays:[1,1]}}); if(mode==='pool'){ const hub=provider.modelHub(f.store),one=hub.upsert({name:'owned charged route',model:config.model,baseUrl:model.baseUrl,requiresKey:false}), two=hub.upsert({name:'owned unused route',model:'owned-B',baseUrl:model.baseUrl,requiresKey:false}); const pool=hub.pool({name:'owned remaining budget',members:[{connectionId:one.id},{connectionId:two.id}]});hub.select({type:'pool',id:pool.id}); } const runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll()); runtime.start(f.session.id,'Read README without changing files.');await runtime.active.get(f.session.id).promise; const run=f.store.core.run.get(f.session.runId); assert.equal(model.requests.length,1);assert.equal(f.session.status,'paused');assert.equal(run.stopReason,'token_limit'); assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.unknownRequests,0);assert.equal(run.consumption.length,1); assert.equal(run.budgetPause.remainingTokens,24900);assert.ok(run.budgetPause.tokenReservation.reservedTokens>24900); assert.equal(run.budgetPause.modelFailure.httpStatus,503);assert.equal(run.budgetPause.tokenAccounting.run.exceededTokens,0); if(mode==='pool')assert.equal(provider.modelHub(f.store).view().attempts.length,1); }); test('native bridge seals calibrated refusal with original consumption and preserves versioned request verification',async t=>{ const f=fixture(t),model=await compatibleFixture(t,()=>({...completion([call('native-owned-read','read_file',{path:'README.md'})]), usage:{prompt_tokens:45000,completion_tokens:100,total_tokens:45100}})),context=controlledWorker(f); context.provider.complete=Provider.prototype.complete.bind(context.provider); f.store.saveConfig({...config,baseUrl:model.baseUrl,maxRunTokens:70000,maxTaskTokens:1000000}); context.definition.budget.maxTokens=70000; const group=await context.manager.plan(f.session,context.definition); await (await context.manager.prepare(f.session,group.id,group.fingerprint)).promise;assert.equal(group.status,'ready',group.summary); await (await context.manager.start(f.session,group.id,group.fingerprint)).promise; const job=[...f.store.core.engine_run.values()].find(value=>value.collaborationId===group.id),session=f.store.session(job.sessionId),run=f.store.core.run.get(job.runId); assert.equal(model.requests.length,1);assert.equal(group.status,'paused');assert.equal(run.stopReason,'engine_input_budget',JSON.stringify({budget:run.budget,usage:run.usage, task:f.store.core.task.get(run.taskId).usage,requests:job.requests.map(value=>({accounted:value.accountedAttempts,observed:value.observedAttempts,max:value.maxAttempts,estimate:value.reservedPerAttempt,status:value.status})),rejections:job.inputRejections?.map(value=>({remaining:value.remainingTokens,reserved:value.tokenReservation?.reservedTokens})),error:job.error})); assert.equal(job.calls.length,1);assert.equal(job.calls[0].status,'returned');assert.equal(job.requests[0].reservationVersion,1); assert.doesNotThrow(()=>verifyEngineModelInput(f.store,session,job,job.requests[0])); const refusal=job.inputRejections[0];assert.ok(refusal.tokenReservation.reservedTokens>refusal.remainingTokens); assert.equal(refusal.tokenReservation.routes[0].sources[0].attemptId,run.consumption[0].attemptId); assert.equal(run.usage.totalTokens,45100);assert.equal(run.usage.requests,1);assert.equal(run.usage.unknownRequests,0); }); // Original R evidence: six reported receipts, the fourth reports 27164 input for a // 12168 local estimate while a same-shape fifth reports 8619. The sealed v1 pause // must still verify byte-for-byte; v2 sets aside only that single isolated outlier. const rDir=new URL('../docs/live-delivery-r-evidence/run/',import.meta.url); const rJson=name=>JSON.parse(fs.readFileSync(new URL(name,rDir),'utf8')); function rEvidence(){ const proof=rJson('analysis/budget-proof.json'),route=proof.pause.tokenReservation.routes[0]; const find=value=>{if(value&&typeof value==='object'){if(Array.isArray(value.consumption)&&value.consumption.length===6)return value.consumption; for(const child of Object.values(value)){const found=find(child);if(found)return found;}}return null;}; const receipts=find(rJson('node/session.json')).map(receipt=>({...receipt,runId:proof.pause.runId})); const config={mode:'compatible',model:route.model,protocol:route.protocol,baseUrl:route.endpoint,maxOutputTokens:2048,contextLimit:16000, modelRouting:{members:[{id:route.connectionId,model:route.model,protocol:route.protocol,baseUrl:route.endpoint}]}}; return {proof,route,receipts,config,body:rJson('node/captures/704d88f7-fbef-4507-9597-22cf79481961-request.json')}; } const rStore=(proof,receipts)=>{const run={id:proof.pause.runId,taskId:proof.pause.taskId,sessionId:'r-session',consumption:receipts}; return {session:{id:'r-session'},store:{core:{run:new Map([[run.id,run]])},owned(kind,id,sessionId){assert.equal(id,run.id);assert.equal(sessionId,run.sessionId);return run;}}};}; test('original R v1/v2 stay verifiable while v3 excludes the isolated input outlier and keeps the ledger intact',()=>{ const {proof,route,receipts,config,body}=rEvidence(),original=structuredClone(receipts),{store,session}=rStore(proof,receipts); const legacy=legacyModelReservation(config,body,receipts); assert.equal(legacy.reservedTokens,29623);assert.deepEqual(legacy.routes[0],route); assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,proof.pause.tokenReservation),proof.pause.tokenReservation); const current=modelReservation(config,body,receipts); assert.equal(current.method,'observed-route-max-v3');assert.equal(current.reservedTokens,15285); assert.ok(current.reservedTokens<=proof.pause.remainingTokens);assert.ok(current.reservedTokens>=12352+2048); const [excluded]=current.routes[0].excludedSources;assert.equal(current.routes[0].excludedSources.length,1); assert.equal(excluded.attemptId,'23226d1c-9ad6-4daa-81f5-1b44f0d57ecc');assert.equal(excluded.reason,'isolated_input_ratio_outlier'); assert.equal(current.routes[0].sources.length,5);assert.equal(current.routes[0].observedTotalFloor,13107); assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,current),current); const v2=legacyV2ModelReservation(config,body,receipts); assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,v2),v2); const forged=structuredClone(current);forged.routes[0].excludedSources=[];forged.routes[0].sources.push(excluded); assert.throws(()=>verifyModelReservation(store,session,proof.pause.taskId,config,body,forged),/不一致/); assert.deepEqual(receipts,original);assert.equal(proof.pause.runUsage.totalTokens,72571); }); test('v3 never excludes a repeated high ratio, the latest receipt, or a short history',()=>{ const {receipts,config,body}=rEvidence(),high=receipts[3]; const repeated=[...receipts,{...high,attemptId:'r-repeat'}]; assert.equal(modelReservation(config,body,repeated).routes[0].excludedSources.length,0); assert.equal(modelReservation(config,body,repeated).reservedTokens,legacyModelReservation(config,body,repeated).reservedTokens); const latest=receipts.slice(0,4); assert.equal(modelReservation(config,body,latest).routes[0].excludedSources.length,0); assert.equal(modelReservation(config,body,latest).reservedTokens,29623); const short=[receipts[0],receipts[3]]; assert.equal(modelReservation(config,body,short).routes[0].excludedSources.length,0); }); test('reservation proofs reject reordered latest high receipts, altered history and duplicates while old snapshots survive later consumption',()=>{ const {proof,receipts,config,body}=rEvidence(),latest=[receipts[0],receipts[1],receipts[3]],{store,session}=rStore(proof,latest); const verify=value=>verifyModelReservation(store,session,proof.pause.taskId,config,body,value); const honest=modelReservation(config,body,latest); assert.equal(honest.routes[0].excludedSources.length,0); for(const compute of [modelReservation,legacyV2ModelReservation]){ const reordered=compute(config,body,[latest[0],latest[2],latest[1]]); assert.equal(reordered.routes[0].excludedSources[0].attemptId,latest[2].attemptId); assert.throws(()=>verify(reordered),/不一致/); } const altered=structuredClone(honest);altered.receiptHistory.reverse();assert.throws(()=>verify(altered),/不一致/); const duplicate=structuredClone(honest);duplicate.routes[0].sources.push(duplicate.routes[0].sources[0]);assert.throws(()=>verify(duplicate),/不一致/); latest.push({...receipts[2],attemptId:'later-receipt'}); assert.deepEqual(verify(honest),honest,'A later response must not rewrite an earlier reservation.'); const run=store.core.run.values().next().value; run.consumption[0]={...run.consumption[0],usage:{prompt_tokens:1,completion_tokens:1,total_tokens:2}}; assert.throws(()=>verify(honest),/不一致/); }); test('input-ratio exclusion preserves independently reported output maxima without rewriting historical v2',()=>{ const {receipts,config,body,proof}=rEvidence(),high={...receipts[3],usage:{prompt_tokens:27164,completion_tokens:9000,total_tokens:36164}}; const ledger=[receipts[0],high,receipts[1]],original=structuredClone(ledger),current=modelReservation(config,body,ledger); assert.equal(current.routes[0].excludedSources[0].attemptId,high.attemptId); assert.equal(current.routes[0].observedOutputTokens,9000); assert.ok(current.reservedTokens>=current.routes[0].localInputTokens+9000); assert.equal(legacyV2ModelReservation(config,body,ledger).routes[0].observedOutputTokens,2048); const {store,session}=rStore(proof,ledger); assert.deepEqual(verifyModelReservation(store,session,proof.pause.taskId,config,body,current),current); assert.deepEqual(ledger,original); });