50 lines
3.9 KiB
JavaScript
50 lines
3.9 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { observedTaskModels } from '../scripts/live-model-observation.mjs';
|
|
|
|
const receipt=(attemptId,model='actual-claude',extra={})=>({attemptId,model,protocol:'chat-completions',connectionId:'owned-connection',
|
|
outcome:'completed',usageStatus:'reported',usage:{prompt_tokens:23,completion_tokens:7,total_tokens:30},...extra});
|
|
const state=(requests,runs)=>({config:{model:'planned-gpt'},execution:{task:{id:'owned-task',usage:{requests}},runs,run:runs.at(-1)}});
|
|
const run=(id,receipts,extra={})=>({id,taskId:'owned-task',consumption:receipts,usage:{requests:receipts.length},...extra});
|
|
|
|
test('actual model comes from owned consumption, excludes other Tasks and deduplicates the current Run',()=>{
|
|
const value=state(2,[run('first',[receipt('one')]),run('second',[receipt('two')]),run('foreign',[receipt('foreign','other')],{taskId:'other-task'})]);
|
|
value.execution.run=value.execution.runs[1];const original=structuredClone(value);
|
|
const report=observedTaskModels(value,'owned-task');
|
|
assert.equal(report.status,'observed');assert.equal(report.model,'actual-claude');assert.equal(report.observedAttempts,2);
|
|
assert.deepEqual(report.models,['actual-claude']);assert.equal(report.routes[0].requests,2);assert.deepEqual(value,original);
|
|
});
|
|
|
|
test('mixed routes retain each original model/protocol/outcome and unknown use is never reported as zero',()=>{
|
|
const value=state(2,[run('mixed',[receipt('one','actual-gpt',{outcome:'unknown',usageStatus:'unknown',usage:null}),
|
|
receipt('two','actual-claude',{protocol:'responses',connectionId:'second'})])]);
|
|
const report=observedTaskModels(value);
|
|
assert.equal(report.status,'mixed');assert.equal(report.model,null);assert.equal(report.routes.length,2);
|
|
assert.equal(report.attempts[0].usage,null);assert.equal(report.routes[0].unknownUsageRequests,1);
|
|
assert.equal(report.attempts[1].protocol,'responses');assert.deepEqual(report.attempts[1].usage,value.execution.run.consumption[1].usage);
|
|
});
|
|
|
|
test('zero dispatch, missing ledger, missing model and truncated Run history never borrow configured model identity',()=>{
|
|
assert.equal(observedTaskModels(state(0,[])).status,'not_observed');
|
|
assert.equal(observedTaskModels({}).status,'incomplete');
|
|
for(const value of [state(3,[run('old',[])]),state(1,[run('missing',[receipt('one',null)])]),state(2,[run('recent',[receipt('last')])])]){
|
|
const report=observedTaskModels(value);assert.equal(report.status,'incomplete');assert.equal(report.model,null);assert.ok(report.issues.length);
|
|
}
|
|
});
|
|
|
|
test('repeated attempt identities across Runs and contradictory usage stay explicit instead of inflating successes',()=>{
|
|
const report=observedTaskModels(state(2,[run('one',[receipt('same')]),run('two',[receipt('same','changed',{usage:null})])]));
|
|
assert.equal(report.status,'incomplete');assert.equal(report.observedAttempts,1);assert.equal(report.model,null);
|
|
assert.ok(report.issues.some(issue=>issue.kind==='conflicting_attempt'));assert.ok(report.issues.some(issue=>issue.kind==='request_coverage_gap'));
|
|
});
|
|
|
|
test('observed route never certifies returned identity and retains mismatches and original source',()=>{
|
|
const legacy=observedTaskModels(state(1,[run('old',[receipt('one')])]));
|
|
assert.equal(legacy.status,'observed');assert.equal(legacy.responseIdentity.allAccepted,false);
|
|
assert.equal(legacy.responseIdentity.counts.historical_unchecked,1);
|
|
const modelIdentity={requestedModel:'requested',reportedModels:['other'],status:'mismatch',complete:true,responseSource:{artifactId:'source',sha256:'a'.repeat(64)}};
|
|
const report=observedTaskModels(state(1,[run('new',[receipt('two','requested',{modelIdentity})])]));
|
|
assert.equal(report.status,'observed');assert.equal(report.model,'requested');assert.equal(report.responseIdentity.allAccepted,false);
|
|
assert.deepEqual(report.responseIdentity.reportedModels,['other']);assert.equal(report.attempts[0].modelIdentity.responseSource.artifactId,'source');
|
|
});
|