Files
Agent-Workbench/tests/live-model-observation.test.mjs
2026-11-03 11:53:07 +08:00

50 lines
3.9 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { observedTaskModels } from '../scripts/live-model-observation.mjs';
const receipt=(attemptId,model='actual-claude',extra={})=>({attemptId,model,protocol:'chat-completions',connectionId:'owned-connection',
outcome:'completed',usageStatus:'reported',usage:{prompt_tokens:23,completion_tokens:7,total_tokens:30},...extra});
const state=(requests,runs)=>({config:{model:'planned-gpt'},execution:{task:{id:'owned-task',usage:{requests}},runs,run:runs.at(-1)}});
const run=(id,receipts,extra={})=>({id,taskId:'owned-task',consumption:receipts,usage:{requests:receipts.length},...extra});
test('actual model comes from owned consumption, excludes other Tasks and deduplicates the current Run',()=>{
const value=state(2,[run('first',[receipt('one')]),run('second',[receipt('two')]),run('foreign',[receipt('foreign','other')],{taskId:'other-task'})]);
value.execution.run=value.execution.runs[1];const original=structuredClone(value);
const report=observedTaskModels(value,'owned-task');
assert.equal(report.status,'observed');assert.equal(report.model,'actual-claude');assert.equal(report.observedAttempts,2);
assert.deepEqual(report.models,['actual-claude']);assert.equal(report.routes[0].requests,2);assert.deepEqual(value,original);
});
test('mixed routes retain each original model/protocol/outcome and unknown use is never reported as zero',()=>{
const value=state(2,[run('mixed',[receipt('one','actual-gpt',{outcome:'unknown',usageStatus:'unknown',usage:null}),
receipt('two','actual-claude',{protocol:'responses',connectionId:'second'})])]);
const report=observedTaskModels(value);
assert.equal(report.status,'mixed');assert.equal(report.model,null);assert.equal(report.routes.length,2);
assert.equal(report.attempts[0].usage,null);assert.equal(report.routes[0].unknownUsageRequests,1);
assert.equal(report.attempts[1].protocol,'responses');assert.deepEqual(report.attempts[1].usage,value.execution.run.consumption[1].usage);
});
test('zero dispatch, missing ledger, missing model and truncated Run history never borrow configured model identity',()=>{
assert.equal(observedTaskModels(state(0,[])).status,'not_observed');
assert.equal(observedTaskModels({}).status,'incomplete');
for(const value of [state(3,[run('old',[])]),state(1,[run('missing',[receipt('one',null)])]),state(2,[run('recent',[receipt('last')])])]){
const report=observedTaskModels(value);assert.equal(report.status,'incomplete');assert.equal(report.model,null);assert.ok(report.issues.length);
}
});
test('repeated attempt identities across Runs and contradictory usage stay explicit instead of inflating successes',()=>{
const report=observedTaskModels(state(2,[run('one',[receipt('same')]),run('two',[receipt('same','changed',{usage:null})])]));
assert.equal(report.status,'incomplete');assert.equal(report.observedAttempts,1);assert.equal(report.model,null);
assert.ok(report.issues.some(issue=>issue.kind==='conflicting_attempt'));assert.ok(report.issues.some(issue=>issue.kind==='request_coverage_gap'));
});
test('observed route never certifies returned identity and retains mismatches and original source',()=>{
const legacy=observedTaskModels(state(1,[run('old',[receipt('one')])]));
assert.equal(legacy.status,'observed');assert.equal(legacy.responseIdentity.allAccepted,false);
assert.equal(legacy.responseIdentity.counts.historical_unchecked,1);
const modelIdentity={requestedModel:'requested',reportedModels:['other'],status:'mismatch',complete:true,responseSource:{artifactId:'source',sha256:'a'.repeat(64)}};
const report=observedTaskModels(state(1,[run('new',[receipt('two','requested',{modelIdentity})])]));
assert.equal(report.status,'observed');assert.equal(report.model,'requested');assert.equal(report.responseIdentity.allAccepted,false);
assert.deepEqual(report.responseIdentity.reportedModels,['other']);assert.equal(report.attempts[0].modelIdentity.responseSource.artifactId,'source');
});