Files
Agent-Workbench/tests/model-identity.test.mjs
2026-11-03 11:53:07 +08:00

120 lines
11 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import {Provider} from '../src/server/provider.mjs';
import {Runtime} from '../src/server/runtime.mjs';
import {Store} from '../src/server/store.mjs';
import {fixture} from './helpers.mjs';
import {modelIdentityObserver,modelIdentitySettings} from '../src/shared/contracts/model-identity.mjs';
import {modelSnapshot} from '../src/shared/contracts/execution.mjs';
import {controlledWorker} from './fixtures/controlled-worker.mjs';
import {createDeliveryProject} from './fixtures/delivery-project.mjs';
import {DeliveryReporter} from '../src/core/delivery-report.mjs';
const config={mode:'compatible',baseUrl:'http://127.0.0.1:1/v1',model:'requested',maxOutputTokens:512};
const usage={prompt_tokens:13,completion_tokens:7,total_tokens:20};
const reply=model=>({model,choices:[{message:{role:'assistant',content:'已收到。'}}],usage});
const frame=value=>'data: '+JSON.stringify(value)+'\n\n';
const stream=(models,protocol)=>protocol==='responses'
? frame({type:'response.created',response:{id:'r',model:models[0]}})+frame({type:'response.completed',response:{id:'r',model:models.at(-1),status:'completed',output:[{type:'message',role:'assistant',content:[{type:'output_text',text:'已收到。'}]}],usage:{input_tokens:13,output_tokens:7,total_tokens:20}}})
: models.map((model,i)=>frame({model,choices:[{index:0,delta:{content:i?'':'已收到。'},finish_reason:i===models.length-1?'stop':null}],...(i===models.length-1?{usage}:{})})).join('')+'data: [DONE]\n\n';
test('identity distinguishes missing, invalid, mixed, exact and explicitly declared aliases without guessing',()=>{
assert.deepEqual(modelIdentitySettings({}),{});
for(const [values,status]of [[[], 'missing'],[['requested'],'matched'],[['other'],'mismatch'],[['requested','other'],'mixed'],[[null],'invalid']]){
const o=modelIdentityObserver('requested');for(const model of values)o.observe({model});assert.equal(o.snapshot(true).status,status);
}
assert.throws(()=>modelIdentitySettings({modelIdentityAliases:['snapshot']}),/依据/);
assert.throws(()=>modelIdentitySettings({modelIdentityAliases:['a','a'],modelIdentityAliasSource:'owned'}),/不重复/);
const o=modelIdentityObserver('requested',{modelIdentityAliases:['snapshot'],modelIdentityAliasSource:'Owned supplier mapping, revision 1'});
o.observe({model:'snapshot'});assert.equal(o.snapshot(true).status,'mapped');assert.equal(o.snapshot(false).status,'incomplete');
assert.equal(o.snapshot(true).mapping.kind,'user-declared');
});
for(const protocol of ['chat-completions','responses'])for(const streaming of [false,true]){
test(protocol+' '+(streaming?'SSE':'JSON')+' rejects a different model before tool dispatch, retaining source and reported usage',async t=>{
const f=fixture(t);let calls=0;
const provider=new Provider({fetch:async()=>{calls++;return new Response(streaming?stream(['other'],protocol):JSON.stringify(protocol==='responses'
? {model:'other',status:'completed',output:[{type:'message',role:'assistant',content:[{type:'output_text',text:'已收到。'}]}],usage:{input_tokens:13,output_tokens:7,total_tokens:20}}:reply('other')),
{headers:{'Content-Type':streaming?'text/event-stream':'application/json','x-request-id':'owned-identity'}});}});
const hub=provider.modelHub(f.store),c=hub.upsert({name:'Identity',model:'requested',baseUrl:config.baseUrl,requiresKey:false,protocol,streaming,
modelIdentityPolicy:'require-match',capabilityDeclarations:{text:'supported',tools:'supported',streaming:'supported'}});hub.select({type:'connection',id:c.id});
const runtime=new Runtime(f.store,provider);f.onCleanup(()=>runtime.stopAll());runtime.start(f.session.id,'读取当前项目。');await runtime.active.get(f.session.id).promise;
assert.equal(calls,1);assert.equal(f.session.status,'paused');assert.equal(f.store.core.run.get(f.session.runId).stopReason,'model_identity');
assert.equal(f.store.core.tool.size,0);assert.equal(f.session.failure.kind,'model_identity');
const attempt=hub.view().attempts[0],run=f.store.core.run.get(f.session.runId),identity=run.consumption[0].modelIdentity;
assert.equal(attempt.modelIdentity.status,'mismatch');assert.equal(identity.status,'mismatch');assert.equal(identity.requestedModel,'requested');
assert.deepEqual(identity.reportedModels,['other']);assert.equal(run.usage.totalTokens,20);assert.equal(run.usage.unknownRequests,0);
assert.equal(identity.responseSource.artifactId,attempt.responseArtifactId);assert.equal(identity.responseSource.sha256,attempt.responseSha256);
assert.ok(f.store.artifactStore.read(f.store.owned('artifact',attempt.responseArtifactId,f.session.id)).includes('other'));
assert.equal(hub.view().connections.find(v=>v.id===c.id).health.blocked,true);
f.store.close();const reopened=new Store(f.data);try{assert.deepEqual(reopened.core.run.get(run.id).consumption,run.consumption);}finally{reopened.close();}
});
}
test('record policy accepts missing identity but keeps it explicit, and strict declared mapping is frozen in the consumption',async()=>{
for(const settings of [{},{modelIdentityPolicy:'require-match',modelIdentityAliases:['snapshot'],modelIdentityAliasSource:'Owned supplier mapping'}]){
const events=[];const provider=new Provider({fetch:async()=>new Response(JSON.stringify(reply(settings.modelIdentityPolicy?'snapshot':undefined)))});
await provider.complete({...config,...settings},[],null,undefined,e=>events.push(e));
const r=events.find(e=>e.consumption).consumption;assert.equal(r.modelIdentity.status,settings.modelIdentityPolicy?'mapped':'missing');assert.equal(r.usage.total_tokens,20);
}
});
test('mixed streamed identities cannot pass strict mode, and interrupted streams retain observed identifiers without authenticating them',async()=>{
for(const interrupted of [false,true]){
const events=[];let calls=0;
const provider=new Provider({fetch:async()=>{calls++;return new Response(interrupted?frame({model:'requested',choices:[{index:0,delta:{content:'partial'}}]}):stream(['requested','other']),{headers:{'Content-Type':'text/event-stream'}});}});
await assert.rejects(provider.complete({...config,streaming:true,modelIdentityPolicy:'require-match'},[],null,undefined,e=>events.push(e)));
const r=events.find(e=>e.consumption).consumption;assert.equal(calls,1);assert.equal(r.modelIdentity.status,interrupted?'incomplete':'mixed');
assert.equal(r.modelIdentity.complete,!interrupted);assert.ok(r.modelIdentity.reportedModels.includes('requested'));
}
});
test('identity refusal never fails over to another pool member and raw response model cannot leak known credentials',async t=>{
const f=fixture(t),secret='owned-identity-private-key';let calls=0;
const provider=new Provider({fetch:async()=>{calls++;return new Response(JSON.stringify(reply(secret)));}}),hub=provider.modelHub(f.store);
const a=hub.upsert({name:'A',model:'requested',baseUrl:config.baseUrl,apiKey:secret,modelIdentityPolicy:'require-match'});
const b=hub.upsert({name:'B',model:'requested',baseUrl:config.baseUrl,requiresKey:false});
const pool=hub.pool({name:'Identity pool',members:[{connectionId:a.id},{connectionId:b.id}],maxAttempts:2});hub.select({type:'pool',id:pool.id});
const runtime=new Runtime(f.store,provider);f.onCleanup(()=>runtime.stopAll());runtime.start(f.session.id,'读取当前项目。');await runtime.active.get(f.session.id).promise;
assert.equal(calls,1);assert.equal(f.session.status,'paused');assert.ok(!JSON.stringify({view:hub.view(),failure:f.session.failure,run:f.store.core.run.get(f.session.runId)}).includes(secret));
});
test('strict refusal remains visible even when the same original response exhausts the budget',async t=>{
const f=fixture(t),provider=new Provider({fetch:async()=>new Response(JSON.stringify({...reply('other'),usage:{total_tokens:9999999}}))});
const hub=provider.modelHub(f.store),c=hub.upsert({name:'Strict',model:'requested',baseUrl:config.baseUrl,requiresKey:false,modelIdentityPolicy:'require-match'});
hub.select({type:'connection',id:c.id});const runtime=new Runtime(f.store,provider);f.onCleanup(()=>runtime.stopAll());
runtime.start(f.session.id,'读取项目。');await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status,'paused');assert.equal(f.session.failure.kind,'model_identity');
assert.equal(f.store.core.run.get(f.session.runId).stopReason,'model_identity');assert.equal(f.session.failure.budgetStopReason,'token_limit');
assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens,9999999);
});
test('strict identity crosses the controlled engine bridge with its original source and zero tool dispatch',async t=>{
const f=fixture(t),context=controlledWorker(f);let calls=0;
context.provider.complete=Provider.prototype.complete.bind(context.provider);
context.provider.fetcher=async()=>{calls++;return new Response(JSON.stringify(reply('other')));};
const hub=context.provider.modelHub(f.store),c=hub.upsert({name:'Strict engine',model:'requested',baseUrl:config.baseUrl,requiresKey:false,modelIdentityPolicy:'require-match'});
hub.select({type:'connection',id:c.id});
const group=await context.manager.plan(f.session,context.definition);
await(await context.manager.prepare(f.session,group.id,group.fingerprint)).promise;
await(await context.manager.start(f.session,group.id,group.fingerprint)).promise;
const session=f.store.session(group.children[0].sessionId),run=f.store.core.run.get(session.runId);
assert.equal(calls,1);assert.equal(session.status,'paused');assert.equal(run.stopReason,'model_identity');
assert.equal(run.usage.totalTokens,20);assert.equal(run.usage.unknownRequests,0);
const job=[...f.store.core.engine_run.values()].find(v=>v.runId===run.id);
assert.equal(job.calls.length,0);assert.equal(job.requests[0].modelIdentity.status,'mismatch');
assert.equal(job.requests[0].modelIdentity.responseSource.sha256,run.consumption[0].modelIdentity.responseSource.sha256);
assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.requiresVerification,false);
});
test('frozen direct connection policy and delivery reports preserve response identity without filling historical gaps',async t=>{
const settings={modelIdentityPolicy:'require-match',modelIdentityAliases:['snapshot'],modelIdentityAliasSource:'Owned revision 1'};
const frozen=modelSnapshot({...config,...settings});settings.modelIdentityAliases.push('later');assert.deepEqual(frozen.modelIdentityAliases,['snapshot']);
const f=fixture(t);createDeliveryProject(f);const run=f.store.core.run.get(f.session.runId),observer=modelIdentityObserver('requested');observer.observe({model:'other'});
run.consumption=[{attemptId:'owned',model:'requested',modelIdentity:observer.snapshot(true),usageStatus:'reported',usage}];
let report=await new DeliveryReporter(f.store).create(f.session);
assert.equal(report.report.runs.at(-1).consumption[0].modelIdentity.status,'mismatch');
assert.match(f.store.artifactStore.read(f.store.core.artifact.get(report.markdownArtifactId)),/请求与返回型号不一致/);
delete run.consumption;run.usage.requests=1;report=await new DeliveryReporter(f.store).create(f.session);
assert.match(f.store.artifactStore.read(f.store.core.artifact.get(report.markdownArtifactId)),/历史记录未保存返回型号核对/);
});