Files
Agent-Workbench/tests/official-model-registry.test.mjs

166 lines
13 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { createHash } from 'node:crypto';
import { modelRegistry, validateModelRegistry, officialModelRecord, modelProvider, modelOverrideSettings, resolveModelSpecification } from '../src/shared/contracts/official-model-registry.mjs';
import { modelParameterProfile, changeReasoningSetting, reasoningSetting } from '../src/shared/contracts/model-parameter-profiles.mjs';
import { capabilityView } from '../src/shared/contracts/model-capabilities.mjs';
import { completionBody } from '../src/shared/contracts/model-request.mjs';
import { modelCapacityDeclaration } from '../src/shared/contracts/model-capacity.mjs';
import { modelSnapshot } from '../src/shared/contracts/execution.mjs';
import { catalogModel } from '../src/shared/contracts/model-catalog.mjs';
import { modelRoutingDecision } from '../src/shared/contracts/model-routing.mjs';
const open = { mode: 'compatible', executionMode: 'unrestricted', capacityMode: 'passthrough', historyPolicy: 'preserve', baseUrl: 'http://127.0.0.1:1/v1' };
const sol = { ...open, protocol: 'responses', model: 'gpt-6.1-sol' };
const claude = { ...open, protocol: 'anthropic-messages', model: 'claude-opus-5-5' };
test('every published fact resolves to its original pinned official document bytes', () => {
const fixture = JSON.parse(fs.readFileSync(new URL('./fixtures/official-model-registry/SOURCE.json', import.meta.url)));
assert.equal(modelRegistry.sources.length, 17);
assert.equal(modelRegistry.records.length, 26);
assert.equal(new Set(modelRegistry.records.map(item => item.model)).size, 13);
assert.deepEqual(modelRegistry.sources, fixture.sources);
for (const source of fixture.sources) {
const bytes = fs.readFileSync(new URL('./fixtures/official-model-registry/' + source.file, import.meta.url));
assert.equal(createHash('sha256').update(bytes).digest('hex'), source.sha256, source.id);
}
assert.equal(Object.isFrozen(modelRegistry.records[0].fields.efforts.value), true);
assert.throws(() => modelRegistry.records[0].fields.efforts.value.push('fabricated'), TypeError);
});
test('registry rejects forged origins, non-JSON values, duplicate keys and malformed selectable options', () => {
const mutations = [
input => input.sources[0].url = 'https://unreviewed.invalid/models',
input => input.sources[0].sha256 = 'unverified',
input => input.records[0].fields.contextLimit.source = input.sources[0].id,
input => input.records[0].fields.contextLimit.value = 0,
input => input.records[0].fields['capabilities.tools'].value = 'verified',
input => input.records[0].fields.efforts.value = 'all',
input => input.records[0].fields.defaultEffort.value = {},
input => input.records.push(structuredClone(input.records[0])),
input => input.records[0].fields.cyclic = { source: input.records[0].fields.efforts.source, value: input },
];
for (const change of mutations) { const input = structuredClone(modelRegistry); change(input); assert.throws(() => validateModelRegistry(input), { status: 400 }); }
const future = structuredClone(modelRegistry); future.records[0].protocol = 'future-data-only-protocol';
assert.equal(validateModelRegistry(future), future, 'A registry reference must not pretend to install a protocol adapter');
});
test('provider, exact case-sensitive model, protocol and explicit API version all bind a reference', () => {
assert.equal(modelProvider({ ...sol, brand: 'DeepSeek' }).id, 'openai');
assert.equal(modelProvider({ model: 'deepseek-owned-alias', brand: 'DeepSeek' }).id, null);
assert.equal(officialModelRecord({ ...sol, model: sol.model + '-custom' }), null);
assert.equal(officialModelRecord({ ...sol, model: sol.model.toUpperCase() }), null);
assert.equal(officialModelRecord({ ...sol, providerId: 'anthropic' }), null);
assert.equal(officialModelRecord({ ...claude, apiVersion: '2023-06-01' }), null);
assert.equal(officialModelRecord({ ...claude, apiVersion: '2026-10-21' }).model, claude.model);
assert.equal(officialModelRecord({ ...sol, protocol: 'anthropic-messages' }), null);
assert.equal(modelParameterProfile({ ...open, protocol: 'chat-completions', model: 'deepseek-owned-alias', brand: 'DeepSeek' }).id, 'openai');
assert.equal(modelParameterProfile({ ...open, protocol: 'chat-completions', model: 'owned', parameterProfile: 'deepseek' }).id, 'deepseek');
});
test('OpenAI exact Chat tool conditions and reasoning-none are independent of manual passthrough', () => {
for (const model of ['gpt-6.1-sol', 'gpt-6.1-terra']) {
assert.equal(capabilityView({ ...open, model, protocol: 'chat-completions' }).tools, 'unsupported');
assert.equal(capabilityView({ ...open, model, protocol: 'responses' }).tools, 'protocol_declared');
}
for (const model of ['gpt-6-sol', 'gpt-6-luna']) {
const connection = { ...open, model, protocol: 'chat-completions' };
assert.equal(capabilityView(connection).tools, 'unsupported');
assert.equal(capabilityView({ ...connection, nativeParameters: { reasoning_effort: 'none' } }).tools, 'protocol_declared');
assert.equal(modelParameterProfile(connection).modes.includes('disabled'), true);
}
assert.deepEqual(modelParameterProfile(sol).efforts, ['low', 'medium', 'high', 'xhigh', 'max']);
assert.deepEqual(modelParameterProfile(sol).modes, ['enabled']);
const body = completionBody({ ...sol, protocol: 'chat-completions' }, [{ role: 'user', content: 'original request' }], []);
assert.equal(Object.hasOwn(body, 'max_tokens'), false);
});
test('DeepSeek official dialect facts distinguish HTTP budgets, structured output and alias retirement', () => {
const messages = { ...open, model: 'deepseek-flash', protocol: 'anthropic-messages' };
const profile = modelParameterProfile(messages), fields = profile.specification.fields;
assert.deepEqual(profile.efforts, ['low', 'high', 'max']); assert.equal(profile.defaultEffort, 'high');
assert.equal(profile.thinkingBudget, 'ignored');
assert.equal(fields.contextLimit.value, 1048576); assert.equal(fields.maxOutputTokens.value, 393216);
assert.equal(fields['request.systemPromptUpdate'].value, 'in-history');
assert.equal(fields['request.maxTokensRequired'].value, false);
assert.equal(Object.hasOwn(completionBody(messages, [{ role: 'user', content: 'intact' }], []), 'max_tokens'), false);
assert.equal(completionBody({ ...messages, nativeParameters: { max_tokens: 700000 } }, [{ role: 'user', content: 'intact' }], []).max_tokens, 700000);
assert.deepEqual(changeReasoningSetting(messages, { output_config: { effort: 'max' } }, 'mode', 'enabled'), { output_config: { effort: 'max' }, thinking: { type: 'enabled' } });
assert.equal(resolveModelSpecification({ ...messages, model: 'deepseek-v4-pro' }).fields['capabilities.vision'].value, 'unsupported');
assert.equal(resolveModelSpecification({ ...messages, protocol: 'chat-completions' }).fields['features.structured_outputs'].value, 'unsupported');
assert.equal(resolveModelSpecification({ ...messages, protocol: 'responses' }).fields['features.structured_outputs'].value, 'supported');
const alias = resolveModelSpecification({ ...messages, model: 'deepseek-v4-flash' });
assert.equal(alias.fields.lifecycle.value, 'deprecated'); assert.equal(alias.fields.aliasOf.value, 'deepseek-flash');
assert.equal(alias.fields['request.originalModelRetired'].value, true);
});
test('catalog values, explicit denial and matching observations retain all conflicting origins', () => {
const connection = { ...sol, revision: 3, credentialRevision: 7, upstreamModel: { id: sol.model, endpoint: sol.baseUrl, maxOutputTokens: 96000,
capabilityDeclarations: { tools: 'unsupported' } }, capabilityDeclarations: { tools: 'unknown' },
capabilityEvidence: { connectionRevision: 3, credentialRevision: 7, checkedAt: '2026-11-07', results: { tools: 'verified' } } };
let fields = resolveModelSpecification(connection).fields;
assert.equal(fields.maxOutputTokens.kind, 'upstream_catalog'); assert.equal(fields.maxOutputTokens.value, 96000);
assert.equal(fields.maxOutputTokens.alternatives[0].value, 128000); assert.equal(fields.maxOutputTokens.conflict, true);
assert.equal(fields['capabilities.tools'].kind, 'revision_bound_observation'); assert.equal(capabilityView(connection, 7).tools, 'verified');
const explicit = { ...connection, modelOverrides: { capabilities: { tools: 'unsupported' } } };
assert.equal(capabilityView(explicit, 7).tools, 'unsupported'); assert.equal(resolveModelSpecification(explicit).fields['capabilities.tools'].conflict, true);
assert.equal(capabilityView(connection, 8).tools, 'unsupported', 'Stale observation is excluded');
const staleCatalog = { ...connection, capabilityEvidence: null, upstreamModel: { ...connection.upstreamModel, endpoint: 'https://other.invalid/v1' } };
assert.equal(resolveModelSpecification(staleCatalog).fields.maxOutputTokens.value, 128000);
});
test('legacy declarations keep their edit uncertainty instead of silently changing old routing settings', () => {
const connection = { ...open, model: 'gpt-6.1-sol', protocol: 'chat-completions', capabilityDeclarations: { tools: 'supported' } };
const field = resolveModelSpecification(connection).fields['capabilities.tools'];
assert.equal(field.kind, 'legacy_connection_declaration'); assert.equal(field.value, 'supported'); assert.equal(field.conflict, true);
assert.equal(resolveModelSpecification({ ...connection, capabilityDeclarationSources: { tools: 'protocol' } }).fields['capabilities.tools'].value, 'unsupported');
});
test('explicit unknown capacity and empty option arrays suppress backfills without overwriting native values', () => {
const overrides = modelOverrideSettings({ maxOutputTokens: null, efforts: [], modes: [], features: { effort: 'unsupported' }, note: 'proxy does not report a maximum' });
const connection = { ...sol, modelOverrides: overrides, nativeParameters: { reasoning: { effort: 'custom', summary: 'auto' }, max_output_tokens: 300000 } };
const before = structuredClone(connection);
assert.equal(modelCapacityDeclaration(connection, 'maxOutputTokens'), null);
assert.deepEqual(modelParameterProfile(connection).efforts, []); assert.deepEqual(modelParameterProfile(connection).modes, []);
assert.equal(completionBody(connection, [{ role: 'user', content: 'intact' }], []).max_output_tokens, 300000);
assert.deepEqual(connection, before);
for (const value of [{ maxOutputTokens: 0 }, { modes: 'all' }, { capabilities: { tools: 'verified' } }, { unknownField: true }]) assert.throws(() => modelOverrideSettings(value), { status: 400 });
assert.deepEqual(modelOverrideSettings(undefined, overrides), overrides);
assert.deepEqual(modelOverrideSettings({}), {}, 'Removing overrides restores future discovery, without mutating past specifications');
});
test('an existing frozen model specification survives a new catalog or registry interpretation', () => {
const original = resolveModelSpecification(sol), connection = { ...sol, modelSpecification: original, modelSpecificationFrozen: true };
const snapshot = modelSnapshot(connection), before = structuredClone(snapshot);
connection.upstreamModel = { id: sol.model, maxOutputTokens: 9000 };
connection.modelOverrides = { maxOutputTokens: 3000 };
assert.deepEqual(resolveModelSpecification(connection), original);
assert.equal(modelCapacityDeclaration(connection, 'maxOutputTokens').value, 128000);
original.fields.maxOutputTokens.value = 1;
assert.deepEqual(snapshot, before);
});
test('catalog no-options and individual denials cannot be mistaken for missing official facts', () => {
const empty = catalogModel({ id: sol.model, effort: { supported_levels: [] } });
assert.deepEqual(modelParameterProfile({ ...sol, upstreamModel: empty }).efforts, []);
const negative = catalogModel({ id: sol.model, capabilities: { effort: { high: false } } });
assert.deepEqual(modelParameterProfile({ ...sol, upstreamModel: negative }).efforts, ['low', 'medium', 'xhigh', 'max']);
const custom = { ...sol, upstreamModel: empty, modelOverrides: { efforts: ['custom'], features: { effort: 'supported' } } };
assert.deepEqual(modelParameterProfile(custom).efforts, ['custom']);
assert.equal(reasoningSetting(sol, { reasoning: { effort: 'max', summary: 'auto' } }).canResetMode, true);
});
test('automatic routing records official protocol conditions and explicit proxy differences; manual selection remains open', () => {
const member = { ...sol, id: 'owned', name: 'owned', protocol: 'chat-completions', revision: 1, credentialRevision: 0 };
const automatic = modelRoutingDecision({ type: 'pool', members: [member] }, ['tools'], { open: true });
assert.equal(automatic.candidates[0].disposition, 'excluded');
assert.equal(automatic.candidates[0].capabilities.tools.source, 'official-model-spec');
assert.equal(automatic.candidates[0].capabilities.tools.specification.conflict, false);
assert.equal(modelRoutingDecision({ type: 'connection', members: [member] }, ['tools'], { open: true }).candidates[0].disposition, 'eligible');
const proxy = { ...member, modelOverrides: { capabilities: { tools: 'supported' } }, upstreamModel: { id: member.model, capabilityDeclarations: { tools: 'unsupported' } } };
const declared = modelRoutingDecision({ type: 'pool', members: [proxy] }, ['tools'], { open: true }).candidates[0];
assert.equal(declared.disposition, 'eligible'); assert.equal(declared.capabilities.tools.source, 'explicit-compatibility');
assert.equal(declared.capabilities.tools.specification.conflict, true);
});