Files
Agent-Workbench/tests/model-catalog.test.mjs
2026-11-12 09:01:34 +08:00

252 lines
24 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { fixture, compatibleFixture, completion, until, launchWorkbench } from './helpers.mjs';
import { Store } from '../src/server/store.mjs';
import { Provider } from '../src/server/provider.mjs';
import { catalogModel } from '../src/shared/contracts/model-catalog.mjs';
import { modelSnapshot } from '../src/shared/contracts/execution.mjs';
import { modelParameterProfile } from '../src/shared/contracts/model-parameter-profiles.mjs';
const create = f => { const provider = new Provider(); provider.setKey(''); return { provider, hub: provider.modelHub(f.store) }; };
const draft = server => ({ name: '自有上游', protocol: 'chat-completions', baseUrl: server.baseUrl, requiresKey: true });
const metadata = id => ({ id, capabilities: { text: true, tools: true, streaming: true, image_input: { supported: true },
thinking: { supported: true, types: { adaptive: { supported: true }, enabled: { supported: false }, disabled: { supported: true } } },
effort: { supported: true, low: { supported: true }, high: { supported: true }, max: { supported: false } }, structured_outputs: { supported: true } }, max_input_tokens: 200000, max_tokens: 8192 });
test('upstream declarations include brand metadata and remain distinct from adapter support or verification', () => {
const value = catalogModel(metadata('claude-owned'));
assert.equal(value.capabilityDeclarations.vision, 'supported'); assert.equal(value.capabilityDeclarations.audio, 'unknown');
assert.deepEqual(value.efforts, ['low', 'high']); assert.deepEqual(value.modes, ['disabled', 'adaptive']);
assert.equal(value.features.structured_outputs, 'supported');
assert.equal(catalogModel({ id: 'only-id' }).capabilityDeclarations.tools, 'unknown');
assert.equal(catalogModel({ id: 'embed', capabilities: { tools: false }, supported_parameters: ['tools'] }).capabilityDeclarations.tools, 'unsupported');
assert.equal(catalogModel({ id: 'multi', architecture: { input_modalities: ['image'], output_modalities: ['text'] } }).capabilityDeclarations.text, 'supported');
for (const id of ['', 'x'.repeat(201), 'x\ny']) assert.throws(() => catalogModel({ id }), /名称/);
});
test('new upstream retrieves more than 100 models without a manual model; choosing several stores one key and frozen metadata', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: Array.from({ length: 180 }, (_, i) => metadata('owned-' + i)) }));
const { provider, hub } = create(f), preview = await hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned-catalog-secret' });
assert.equal(preview.entries.length, 180); assert.equal(preview.complete, true); assert.equal(server.requests.length, 1);
assert.equal(server.requests[0].input, null); assert.equal(server.requests[0].url, '/v1/models');
assert.equal(server.requests[0].headers.authorization, 'Bearer owned-catalog-secret');
assert.equal(hub.state.connections.length, 0); assert.equal(hub.attempts.length, 0);
const result = hub.modelCatalog().apply({ previewId: preview.id, models: ['owned-3', 'owned-7', 'owned-150'], routing: 'manual', defaultModel: 'owned-7' });
const [source, child] = result.added.map(id => hub.connection(id));
assert.equal(provider.credentials.size, 1); assert.equal(hub.credential(source), hub.credential(child));
assert.equal(child.credentialSource.connectionId, source.id); assert.equal(child.upstreamModel.id, 'owned-7');
assert.equal(child.contextLimit, 200000); assert.equal(child.maxOutputTokens, 8192);
assert.equal(hub.status(child).capabilities.vision, 'protocol_declared'); assert.equal(hub.status(child).capabilityEvidence, undefined);
assert.equal(f.store.config.mode, 'compatible'); assert.equal(hub.state.selected.id, child.id);
const prepared = hub.prepare(f.store.config), snapshot = modelSnapshot(prepared);
assert.equal(snapshot.routing.members[0].upstreamModel.id, 'owned-7');
assert.equal(JSON.stringify(hub.state).includes('owned-catalog-secret'), false);
assert.equal(JSON.stringify(hub.view()).includes('owned-catalog-secret'), false);
assert.throws(() => hub.modelCatalog().apply({ previewId: preview.id, models: ['owned-3'] }), /过期/);
});
test('Responses and DeepSeek list use bearer; Messages root and v1 endpoints paginate with selected authentication', async t => {
const f = fixture(t), server = await compatibleFixture(t, (url) => url.includes('after_id=') ? { data: [metadata('owned-b')], has_more: false } : { data: [metadata('owned-a')], has_more: true, last_id: 'owned-a' });
const { hub } = create(f);
for (const auth of ['api-key', 'bearer']) for (const baseUrl of [server.baseUrl, server.baseUrl.replace(/\/v1$/, '')]) {
const preview = await hub.modelCatalog().fetch({ draft: { ...draft(server), baseUrl, protocol: 'anthropic-messages', messagesAuth: auth }, apiKey: 'owned-msg' });
assert.deepEqual(preview.entries.map(value => value.id), ['owned-a', 'owned-b']);
const last = server.requests.at(-1); assert.equal(last.url, '/v1/models?after_id=owned-a&limit=100');
assert.equal(last.headers['anthropic-version'], '2023-06-01');
assert.equal(last.headers['x-api-key'], auth === 'api-key' ? 'owned-msg' : undefined);
assert.equal(last.headers.authorization, auth === 'bearer' ? 'Bearer owned-msg' : undefined);
hub.modelCatalog().cancel(preview.id);
}
});
test('refresh preserves existing key objects, encrypted block, parameters and user capability overrides without re-encryption', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: [metadata('source'), metadata('child')] }));
const { provider, hub } = create(f), source = hub.upsert({ ...draft(server), model: 'source', apiKey: 'owned-persist', persistKey: true });
const credential = provider.credentials.get(source.id), block = structuredClone(f.store.storage.meta('modelCredentials'));
const preview = await hub.modelCatalog().fetch({ id: source.id, revision: source.revision });
const first = hub.modelCatalog().apply({ previewId: preview.id, models: ['source', 'child'] });
assert.deepEqual(first.existing, [source.id]); const child = hub.connection(first.added[0]);
hub.setNativeParameters({ id: child.id, revision: child.revision, nativeParameters: { reasoning_effort: 'high', seed: 3 } });
const edited = hub.upsert({ ...hub.connection(child.id), name: '用户命名', capabilityDeclarations: { vision: 'unsupported' } });
const refresh = await hub.modelCatalog().fetch({ id: source.id, revision: source.revision });
const second = hub.modelCatalog().apply({ previewId: refresh.id, models: ['child'], routing: 'manual', defaultModel: 'child' });
assert.deepEqual(second.added, []); assert.deepEqual(second.existing, [child.id]);
assert.equal(hub.connection(child.id).name, '用户命名'); assert.deepEqual(hub.connection(child.id).nativeParameters, edited.nativeParameters);
assert.equal(hub.connection(child.id).capabilityDeclarations.vision, 'unsupported');
assert.equal(provider.credentials.get(source.id), credential); assert.equal(hub.credential(child), credential);
assert.deepEqual(f.store.storage.meta('modelCredentials'), block); assert.equal(provider.credentials.size, 1);
assert.throws(() => hub.manageCredential({ id: child.id, revision: edited.revision, action: 'forget' }), /源连接/);
assert.equal(hub.vault().status(child).status, 'reference');
assert.throws(() => hub.upsert({ ...edited, baseUrl: 'https://changed.test' }), /共用上游/);
});
test('shared credentials restore from the original saved block; source disable, endpoint changes and key replacement cannot revive frozen runs', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: [metadata('owned-a'), metadata('owned-b')] }));
let { hub, provider } = create(f);
const preview = await hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned-original' });
const result = hub.modelCatalog().apply({ previewId: preview.id, models: ['owned-a', 'owned-b'], persistKey: true, routing: 'manual', defaultModel: 'owned-b' });
const [sourceId, childId] = result.added, old = modelSnapshot(hub.prepare(f.store.config));
f.store.close(); const reopened = new Store(f.data); f.onCleanup(() => reopened.close()); provider = new Provider(); provider.setKey(''); hub = provider.modelHub(reopened);
const source = hub.connection(sourceId), child = hub.connection(childId), credential = hub.credential(source);
assert.equal(credential.key, 'owned-original'); assert.equal(hub.credential(child), credential); assert.equal(provider.credentials.size, 1);
const member = hub.prepare(reopened.config).modelRouting.members[0]; assert.equal(hub.available(member), true);
hub.toggle(sourceId, false); assert.equal(hub.available(member), false); assert.equal(hub.status(child).hasKey, false);
hub.toggle(sourceId, true); assert.equal(hub.available(member), false, 'a source generation change invalidates the frozen member');
let owner = hub.connection(sourceId); hub.upsert({ ...owner, baseUrl: 'https://elsewhere.test' });
assert.equal(hub.credential(child), undefined); assert.equal(hub.available(member), false);
owner = hub.connection(sourceId); hub.upsert({ ...owner, baseUrl: server.baseUrl, apiKey: 'owned-new' });
assert.equal(hub.credential(child).revision, credential.revision + 1); assert.equal(hub.available(member), false);
assert.equal(old.routing.members[0].credentialRevision, credential.revision); assert.equal(old.routing.members[0].upstreamModel.id, 'owned-b');
});
test('automatic selection reuses capability and health filtering, manual selection sends the exact chosen model and shared auth', async t => {
const f = fixture(t), server = await compatibleFixture(t, (_url, input) => input ? { ...completion([], 'OK'), model: input.model } : { data: [
{ id: 'text-only', capabilities: { text: true, tools: false } }, metadata('tools-and-vision'),
] });
const { hub, provider } = create(f), preview = await hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned-auto' });
const result = hub.modelCatalog().apply({ previewId: preview.id, models: ['text-only', 'tools-and-vision'], routing: 'auto' });
const config = hub.prepare(f.store.config), before = modelSnapshot(config), pool = hub.state.pools[0];
assert.equal(hub.state.selected.type, 'pool'); assert.equal(pool.members.length, 2);
const lease = await hub.acquire(config.modelRouting, randomUUID(), new Set(), undefined, Date.now() + 1000, ['tools', 'vision']);
assert.equal(lease.member.model, 'tools-and-vision'); lease.release();
hub.select({ type: 'connection', id: result.added[1] });
await provider.complete(hub.prepare(f.store.config), [{ role: 'user', content: 'owned' }]);
assert.equal(server.requests.at(-1).input.model, 'tools-and-vision'); assert.equal(server.requests.at(-1).headers.authorization, 'Bearer owned-auto');
assert.equal(before.routing.type, 'pool'); assert.equal(before.routing.members.length, 2); assert.equal(hub.attempts.length, 1);
});
test('unknown catalog capabilities require explicit declarations; upstream negative and retired models cannot be overridden', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: [{ id: 'unknown' }, { id: 'negative', capabilities: { tools: false } }, { id: 'retired', lifecycle: 'retired' }] }));
const { hub } = create(f), preview = await hub.modelCatalog().fetch({ draft: { ...draft(server), requiresKey: false } });
assert.throws(() => hub.modelCatalog().apply({ previewId: preview.id, models: ['retired'] }), /退役/);
const result = hub.modelCatalog().apply({ previewId: preview.id, models: ['unknown', 'negative'], declarations: { text: 'supported', tools: 'supported' } });
assert.equal(hub.status(hub.connection(result.added[0])).capabilities.tools, 'protocol_declared');
assert.equal(hub.status(hub.connection(result.added[1])).capabilities.tools, 'unsupported');
});
test('model reasoning controls use declared effort and mode ranges without silently enabling unsupported options', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: [metadata('claude-owned')] }));
const { hub } = create(f), preview = await hub.modelCatalog().fetch({ draft: { ...draft(server), protocol: 'anthropic-messages', requiresKey: false } });
const result = hub.modelCatalog().apply({ previewId: preview.id, models: ['claude-owned'] }), member = hub.connection(result.added[0]);
const profile = modelParameterProfile(member); assert.deepEqual(profile.efforts, ['low', 'high']); assert.deepEqual(profile.modes, ['disabled', 'adaptive']);
assert.throws(() => hub.setNativeParameters({ id: member.id, revision: member.revision, nativeParameters: { thinking: { type: 'enabled', budget_tokens: 1024 } } }), /推理模式/);
assert.throws(() => hub.setNativeParameters({ id: member.id, revision: member.revision, nativeParameters: { output_config: { effort: 'max' } } }), /推理强度/);
hub.setNativeParameters({ id: member.id, revision: member.revision, nativeParameters: { thinking: { type: 'adaptive' }, output_config: { effort: 'high' } } });
});
test('cancelled, expired, changed-source and failed catalogs cannot save or overwrite a later selection', async t => {
const f = fixture(t); let release;
const server = await compatibleFixture(t, async () => { await new Promise(resolve => { release = resolve; }); return { data: [metadata('owned')] }; });
const { hub } = create(f), source = hub.upsert({ ...draft(server), model: 'source', apiKey: 'owned' }), id = randomUUID();
const pending = hub.modelCatalog().fetch({ id: source.id, revision: source.revision, requestId: id });
await until(() => !!release); hub.modelCatalog().cancel(id); release(); await assert.rejects(pending);
assert.equal(hub.catalog.previews.size, 0); assert.equal(hub.catalog.active.size, 0);
assert.throws(() => hub.catalog.apply({ previewId: id, models: ['owned'] }), /取消/);
const alreadyCancelled = randomUUID(); hub.catalog.cancel(alreadyCancelled);
await assert.rejects(hub.catalog.fetch({ draft: draft(server), apiKey: 'owned', requestId: alreadyCancelled }), /取消/);
assert.equal(server.requests.length, 1, 'cancel received before fetch also prevents a late dispatch');
const pending2 = hub.catalog.fetch({ id: source.id, revision: source.revision }); await until(() => hub.catalog.active.size > 0);
hub.toggle(source.id, false); await until(() => server.requests.length === 2); release(); await assert.rejects(pending2, /变化/);
assert.equal(hub.catalog.previews.size, 0); hub.toggle(source.id, true);
const pending3 = hub.catalog.fetch({ id: source.id, revision: hub.connection(source.id).revision }); await until(() => server.requests.length === 3); release();
const preview = await pending3; hub.catalog.previews.get(preview.id).expires = 0;
assert.throws(() => hub.catalog.apply({ previewId: preview.id, models: ['owned'] }), /过期/);
});
test('pagination loops, conflicting models and upstream errors fail once without creating members or model attempts', async t => {
const f = fixture(t); let scenario = 'loop';
const server = await compatibleFixture(t, (_url, _input, requests) => scenario === 'loop' ? { data: [{ id: 'same' }], has_more: true, last_id: 'same' }
: scenario === 'conflict' ? { data: [{ id: 'same' }, { id: 'same', display_name: 'other' }] }
: { status: 503, body: { error: { message: 'owned-catalog upstream unavailable' } } });
const { hub } = create(f);
await assert.rejects(hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned' }), /分页/); assert.equal(server.requests.length, 2);
scenario = 'conflict'; await assert.rejects(hub.catalog.fetch({ draft: draft(server), apiKey: 'owned' }), /冲突/);
scenario = 'error'; await assert.rejects(hub.catalog.fetch({ draft: draft(server), apiKey: 'owned' })); assert.equal(server.requests.length, 4);
assert.equal(hub.state.connections.length, 0); assert.equal(hub.attempts.length, 0); assert.equal(hub.catalog.previews.size, 0);
});
test('limits and transaction failures preserve original registry, config, cipher and preview for explicit correction', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: Array.from({ length: 101 }, (_, i) => ({ id: 'owned-' + i })) }));
const { hub, provider } = create(f), preview = await hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned-batch' });
assert.throws(() => hub.catalog.apply({ previewId: preview.id, models: preview.entries.slice(0, 33).map(value => value.id), routing: 'auto' }), /32/);
assert.throws(() => hub.catalog.apply({ previewId: preview.id, models: ['owned-0', 'owned-0'] }), /重复/);
assert.throws(() => hub.catalog.apply({ previewId: preview.id, models: ['foreign'] }), /不属于/);
const registry = structuredClone(hub.state), config = structuredClone(f.store.config), setMeta = f.store.storage.setMeta.bind(f.store.storage);
t.mock.method(f.store.storage, 'setMeta', (key, value) => { if (key === 'modelRegistry') throw new Error('owned commit failure'); return setMeta(key, value); });
assert.throws(() => hub.catalog.apply({ previewId: preview.id, models: ['owned-0', 'owned-1'], persistKey: true, routing: 'auto' }), /owned commit failure/);
assert.deepEqual(hub.state, registry); assert.deepEqual(f.store.config, config); assert.equal(provider.credentials.size, 0);
assert.equal(f.store.storage.meta('modelCredentials', null), null); assert.equal(hub.catalog.previews.has(preview.id), true);
});
test('a model that switches from a shared to an explicit credential receives a new revision for opaque-history isolation', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: [metadata('a'), metadata('b')] }));
const { hub } = create(f), preview = await hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned-shared' });
const result = hub.catalog.apply({ previewId: preview.id, models: ['a', 'b'] }), child = hub.connection(result.added[1]);
const before = hub.credential(child); hub.upsert({ ...child, apiKey: 'owned-independent' });
const after = hub.connection(child.id); assert.equal(after.credentialSource, undefined); assert.equal(hub.credential(after).revision, before.revision + 1);
assert.equal(hub.credential(after).key, 'owned-independent'); assert.equal(hub.credential(hub.connection(result.added[0])), before);
});
test('compiled HTTP directory flow persists selection and references, gates changes during fetch and permits explicit cancel', async t => {
const f = fixture(t); f.store.close(); let wait = false, release;
const server = await compatibleFixture(t, async () => { if (wait) await new Promise(resolve => { release = resolve; }); return { data: [metadata('api-a'), metadata('api-b')] }; });
const host = await launchWorkbench(t, f.data);
const preview = await host.api('/model-center/catalog', { draft: draft(server), apiKey: 'owned-http-key' });
const result = await host.api('/model-center/catalog/apply', { previewId: preview.id, models: ['api-a', 'api-b'], routing: 'auto' });
assert.equal(result.added.length, 2); assert.equal(result.center.selected.type, 'pool');
const selected = await host.api('/model-center/selection'); assert.equal(selected.config.mode, 'compatible'); assert.equal(selected.modelSelection.type, 'pool');
const source = result.center.connections.find(value => value.id === result.added[0]); wait = true;
const requestId = randomUUID(), pending = host.api('/model-center/catalog', { id: source.id, revision: source.revision, requestId });
const observed = pending.then(value => ({ value }), error => ({ error }));
await until(() => !!release);
await assert.rejects(host.api('/model-center/connections', { ...draft(server), model: 'blocked' }), cause => cause.status === 409);
await host.api('/model-center/catalog/cancel', { id: requestId }); release(); assert.ok((await observed).error);
await assert.rejects(host.api('/model-center/catalog/apply', { previewId: requestId, models: ['api-a'] }), cause => cause.status === 409);
assert.equal((await host.api('/model-center')).attempts.length, 0);
await host.close(); const store = new Store(f.data); f.onCleanup(() => store.close());
const provider = new Provider(); provider.setKey(''); const hub = provider.modelHub(store);
assert.equal(hub.state.selected.type, 'pool'); assert.equal(hub.connection(result.added[1]).credentialSource.connectionId, source.id);
assert.equal(hub.status(hub.connection(result.added[1])).status, 'configured', 'normal catalog entry restores its default encrypted credential');
assert.equal(hub.vault().status(hub.connection(source.id)).status, 'saved');
assert.equal(JSON.stringify(hub.view()).includes('owned-http-key'), false);
});
test('large directories declare partial coverage and a batch crossing the local connection limit leaves no partial members', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: Array.from({ length: 1000 }, (_, i) => ({ id: 'many-' + i })), has_more: true, last_id: 'many-999' }));
const { hub } = create(f), preview = await hub.modelCatalog().fetch({ draft: { ...draft(server), requiresKey: false } });
assert.equal(preview.complete, false); assert.match(preview.warning, /后续页|部分/); assert.equal(preview.entries.length, 1000); assert.equal(server.requests.length, 1);
for (let i = 0; i < 99; i++) hub.upsert({ ...draft(server), name: 'existing-' + i, model: 'existing-' + i, requiresKey: false });
const previous = structuredClone(hub.state);
assert.throws(() => hub.catalog.apply({ previewId: preview.id, models: ['many-1', 'many-2'] }), /100个/);
assert.deepEqual(hub.state, previous); assert.equal(hub.catalog.previews.has(preview.id), true);
});
test('a distinct upstream account at the same endpoint and model never silently reuses another account', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: [metadata('same-model')] })), { hub } = create(f);
const first = hub.upsert({ ...draft(server), model: 'same-model', apiKey: 'owned-account-first' });
const preview = await hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned-account-second' });
const result = hub.catalog.apply({ previewId: preview.id, models: ['same-model'] });
assert.equal(result.added.length, 1); assert.deepEqual(result.existing, []);
assert.notEqual(result.added[0], first.id); assert.equal(hub.credential(hub.connection(result.added[0])).key, 'owned-account-second');
assert.equal(hub.credential(hub.connection(first.id)).key, 'owned-account-first');
});
test('source changes invalidate an in-flight child probe and host shutdown cancels directory work and drops transient previews', async t => {
const f = fixture(t); let mode = 'normal', release;
const server = await compatibleFixture(t, async (_url, input) => {
if (mode === 'wait') await new Promise(resolve => { release = resolve; });
return input ? { ...completion([], 'OK'), model: input.model } : { data: [metadata('a'), metadata('b')] };
});
const { provider, hub } = create(f), preview = await hub.modelCatalog().fetch({ draft: draft(server), apiKey: 'owned-stop' });
const result = hub.catalog.apply({ previewId: preview.id, models: ['a', 'b'] }), source = hub.connection(result.added[0]);
mode = 'wait'; const probing = hub.probe(result.added[1]).then(value => ({ value }), error => ({ error }));
await until(() => !!release); hub.toggle(source.id, false); release(); assert.match((await probing).error.message, /变化/);
assert.equal(hub.state.capabilityResults?.[result.added[1]], undefined); hub.toggle(source.id, true); mode = 'normal';
await hub.catalog.fetch({ id: source.id, revision: hub.connection(source.id).revision }); assert.equal(hub.catalog.previews.size, 1);
release = null; mode = 'wait'; const fetching = hub.catalog.fetch({ id: source.id, revision: hub.connection(source.id).revision }).then(value => ({ value }), error => ({ error }));
await until(() => !!release); await provider.abortRequests(); release(); assert.ok((await fetching).error);
assert.equal(hub.catalog.previews.size, 0); assert.equal(hub.catalog.active.size, 0);
await assert.rejects(hub.catalog.fetch({ draft: draft(server), apiKey: 'owned' }), /停止/);
});