Files
Agent-Workbench/tests/catalog-refresh.test.mjs
2026-11-07 22:39:22 +08:00

103 lines
8.5 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { fixture, compatibleFixture, launchWorkbench } from './helpers.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Store } from '../src/server/store.mjs';
import { catalogModel } from '../src/shared/contracts/model-catalog.mjs';
import { completionBody, requestVariants } from '../src/shared/contracts/model-request.mjs';
import { modelSnapshot } from '../src/shared/contracts/execution.mjs';
test('catalog keeps published camel-case input capacity separate from an unknown context maximum', () => {
const model = catalogModel({ name: 'owned-model', displayName: 'Owned model', inputTokenLimit: 200000, outputTokenLimit: 128000 });
assert.equal(model.name, 'Owned model');
assert.equal(model.contextLimit, null); assert.equal(model.maxInputTokens, 200000); assert.equal(model.maxOutputTokens, 128000);
const distinct = catalogModel({ id: 'distinct-capacity', context_window: 300000, inputTokenLimit: 200000, outputTokenLimit: 128000 });
assert.equal(distinct.contextLimit, 300000); assert.equal(distinct.maxInputTokens, 200000); assert.equal(distinct.maxOutputTokens, 128000);
const missing = catalogModel({ id: 'no-capacity', outputTokenLimit: 0 });
assert.equal(missing.contextLimit, null); assert.equal(missing.maxInputTokens, null); assert.equal(missing.maxOutputTokens, null);
});
test('refresh updates only selected upstream declarations and future Messages wire, preserving saved credentials and frozen requests', async t => {
const f = fixture(t, { modernDefaults: true });
let maximum = 64000;
const upstream = await compatibleFixture(t, () => ({ data: ['source', 'child', 'untouched'].map(id => ({ id,
max_input_tokens: 200000, max_tokens: maximum, capabilities: { tools: true, thinking: { supported: true }, effort: { high: { supported: true } } } })) }));
const provider = new Provider(), hub = provider.modelHub(f.store);
const source = hub.upsert({ name: 'Owned source', model: 'source', protocol: 'anthropic-messages', baseUrl: upstream.baseUrl,
requiresKey: true, apiKey: 'owned-refresh-key', persistKey: true, maxOutputTokens: 2048 });
const keyObject = provider.credentials.get(source.id), encrypted = structuredClone(f.store.storage.meta('modelCredentials'));
const first = await hub.modelCatalog().fetch({ id: source.id, revision: source.revision });
const result = hub.modelCatalog().apply({ previewId: first.id, models: ['source', 'child', 'untouched'] });
assert.deepEqual(result.existing, [source.id]);
assert.equal(hub.connection(source.id).upstreamModel.maxOutputTokens, 64000);
assert.equal(hub.connection(source.id).revision, source.revision, 'directory facts do not change a credential-bound connection revision');
const child = hub.connection(result.added[0]);
hub.setNativeParameters({ id: child.id, revision: child.revision, nativeParameters: { max_tokens: 123456 } });
hub.upsert({ ...hub.connection(child.id), name: 'My chosen name', capabilityDeclarations: { tools: 'unsupported' }, enabled: false });
hub.select({ type: 'connection', id: source.id });
const frozen = hub.prepare(f.store.config), snapshot = modelSnapshot(frozen);
const frozenBody = completionBody(frozen, [{ role: 'user', content: 'Owned original request' }], []);
const previous = structuredClone(hub.state), childBefore = structuredClone(hub.connection(child.id));
maximum = 256000;
const preview = await hub.catalog.fetch({ id: source.id, revision: source.revision });
const refresh = hub.catalog.apply({ previewId: preview.id, models: ['child', 'source'] });
assert.deepEqual(refresh.added, []); assert.deepEqual(refresh.existing, [child.id, source.id]);
const requestBinding = { credentialRevision: keyObject.revision, contract: { schemaVersion: 1, protocol: 'anthropic-messages',
requestUrl: upstream.baseUrl + '/models', headers: [['anthropic-version', '2023-06-01'], ['content-type', 'application/json'], ['x-api-key', '{{credential}}']] } };
assert.deepEqual(hub.connection(child.id), { ...childBefore, upstreamModel: { ...preview.entries.find(model => model.id === 'child'),
fetchedAt: preview.fetchedAt, endpoint: upstream.baseUrl, requestBinding } });
assert.deepEqual(hub.connection(source.id).upstreamModel.requestBinding, requestBinding);
assert.deepEqual(hub.connection(result.added[1]), previous.connections.find(model => model.id === result.added[1]));
assert.equal(provider.credentials.get(source.id), keyObject); assert.equal(hub.credential(hub.connection(source.id)), keyObject);
assert.deepEqual(f.store.storage.meta('modelCredentials'), encrypted);
assert.equal(hub.vault().status(hub.connection(source.id)).status, 'saved');
assert.equal(hub.connection(source.id).maxOutputTokens, 2048, 'the user declaration remains editable metadata');
assert.deepEqual(modelSnapshot(frozen), snapshot);
assert.equal(requestVariants(frozenBody)[0].max_tokens, 64000);
assert.equal(requestVariants(completionBody(hub.prepare(f.store.config), [{ role: 'user', content: 'Next request' }], []))[0].max_tokens, 256000);
hub.toggle(child.id, true); hub.select({ type: 'connection', id: child.id });
assert.equal(requestVariants(completionBody(hub.prepare(f.store.config), [{ role: 'user', content: 'Explicit output' }], []))[0].max_tokens, 123456);
f.store.close();
const reopened = new Store(f.data); f.onCleanup(() => reopened.close());
const restored = new Provider().modelHub(reopened);
assert.equal(restored.connection(source.id).upstreamModel.maxOutputTokens, 256000);
assert.equal(restored.status(restored.connection(source.id)).hasKey, true);
assert.equal(upstream.requests.length, 2, 'only directory requests were made');
});
test('failed catalog refresh transaction leaves existing metadata, credentials and usable preview intact', async t => {
const f = fixture(t, { modernDefaults: true });
const upstream = await compatibleFixture(t, () => ({ data: [{ id: 'owned', max_tokens: 128000 }] }));
const provider = new Provider(), hub = provider.modelHub(f.store);
const source = hub.upsert({ name: 'Owned', model: 'owned', baseUrl: upstream.baseUrl, apiKey: 'owned-rollback' });
const preview = await hub.modelCatalog().fetch({ id: source.id, revision: source.revision });
const registry = structuredClone(hub.state), key = provider.credentials.get(source.id);
const setMeta = f.store.storage.setMeta.bind(f.store.storage);
f.store.storage.setMeta = (name, value) => { if (name === 'modelRegistry') throw new Error('owned refresh transaction failed'); return setMeta(name, value); };
try { assert.throws(() => hub.catalog.apply({ previewId: preview.id, models: ['owned'] }), /owned refresh transaction failed/); }
finally { f.store.storage.setMeta = setMeta; }
assert.deepEqual(hub.state, registry); assert.equal(provider.credentials.get(source.id), key);
assert.equal(hub.catalog.previews.has(preview.id), true);
hub.catalog.apply({ previewId: preview.id, models: ['owned'] });
assert.equal(hub.connection(source.id).upstreamModel.maxOutputTokens, 128000);
});
test('compiled HTTP catalog refresh returns new upstream facts while preserving connection revision and user parameters', async t => {
const f = fixture(t, { modernDefaults: true }); f.store.close();
let maximum = 64000;
const upstream = await compatibleFixture(t, () => ({ data: [{ id: 'owned-http', max_input_tokens: 200000, max_tokens: maximum }] }));
const host = await launchWorkbench(t, f.data);
const first = await host.api('/model-center/catalog', { draft: { name: 'Owned HTTP', baseUrl: upstream.baseUrl,
protocol: 'anthropic-messages', requiresKey: false } });
const added = await host.api('/model-center/catalog/apply', { previewId: first.id, models: ['owned-http'] });
const before = added.center.connections.find(connection => connection.id === added.added[0]);
maximum = 256000;
const preview = await host.api('/model-center/catalog', { id: before.id, revision: before.revision });
const result = await host.api('/model-center/catalog/apply', { previewId: preview.id, models: ['owned-http'] });
const after = result.center.connections.find(connection => connection.id === before.id);
assert.deepEqual(result.added, []); assert.deepEqual(result.existing, [before.id]);
assert.equal(after.upstreamModel.maxOutputTokens, 256000); assert.equal(after.revision, before.revision);
assert.equal(after.maxOutputTokens, before.maxOutputTokens); assert.deepEqual(after.nativeParameters, before.nativeParameters);
assert.equal(upstream.requests.length, 2); assert.equal(result.center.attempts.length, 0);
});