Files
Agent-Workbench/tests/model-center.test.mjs
2026-11-12 09:01:34 +08:00

200 lines
16 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { fixture, compatibleFixture, completion, call, until, launchWorkbench, assertToolPairs } from './helpers.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Store } from '../src/server/store.mjs';
import { ConfigImporter } from '../src/core/models/config-import.mjs';
const add = (hub, server, name, extra = {}) => hub.upsert({ name, model: name, baseUrl: server.baseUrl, requiresKey: true, apiKey: 'owned-' + name, ...extra });
test('model registry keeps original credentials/endpoint intact, rejects stale edits, and forgets only volatile keys on restart', t => {
const f = fixture(t), provider = new Provider(); provider.setKey('owned-legacy');
f.store.saveConfig({ baseUrl: 'http://127.0.0.1:12000/v1', model: 'old' });
const hub = provider.modelHub(f.store), connection = add(hub, { baseUrl: 'http://127.0.0.1:13000/v1' }, 'new', { persistKey: false });
hub.select({ type: 'connection', id: connection.id });
assert.equal(f.store.config.baseUrl, 'http://127.0.0.1:12000/v1'); assert.equal(provider.apiKey, 'owned-legacy');
assert.equal(provider.prepareConfig(f.store.config, f.store).baseUrl, 'http://127.0.0.1:13000/v1');
assert.throws(() => hub.upsert({ ...connection, revision: 0 }), /刷新/);
assert.equal(JSON.stringify(hub.view()).includes('owned-new'), false);
const stored = JSON.stringify(f.store.storage.meta('modelRegistry')); assert.equal(stored.includes('owned-'), false);
f.store.close(); const reopened = new Store(f.data); f.onCleanup(() => reopened.close());
const fresh = new Provider(); fresh.setKey(''); const restored = fresh.modelHub(reopened).view();
assert.equal(restored.connections.find(value => value.id === connection.id).status, 'needs_key');
assert.equal(restored.selected.id, connection.id);
});
test('config import previews exact revisions, keeps secrets volatile, syncs only when enabled and preserves manual edits on conflicts', t => {
const f = fixture(t), provider = new Provider(), hub = provider.modelHub(f.store), importer = new ConfigImporter(hub);
const filename = path.join(f.root, 'connections.json');
const write = (model, name = 'Owned import') => fs.writeFileSync(filename, JSON.stringify({ schema: 1, connections: [{ name, model,
baseUrl: 'http://127.0.0.1:12345/v1', apiKey: 'import-owned-secret' }] }));
write('fixture-one');
const preview = importer.preview('workbench-json', filename);
assert.equal(JSON.stringify(preview).includes('import-owned-secret'), false);
write('fixture-changed'); assert.throws(() => importer.apply(preview.id), /来源配置已变化/);
write('fixture-one');
const accepted = importer.preview('workbench-json', filename), result = importer.apply(accepted.id, undefined, false);
assert.equal(result.imported.length, 1);
const sourceId = Object.keys(hub.state.imports)[0], initial = hub.connection(result.imported[0]);
assert.equal(initial.model, 'fixture-one'); assert.equal(importer.read('workbench-json', filename).entries.length, 1);
assert.equal(JSON.stringify(f.store.storage.meta('modelRegistry')).includes('import-owned-secret'), false);
write('fixture-two'); assert.deepEqual(importer.sync(), []); assert.equal(hub.connection(initial.id).model, 'fixture-one');
importer.toggle(sourceId, true); const synced = importer.sync(); assert.equal(synced[0].imported.length, 1);
assert.equal(hub.connection(initial.id).model, 'fixture-two');
const current = hub.connection(initial.id); hub.upsert({ ...current, name: 'Manual owner edit' });
write('fixture-three'); const conflicted = importer.sync(); assert.deepEqual(conflicted[0].conflicts, ['Manual owner edit']);
assert.equal(hub.connection(initial.id).name, 'Manual owner edit'); assert.equal(hub.connection(initial.id).model, 'fixture-two');
assert.ok(hub.state.imports[sourceId].lastCheckedAt);
importer.toggle(sourceId, false); write('fixture-four'); assert.deepEqual(importer.sync(), []);
assert.equal(hub.connection(initial.id).model, 'fixture-two');
const workbuddy = importer.discover().sources.find(value => value.source === 'workbuddy');
assert.equal(workbuddy.supported, false); assert.equal(workbuddy.available, false);
});
test('real HTTP pool fails over 429 to another model and never repeats completed tools; attempts seal actual request bytes and Run identity', async t => {
const f = fixture(t), bad = await compatibleFixture(t, () => ({ status: 429, headers: { 'retry-after': '2' }, body: { error: { message: 'owned rate limit' } } }));
const good = await compatibleFixture(t, (_url, body) => body.messages.some(value => value.role === 'tool') ? completion([], 'Done') : completion([call('pool_read', 'list_files', { path: '', depth: 1 })]));
const provider = new Provider(), hub = provider.modelHub(f.store), first = add(hub, bad, 'rate-limited'), second = add(hub, good, 'working');
const pool = hub.pool({ name: 'Owned pool', members: [{ connectionId: first.id }, { connectionId: second.id }], sticky: true });
hub.select({ type: 'pool', id: pool.id });
const runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll());
runtime.start(f.session.id, 'List files and finish.');
await until(() => !runtime.isActive(f.session.id), 15000);
assert.equal(f.session.status, 'completed'); assert.equal(bad.requests.length, 1); assert.equal(good.requests.length, 2);
assert.equal(f.session.events.filter(value => value.tool === 'list_files' && value.phase === 'completed').length, 1);
assertToolPairs(f.session.providerMessages);
const attempts = hub.view().attempts; assert.equal(attempts.length, 3); assert.equal(attempts.filter(value => value.status === 'failed').length, 1);
const usage = f.store.core.run.get(f.session.runId).usage;
assert.equal(usage.requests, 3); assert.equal(usage.unknownRequests, 1); assert.equal(usage.totalTokens, 336);
assert.equal(f.store.core.run.get(f.session.runId).consumption.length, 3);
assert.ok(attempts.every(value => value.taskId === f.session.taskId && value.runId === f.session.runId));
for (const attempt of attempts) {
const artifact = f.store.owned('artifact', attempt.requestArtifactId, f.session.id);
const body = JSON.parse(f.store.artifactStore.read(artifact));
assert.equal(body.model, attempt.model); assert.equal(artifact.sha256, attempt.requestSha256);
}
const rateLimit = hub.view().connections.find(value => value.id === first.id).health;
assert.equal(Date.parse(rateLimit.lastFailureAt) + 2000, rateLimit.cooldownUntil, 'Retry-After is reflected in the recorded cooldown even if the test finishes after it expires');
assert.equal(good.requests[0].headers.authorization, 'Bearer owned-working');
assert.equal(bad.requests[0].headers.authorization, 'Bearer owned-rate-limited');
});
test('explicit Run affinity rotates weighted pool members on new Runs; disabled and missing credentials are excluded without requests', async t => {
const f = fixture(t), a = await compatibleFixture(t, () => completion()), b = await compatibleFixture(t, () => completion());
const provider = new Provider(), hub = provider.modelHub(f.store), first = add(hub, a, 'A'), second = add(hub, b, 'B');
const pool = hub.pool({ name: 'Weighted', strategy: 'weighted', affinityScope: 'run', members: [{ connectionId: first.id, weight: 2 }, { connectionId: second.id, weight: 1 }] });
hub.select({ type: 'pool', id: pool.id }); const runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll());
for (let index = 0; index < 3; index++) { runtime.start(f.session.id, 'Finish ' + index); await until(() => !runtime.isActive(f.session.id)); }
assert.equal(a.requests.length, 2); assert.equal(b.requests.length, 1);
hub.toggle(first.id, false);
runtime.start(f.session.id, 'Use enabled member'); await until(() => !runtime.isActive(f.session.id));
assert.equal(a.requests.length, 2); assert.equal(b.requests.length, 2);
hub.toggle(second.id, false); assert.throws(() => runtime.start(f.session.id, 'Nothing available'), /没有可用/);
});
test('401 blocks the credential revision; updating the key restores availability without leaking error echoes', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ status: 401, body: { error: { message: 'invalid owned-secret' } } }));
const provider = new Provider(), hub = provider.modelHub(f.store), connection = add(hub, server, 'secret', { apiKey: 'owned-secret' });
hub.select({ type: 'connection', id: connection.id }); const runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll());
runtime.start(f.session.id, 'Try'); await until(() => !runtime.isActive(f.session.id));
assert.equal(server.requests.length, 1); assert.equal(hub.view().connections.find(value => value.id === connection.id).status, 'needs_attention');
assert.equal(JSON.stringify(f.session).includes('owned-secret'), false); assert.equal(JSON.stringify(hub.view()).includes('owned-secret'), false);
const changed = hub.upsert({ ...connection, apiKey: 'replacement' }); assert.equal(changed.status, 'configured');
});
test('queued request cancellation and in-flight disable release only owned leases and do not dispatch another HTTP request', async t => {
const f = fixture(t); let release;
const server = await compatibleFixture(t, async () => { await new Promise(resolve => { release = resolve; }); return completion(); });
const provider = new Provider(), hub = provider.modelHub(f.store), value = add(hub, server, 'queue'); hub.select({ type: 'connection', id: value.id });
const config = hub.prepare(f.store.config), first = new AbortController(), second = new AbortController();
const payload = { model: value.model, messages: [] };
const execute = (selected, key, body) => provider.request(selected, '/chat/completions', body, first.signal, key, undefined, { attempts: 1 });
const pending = hub.complete(config, payload, first.signal, null, null, execute);
await until(() => !!release);
const queued = hub.complete(config, payload, second.signal, null, null, execute);
await until(() => hub.waiting === 1); second.abort(); await assert.rejects(queued);
hub.toggle(value.id, false); first.abort(); await assert.rejects(pending); release();
assert.equal(server.requests.length, 1); assert.equal(hub.leases.get(value.id), 0); assert.equal(hub.waiting, 0);
const attempt = hub.view().attempts.at(-1);
assert.equal(attempt.status, 'interrupted');
const health = hub.health.get(value.id + ':' + attempt.connectionRevision + ':' + attempt.credentialRevision);
assert.equal(health.state, 'interrupted'); assert.equal(health.cooldownUntil, undefined);
});
test('compiled model-center HTTP API persists public profiles, blocks forged origins, and local operations never invoke a model', async t => {
const f = fixture(t); f.store.close(); const host = await launchWorkbench(t, f.data);
const saved = await host.api('/model-center/connections', { name: 'HTTP own profile', model: 'owned-model', baseUrl: 'http://127.0.0.1:12000/v1', apiKey: 'owned-http-key' });
assert.equal(saved.hasKey, true); assert.equal(saved.apiKey, undefined);
const view = await host.api('/model-center'); assert.ok(view.connections.some(value => value.id === saved.id));
const operations = await host.api('/operations'); assert.equal(operations.modelRequests, 0); assert.ok(operations.checks.some(value => value.id === 'database' && value.status === 'passed'));
const response = await fetch(host.url + '/api/model-center/connections', { method: 'POST', headers: { 'Content-Type': 'application/json', Origin: 'https://unowned.example', 'X-Workbench-Token': host.bootstrap.token }, body: '{}' });
assert.equal(response.status, 403);
assert.equal(fs.readFileSync(path.join(f.data, 'workbench.sqlite')).includes(Buffer.from('owned-http-key')), false);
});
test('model registry disk failures restore routing, pool profiles and credential references without partial updates', t => {
const f = fixture(t), provider = new Provider(); provider.setKey('owned-original-fixture');
const hub = provider.modelHub(f.store), connection = add(hub, { baseUrl: 'http://127.0.0.1:12000/v1' }, 'atomic');
const state = structuredClone(hub.state), credential = provider.credentials.get(connection.id);
const fail = () => { f.store.storage.faultInjector = phase => { if (phase === 'before_commit') throw new Error('fixture failed commit'); }; };
for (const action of [
() => hub.toggle(connection.id, false),
() => hub.upsert({ ...connection, name: 'Uncommitted rename', apiKey: 'uncommitted-fixture-key' }),
() => hub.pool({ name: 'Uncommitted pool', members: [{ connectionId: connection.id }] }),
]) {
fail(); assert.throws(action, /fixture failed commit/);
assert.deepEqual(hub.state, state); assert.deepEqual(f.store.storage.meta('modelRegistry'), state);
assert.equal(provider.credentials.get(connection.id), credential);
assert.equal(provider.apiKey, 'owned-original-fixture');
}
f.store.storage.faultInjector = null;
const disabled = hub.toggle(connection.id, false); assert.equal(disabled.enabled, false);
assert.deepEqual(f.store.storage.meta('modelRegistry'), hub.state);
});
test('model selection and saved configuration commit together and failures preserve the original mode and endpoint', t => {
const f = fixture(t), provider = new Provider(); provider.setKey('owned-legacy-fixture');
const hub = provider.modelHub(f.store), connection = add(hub, { baseUrl: 'http://127.0.0.1:12000/v1' }, 'selected');
const state = structuredClone(hub.state), config = structuredClone(f.store.config);
f.store.storage.faultInjector = phase => { if (phase === 'before_commit') throw new Error('fixture selection failure'); };
assert.throws(() => hub.select({ type: 'connection', id: connection.id }), /fixture selection failure/);
assert.deepEqual(hub.state, state); assert.deepEqual(f.store.config, config);
assert.deepEqual(f.store.storage.meta('modelRegistry'), state); assert.deepEqual(f.store.storage.meta('config'), config);
f.store.storage.faultInjector = null;
hub.select({ type: 'connection', id: connection.id });
const selected = structuredClone(hub.state), selectedConfig = structuredClone(f.store.config);
f.store.storage.faultInjector = phase => { if (phase === 'before_commit') throw new Error('fixture original config failure'); };
assert.throws(() => hub.select({ type: 'legacy' }, { mode: 'demo', model: 'uncommitted-model' }), /fixture original config failure/);
assert.deepEqual(hub.state, selected); assert.deepEqual(f.store.config, selectedConfig);
assert.equal(provider.apiKey, 'owned-legacy-fixture');
});
test('configuration imports roll back the entire batch and retain one usable preview after validation or commit failure', t => {
const f = fixture(t), provider = new Provider(), hub = provider.modelHub(f.store), importer = new ConfigImporter(hub);
const existing = add(hub, { baseUrl: 'http://127.0.0.1:12000/v1' }, 'preexisting');
const before = structuredClone(hub.state), credential = provider.credentials.get(existing.id), filename = path.join(f.root, 'atomic-import.json');
const entries = [1, 2].map(index => ({ id: index, name: 'Fixture import ' + index, model: 'fixture-model',
baseUrl: 'http://127.0.0.1:12345/v1', apiKey: 'owned-import-key-' + index }));
fs.writeFileSync(filename, JSON.stringify({ schema: 1, connections: entries }));
const preview = importer.preview('workbench-json', filename);
f.store.storage.faultInjector = phase => { if (phase === 'before_commit') throw new Error('fixture import failure'); };
assert.throws(() => importer.apply(preview.id, true), /fixture import failure/);
assert.deepEqual(hub.state, before); assert.deepEqual(f.store.storage.meta('modelRegistry'), before);
assert.equal(provider.credentials.size, 1); assert.equal(provider.credentials.get(existing.id), credential);
assert.equal(importer.previews.has(preview.id), true);
f.store.storage.faultInjector = null;
const imported = importer.apply(preview.id, true); assert.equal(imported.imported.length, 2);
assert.equal(importer.previews.has(preview.id), false);
assert.deepEqual(f.store.storage.meta('modelRegistry'), hub.state);
const validState = structuredClone(hub.state), validCredentials = new Map(provider.credentials);
entries[1].apiKey = 'x'.repeat(16001);
fs.writeFileSync(filename, JSON.stringify({ schema: 1, connections: entries }));
const invalid = importer.preview('workbench-json', filename);
assert.throws(() => importer.apply(invalid.id), /凭据格式无效/);
assert.deepEqual(hub.state, validState); assert.deepEqual(provider.credentials, validCredentials);
assert.deepEqual(f.store.storage.meta('modelRegistry'), validState);
});