Files
Agent-Workbench/tests/model-capacity.test.mjs

113 lines
9.3 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { fixture, compatibleFixture, completion, until } from './helpers.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { DEFAULT_CONFIG, cleanConfig } from '../src/core/storage/legacy-import.mjs';
import { validateRuntimeBudgets } from '../src/core/models/runtime-settings.mjs';
import { capacitySettings } from '../src/shared/contracts/model-capacity.mjs';
import { completionBody } from '../src/shared/contracts/model-request.mjs';
import { budgetFor, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
test('fresh capacity defaults are independent from legacy frozen budgets; valid large capacities are not clamped', () => {
assert.equal(DEFAULT_CONFIG.capacityMode, 'passthrough');
assert.equal(DEFAULT_CONFIG.maxRunTokens, 0); assert.equal(DEFAULT_CONFIG.maxTaskTokens, 0);
assert.ok(DEFAULT_CONFIG.maxOutputTokens >= 16384);
const old = cleanConfig({ contextLimit: 16000, maxOutputTokens: 2048 });
assert.equal(old.capacityMode, 'manual'); assert.equal(old.maxOutputTokens, 2048);
assert.deepEqual(capacitySettings({ contextLimit: 1000000, maxOutputTokens: 131072 }), { contextLimit: 1000000, maxOutputTokens: 131072 });
for (const value of [-1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '64000']) assert.throws(() => capacitySettings({ maxOutputTokens: value }));
assert.deepEqual(validateRuntimeBudgets({ capacityMode: 'model' }), { capacityMode: 'model' });
assert.equal(validateRuntimeBudgets({ maxOutputTokens: 65536 }).capacityMode, 'manual');
assert.throws(() => validateRuntimeBudgets({ capacityMode: 'unlimited' }));
});
test('model mode removes the second global cap; manual mode and connection edits retain explicit limits', t => {
const f = fixture(t), hub = new Provider().modelHub(f.store);
const c = hub.upsert({ name: 'Large', model: 'owned', baseUrl: 'http://127.0.0.1:1', requiresKey: false, contextLimit: 256000, maxOutputTokens: 65536 });
hub.select({ type: 'connection', id: c.id });
const low = { ...f.store.config, contextLimit: 16000, maxOutputTokens: 2048 };
assert.equal(hub.prepare({ ...low, capacityMode: 'model' }).maxOutputTokens, 65536);
assert.equal(hub.prepare({ ...low, capacityMode: 'manual' }).maxOutputTokens, 2048);
const changed = hub.upsert({ ...c, contextLimit: undefined, maxOutputTokens: undefined, name: 'Renamed' });
assert.equal(changed.maxOutputTokens, 65536); assert.equal(changed.contextLimit, 256000);
const b = budgetFor({ ...low, capacityMode: 'model', contextLimit: 256000, maxOutputTokens: 65536 });
assert.ok(b.targetInputTokens > 16000); assert.equal(b.targetInputTokens, b.inputBudget);
});
test('upstream capacities survive a 2048 source connection and are preserved in every protocol wire field', async t => {
const f = fixture(t), server = await compatibleFixture(t, () => ({ data: [{ id: 'owned-large', context_length: 1000000, max_output_tokens: 131072 }] }));
const hub = new Provider().modelHub(f.store), source = hub.upsert({ name: 'Small source', model: 'owned-source', baseUrl: server.baseUrl, requiresKey: false, contextLimit: 16000, maxOutputTokens: 2048 });
const preview = await hub.modelCatalog().fetch({ id: source.id, revision: source.revision });
const result = hub.modelCatalog().apply({ previewId: preview.id, models: ['owned-large'], routing: 'manual', defaultModel: 'owned-large' });
const target = hub.connection(result.added[0]);
assert.equal(target.contextLimit, 1000000); assert.equal(target.maxOutputTokens, 131072);
for (const [protocol, field] of [['chat-completions', 'max_tokens'], ['responses', 'max_output_tokens'], ['anthropic-messages', 'max_tokens']]) {
const body = completionBody({ ...target, protocol, outputTokenField: undefined, parameterProfile: 'auto' }, [{ role: 'user', content: 'Keep the configured capacity.' }], []);
assert.equal(body[field], 131072);
}
assert.equal(server.requests.length, 1, 'only one local catalog request, no model calls');
});
test('actual HTTP sends the selected output capacity and preserves a long complete answer and raw response', async t => {
const f = fixture(t, { modernDefaults: true }), text = 'A'.repeat(30000) + 'END-OF-COMPLETE-ANSWER';
f.store.config.capacityMode = 'model';
const server = await compatibleFixture(t, () => completion([], text));
const provider = new Provider(), hub = provider.modelHub(f.store);
const connection = hub.upsert({ name: 'Owned large answer', model: 'owned', baseUrl: server.baseUrl, requiresKey: false, maxOutputTokens: 32768 });
hub.select({ type: 'connection', id: connection.id });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Reply with reference text only; do not create or modify files.');
await until(() => !runtime.isActive(f.session.id), 15000);
assert.equal(f.session.status, 'completed'); assert.equal(server.requests.length, 1);
assert.equal(server.requests[0].input.max_tokens, 32768);
assert.equal(f.session.providerMessages.at(-1).content, text);
const attempt = hub.attempts.at(-1), artifact = f.store.core.artifact.get(attempt.responseArtifactId);
assert.equal(JSON.parse(f.store.artifactStore.read(artifact)).choices[0].message.content, text);
});
test('automatic routing sends each member own output capacity rather than the smallest pool member allowance', async t => {
const f = fixture(t, { modernDefaults: true }); f.store.config.capacityMode = 'model'; f.store.config.executionMode = 'restricted'; let count = 0;
const server = await compatibleFixture(t, () => ++count === 1 ? { status: 503, body: { error: { message: 'Owned retry boundary' }, usage: { prompt_tokens: 20, completion_tokens: 0, total_tokens: 20 } } } : completion([], 'Complete.'));
const provider = new Provider(), hub = provider.modelHub(f.store);
const members = [4096, 32768].map((maxOutputTokens, index) => hub.upsert({ name: 'Owned route ' + index, model: 'owned-' + index, baseUrl: server.baseUrl, requiresKey: false, contextLimit: 128000, maxOutputTokens }));
const pool = hub.pool({ name: 'Owned unequal capacity', members: members.map(member => ({ connectionId: member.id, weight: 1 })), strategy: 'round-robin', sticky: false, maxAttempts: 2, queueTimeoutMs: 15000 });
hub.select({ type: 'pool', id: pool.id });
f.store.config.maxRunTokens = 400000; f.store.config.maxTaskTokens = 800000;
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Reply only, do not modify files.'); await until(() => !runtime.isActive(f.session.id), 15000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.events.slice(-4)));
assert.equal(server.requests.length, 2);
for (const request of server.requests) assert.equal(request.input.max_tokens, members.find(member => member.model === request.input.model).maxOutputTokens);
assert.equal(new Set(server.requests.map(row => row.input.max_tokens)).size, 2);
assert.equal([...f.store.core.context_plan.values()][0].budget.outputReservation, 32768);
for (const plan of f.store.core.context_plan.values()) verifyPlanEvidence(f.store, f.session, plan);
});
test('mixed windows reserve each route own output without subtracting a different member output from the smallest context', async t => {
const f = fixture(t, { modernDefaults: true }); f.store.config.capacityMode = 'model'; f.store.config.executionMode = 'restricted'; let count = 0;
const server = await compatibleFixture(t, () => ++count === 1
? { status: 503, body: { error: { message: 'Owned mixed-window failover' }, usage: { prompt_tokens: 20, completion_tokens: 0, total_tokens: 20 } } }
: completion([], 'Complete mixed-window answer.'));
const provider = new Provider(), hub = provider.modelHub(f.store);
const members = [[128000, 16384], [1048576, 393216]].map(([contextLimit, maxOutputTokens], index) => hub.upsert({
name: 'Owned mixed window ' + index, model: 'mixed-' + index, baseUrl: server.baseUrl, requiresKey: false, contextLimit, maxOutputTokens,
}));
const pool = hub.pool({ name: 'Owned mixed windows', members: members.map(member => ({ connectionId: member.id })), sticky: false, maxAttempts: 2 });
hub.select({ type: 'pool', id: pool.id });
f.store.config.maxRunTokens = 2000000; f.store.config.maxTaskTokens = 10000000;
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Answer only; do not modify files or run checks.');
await until(() => !runtime.isActive(f.session.id), 15000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.failure));
assert.equal(server.requests.length, 2);
for (const request of server.requests) assert.equal(request.input.max_tokens, members.find(member => member.model === request.input.model).maxOutputTokens);
const budget = [...f.store.core.context_plan.values()][0].budget;
assert.equal(budget.inputBudget, 128000 - 16384 - budget.safetyTokens);
assert.equal(budget.outputReservation, 393216);
const prepared = hub.prepare(f.store.config);
const smallCumulative = budgetFor(prepared, { budget: { maxTokens: 200000 } });
assert.ok(smallCumulative.inputBudget < 0); assert.equal(smallCumulative.limitingReason, 'token_limit');
for (const plan of f.store.core.context_plan.values()) verifyPlanEvidence(f.store, f.session, plan);
});