Files
Agent-Workbench/tests/official-model-capacity.test.mjs

76 lines
5.3 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { compatibleFixture } from './helpers.mjs';
import { Provider } from '../src/server/provider.mjs';
import * as capacity from '../src/shared/contracts/model-capacity.mjs';
import { completionBody, requestVariants } from '../src/shared/contracts/model-request.mjs';
const config = { mode: 'compatible', executionMode: 'unrestricted', capacityMode: 'passthrough',
protocol: 'anthropic-messages', model: 'claude-opus-5-5', contextLimit: 16000, maxOutputTokens: 2048 };
const input = [{ role: 'user', content: 'OWNED START ' + 'original '.repeat(12000) + ' ORIGINAL END' }];
test('published exact Claude model IDs use the official synchronous maximum without clipping original input or changing settings', () => {
for (const model of ['claude-fable-5-1', 'claude-opus-5-5', 'claude-sonnet-5-5', 'claude-haiku-5-5']) {
const value = { ...config, model }, before = structuredClone(value);
assert.equal(completionBody(value, input, []).max_tokens, 128000);
assert.equal(capacity.messagesOutputSetting(value).source, 'official_model_spec');
assert.equal(completionBody(value, input, []).messages[0].content[0].text, input[0].content);
assert.deepEqual(value, before);
}
});
test('official declarations fill only missing catalog fields and never override explicit native output or a matching gateway maximum', () => {
const value = { ...config, upstreamModel: { id: config.model, maxOutputTokens: 96000 } };
assert.equal(capacity.modelCapacityDeclaration(value, 'contextLimit').value, 1000000);
assert.equal(capacity.modelCapacityDeclaration(value, 'maxOutputTokens').source, 'upstream_catalog');
assert.equal(completionBody(value, input, []).max_tokens, 96000);
assert.equal(completionBody({ ...value, nativeParameters: { max_tokens: 300000 } }, input, []).max_tokens, 300000);
assert.equal(capacity.messagesOutputSetting({ ...value, nativeParameters: { max_tokens: 0 } }).value, 0);
assert.equal(capacity.modelCapacityDeclaration({ ...value, upstreamModel: { ...value.upstreamModel, id: 'different' } }, 'maxOutputTokens').value, 128000);
});
test('unpublished names are not matched by family, suffix, casing or branding; optional Chat and Responses output remains absent', () => {
for (const model of ['unknown-model', 'claude-opus-5-5-custom', 'claude-opus-99-1', 'CLAUDE-OPUS-5-5']) {
const value = { ...config, model, brand: 'Anthropic' };
assert.equal(capacity.modelCapacityDeclaration(value, 'maxOutputTokens'), null);
assert.equal(capacity.messagesOutputSetting(value).source, 'connection_declaration');
assert.equal(completionBody(value, input, []).max_tokens, 2048);
}
for (const protocol of ['chat-completions', 'responses']) {
const body = completionBody({ ...config, protocol }, input, []);
for (const field of ['max_tokens', 'max_completion_tokens', 'max_output_tokens']) assert.equal(Object.hasOwn(body, field), false);
}
});
test('each prepared route uses its own model capacity and explicit native choice', () => {
const members = [
{ id: 'official', model: config.model, protocol: 'anthropic-messages', maxOutputTokens: 2048 },
{ id: 'native', model: 'claude-sonnet-5-5', protocol: 'anthropic-messages', nativeParameters: { max_tokens: 320000 }, maxOutputTokens: 2048 },
{ id: 'unknown', model: 'owned-other', protocol: 'anthropic-messages', maxOutputTokens: 65536 },
{ id: 'chat', model: config.model, protocol: 'chat-completions', maxOutputTokens: 2048 },
];
const variants = requestVariants(completionBody({ ...config, modelRouting: { members } }, input, []));
assert.deepEqual(variants.map(body => body.max_tokens), [128000, 320000, 65536, undefined]);
});
test('actual owned Messages HTTP uses official max_tokens, retains upstream refusal and consumption, and never retries', async t => {
let failure = false;
const upstream = await compatibleFixture(t, () => failure
? { status: 400, body: { error: { type: 'upstream_error', message: 'owned gateway rejected this maximum' }, type: 'error' } }
: { body: { id: 'msg_owned', type: 'message', role: 'assistant', model: config.model, stop_reason: 'end_turn',
content: [{ type: 'text', text: 'Owned response' }], usage: { input_tokens: 10, output_tokens: 5 } } });
const provider = new Provider(), events = []; t.after(() => provider.abortRequests());
const actual = { ...config, baseUrl: upstream.baseUrl };
const result = await provider.complete(actual, input, null, undefined, event => events.push(event), null, []);
assert.equal(result.message.content, 'Owned response'); assert.equal(upstream.requests[0].input.max_tokens, 128000);
assert.equal(upstream.requests[0].input.messages[0].content[0].text, input[0].content);
assert.equal(events.find(event => event.consumption).consumption.usage.total_tokens, 15);
failure = true;
await assert.rejects(provider.complete(actual, input, null, undefined, undefined, null, []), cause => {
assert.equal(cause.diagnostic.httpStatus, 400); assert.equal(cause.diagnostic.origin, 'upstream');
assert.equal(cause.diagnostic.retryDecision, 'passthrough');
assert.match(cause.diagnostic.upstream.rawPreview, /owned gateway rejected this maximum/); return true;
});
assert.equal(upstream.requests.length, 2); assert.equal(actual.maxOutputTokens, 2048);
});