Files
Agent-Workbench/tests/model-parameter-fidelity.test.mjs

195 lines
14 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import http from 'node:http';
import { changeReasoningSetting, modelParameterProfile, assertModelParameterSupport, reasoningSetting } from '../src/shared/contracts/model-parameter-profiles.mjs';
import { officialModelParameters } from '../src/shared/contracts/official-model-parameters.mjs';
import { catalogModel } from '../src/shared/contracts/model-catalog.mjs';
import { completionBody, actualRequest } from '../src/shared/contracts/model-request.mjs';
import { nativeParameterSettings } from '../src/shared/contracts/native-parameters.mjs';
import { Provider } from '../src/server/provider.mjs';
const open = { executionMode: 'unrestricted', capacityMode: 'passthrough', mode: 'compatible', baseUrl: 'http://127.0.0.1:1/v1' };
const messages = { ...open, protocol: 'anthropic-messages', model: 'claude-opus-5-5' };
const responses = { ...open, protocol: 'responses', model: 'owned-responses' };
const chat = { ...open, protocol: 'chat-completions', model: 'owned-chat' };
const format = { type: 'json_schema', schema: { type: 'object', properties: { result: { type: 'string' } } } };
test('effort edits and reset preserve summary, format and unrelated nested native fields', () => {
for (const [connection, key, siblings] of [
[responses, 'reasoning', { summary: 'auto', opaque_option: { audit: ['unchanged'] } }],
[messages, 'output_config', { format, opaque_option: { audit: ['unchanged'] } }],
]) {
const original = { [key]: { effort: 'high', ...structuredClone(siblings) }, metadata: { owner: 'owned-test' } };
const snapshot = structuredClone(original);
const changed = changeReasoningSetting(connection, original, 'effort', 'max');
assert.deepEqual(changed, { ...snapshot, [key]: { effort: 'max', ...siblings } });
assert.deepEqual(changeReasoningSetting(connection, changed, 'effort', ''), { ...snapshot, [key]: siblings });
assert.deepEqual(original, snapshot);
assert.notEqual(changed[key].opaque_option, original[key].opaque_option);
assert.deepEqual(changeReasoningSetting(connection, { [key]: { effort: 'high' } }, 'effort', ''), {});
}
});
test('Messages mode/default/budget controls keep independent effort and thinking display', () => {
const connection = { ...messages, parameterProfile: 'anthropic' };
const original = { output_config: { effort: 'high', format }, thinking: { type: 'enabled', budget_tokens: 4096, display: 'summarized', custom: { stable: true } } };
const snapshot = structuredClone(original);
for (const mode of ['adaptive', 'disabled', 'between_tools']) {
const changed = changeReasoningSetting(connection, original, 'mode', mode);
assert.deepEqual(changed, { output_config: snapshot.output_config, thinking: { type: mode, display: 'summarized', custom: { stable: true } } });
}
const defaultMode = changeReasoningSetting(connection, original, 'mode', 'default');
assert.deepEqual(defaultMode, snapshot, 'Unknown default must not strip required type or unrelated fields');
assert.equal(reasoningSetting(connection, original).canResetMode, false);
assert.deepEqual(changeReasoningSetting(messages, original, 'mode', 'default'), {
output_config: snapshot.output_config, thinking: { type: 'adaptive', display: 'summarized', custom: { stable: true } },
});
const contradicted = { ...messages, upstreamModel: catalogModel({ id: messages.model, capabilities: { thinking: { types: { adaptive: false } } } }) };
assert.equal(reasoningSetting(contradicted, original).canResetMode, false);
assert.deepEqual(changeReasoningSetting(contradicted, original, 'mode', 'default'), snapshot);
assert.deepEqual(changeReasoningSetting(connection, original, 'budget', 8192), { ...snapshot, thinking: { ...snapshot.thinking, budget_tokens: 8192 } });
assert.deepEqual(changeReasoningSetting(connection, { thinking: { type: 'adaptive', display: 'summarized' } }, 'mode', 'enabled'), {
thinking: { type: 'enabled', display: 'summarized', budget_tokens: 1024 },
});
assert.deepEqual(changeReasoningSetting(connection, { thinking: { type: 'enabled', budget_tokens: 1024 } }, 'mode', 'default'), {});
assert.deepEqual(original, snapshot);
});
test('Responses on/off/default and dialect changes retain sibling fields', () => {
const original = { reasoning: { effort: 'high', summary: 'auto' }, thinking: { type: 'enabled', display: 'custom-visible' } };
assert.deepEqual(changeReasoningSetting(responses, original, 'mode', 'disabled'), {
reasoning: { effort: 'none', summary: 'auto' }, thinking: original.thinking,
});
assert.deepEqual(changeReasoningSetting(responses, original, 'mode', 'default'), { reasoning: { summary: 'auto' }, thinking: original.thinking });
assert.deepEqual(changeReasoningSetting(responses, { reasoning: { summary: 'auto' } }, 'mode', 'enabled'), { reasoning: { summary: 'auto', effort: 'medium' } });
});
for (const [model, modes, defaultEffort, conditionalMode] of [
['claude-fable-5-1', ['adaptive'], 'high', null],
['claude-opus-5-5', ['adaptive'], 'medium', null],
['claude-sonnet-5-5', ['adaptive', 'between_tools'], 'high', 'between_tools'],
['claude-haiku-5-5', ['disabled', 'adaptive'], 'medium', 'disabled'],
]) test('exact official Messages profile for ' + model, () => {
const profile = modelParameterProfile({ ...messages, model });
assert.deepEqual(profile.modes, modes);
assert.deepEqual(profile.efforts, ['low', 'medium', 'high', 'xhigh', 'max']);
assert.equal(profile.defaultEffort, defaultEffort);
assert.equal(profile.sourceKind, 'official_model_spec');
assert.equal(profile.officialSource.url, 'https://platform.claude.com/docs/en/build-with-claude/thinking');
assert.deepEqual(profile.modeEfforts, conditionalMode ? { [conditionalMode]: ['low', 'medium', 'high'] } : {});
assert.equal(profile.modes.includes('enabled'), false);
assert.equal(Object.isFrozen(officialModelParameters(model).modes), true);
});
test('exact-ID fallback does not guess aliases or other protocols; explicit compatibility and native values remain available', () => {
assert.equal(officialModelParameters('claude-opus-5-5-custom'), null);
assert.equal(officialModelParameters('claude-opus-5-5-20261106'), null);
assert.equal(modelParameterProfile({ ...chat, model: messages.model }).sourceKind, 'compatibility_profile');
const manual = { ...messages, parameterProfile: 'anthropic' };
assert.equal(modelParameterProfile(manual).modes.includes('enabled'), true);
assert.equal(modelParameterProfile(manual).officialSource, null);
const native = { thinking: { type: 'enabled', budget_tokens: 1024, display: 'summarized' }, output_config: { effort: 'max', format } };
assert.deepEqual(nativeParameterSettings(messages.protocol, native, undefined, 'auto', open).nativeParameters, native);
const body = completionBody({ ...messages, nativeParameters: native }, [{ role: 'user', content: 'owned' }], []);
assert.deepEqual(body.thinking, native.thinking);
assert.deepEqual(body.output_config, native.output_config);
assert.equal(body.max_tokens, 128000);
});
test('catalog preserves between_tools and explicit negative modes/efforts instead of treating them as absent', () => {
const item = { id: 'owned-model', capabilities: { thinking: { supported: true, types: {
adaptive: { supported: false }, enabled: false, disabled: false, between_tools: { supported: true },
} }, effort: { supported: true, low: false, medium: false, high: false, xhigh: false, max: false } } };
const upstreamModel = catalogModel(item), connection = { ...messages, model: item.id, upstreamModel };
assert.deepEqual(upstreamModel.modes, ['between_tools']);
assert.equal(upstreamModel.modeSupport.adaptive, 'unsupported');
assert.equal(upstreamModel.effortSupport.high, 'unsupported');
assert.deepEqual(modelParameterProfile(connection).modes, ['between_tools']);
assert.deepEqual(modelParameterProfile(connection).efforts, []);
const allNegative = catalogModel({ ...item, capabilities: { ...item.capabilities,
thinking: { supported: true, types: Object.fromEntries(['adaptive', 'disabled', 'enabled', 'between_tools'].map(mode => [mode, false])) } } });
assert.deepEqual(modelParameterProfile({ ...connection, upstreamModel: allNegative }).modes, []);
assert.throws(() => assertModelParameterSupport({ ...connection, upstreamModel: allNegative }, { thinking: { type: 'adaptive' } }), /不支持/);
const absent = catalogModel({ id: item.id });
assert.equal(modelParameterProfile({ ...connection, upstreamModel: absent }).modes.includes('enabled'), true);
const officialAbsent = modelParameterProfile({ ...messages, upstreamModel: catalogModel({ id: messages.model }) });
assert.deepEqual(officialAbsent.modes, ['adaptive']);
});
test('stale catalog identity is ignored; actual upstream positive and negative facts retain their provenance', () => {
const stale = catalogModel({ id: 'different-model', capabilities: { thinking: false, effort: false } });
assert.deepEqual(modelParameterProfile({ ...messages, upstreamModel: stale }).modes, ['adaptive']);
const upstreamModel = catalogModel({ id: messages.model, capabilities: { thinking: { supported: true, types: { enabled: true, adaptive: false } } } });
const profile = modelParameterProfile({ ...messages, upstreamModel });
assert.equal(profile.sourceKind, 'upstream_model_list');
assert.deepEqual(profile.modes, ['enabled']);
});
test('explicit stream_options survive each routed Chat request without mutation; absence alone adds usage default', () => {
for (const stream_options of [{ include_usage: false, include_obfuscation: false }, {}, null]) {
const nativeParameters = { stream_options, custom: { preserve: ['owned'] } }, original = structuredClone(nativeParameters);
const member = { id: 'owned-member', ...chat, streaming: true, nativeParameters };
const planned = completionBody({ ...chat, modelRouting: { members: [member] } }, [{ role: 'user', content: 'owned' }], []);
const body = actualRequest(planned, member);
assert.deepEqual(body.stream_options, stream_options);
assert.deepEqual(body.custom, original.custom);
assert.deepEqual(nativeParameters, original);
assert.equal(body.max_tokens, undefined);
assert.equal(body.max_completion_tokens, undefined);
}
assert.deepEqual(completionBody({ ...chat, streaming: true }, [], []).stream_options, { include_usage: true });
assert.equal(completionBody(chat, [], []).stream_options, undefined);
});
test('actual local HTTP wire preserves edited objects, explicit stream options and provider errors without retry', async t => {
const requests = [];
const server = http.createServer(async (req, res) => {
const chunks = []; for await (const chunk of req) chunks.push(chunk);
const body = JSON.parse(Buffer.concat(chunks)); requests.push({ route: req.url, body });
if (body.model === 'owned-error') {
res.writeHead(400, { 'content-type': 'application/json', 'request-id': 'owned-fidelity-400' });
res.end(JSON.stringify({ error: { type: 'owned_upstream_error', message: 'owned original upstream rejection' } }));
} else if (req.url.endsWith('/chat/completions')) {
res.writeHead(200, { 'content-type': 'text/event-stream' });
res.end('data: ' + JSON.stringify({ model: body.model, choices: [{ index: 0, delta: { content: 'owned local answer' }, finish_reason: 'stop' }] }) + '\n\ndata: [DONE]\n\n');
} else if (req.url.endsWith('/responses')) {
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({ id: 'owned-resp', model: body.model, status: 'completed', output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'owned local answer' }] }], usage: { input_tokens: 7, output_tokens: 3 } }));
} else {
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({ id: 'owned-msg', type: 'message', role: 'assistant', model: body.model, content: [{ type: 'text', text: 'owned local answer' }], stop_reason: 'end_turn', usage: { input_tokens: 7, output_tokens: 3 } }));
}
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
// Do not construct Provider: that constructor reads the optional real env Key.
const provider = Object.create(Provider.prototype); provider.apiKey = '';
provider.fetcher = (url, options) => { assert.equal(new URL(url).hostname, '127.0.0.1'); return fetch(url, options); };
const baseUrl = 'http://127.0.0.1:' + server.address().port + '/v1';
const nativeMessages = changeReasoningSetting(messages, { output_config: { effort: 'high', format }, thinking: { type: 'adaptive', display: 'summarized' } }, 'effort', 'max');
const nativeResponses = changeReasoningSetting(responses, { reasoning: { summary: 'auto' }, text: { format } }, 'effort', 'high');
const configs = [
{ ...chat, streaming: true, nativeParameters: { stream_options: { include_usage: false, include_obfuscation: false } } },
{ ...responses, nativeParameters: nativeResponses },
{ ...messages, nativeParameters: nativeMessages },
];
for (const config of configs) {
const result = await provider.complete({ ...config, baseUrl }, [{ role: 'user', content: 'owned fixture only' }], undefined, undefined, undefined, null, []);
assert.equal(result.message.content, 'owned local answer');
const wire = requests.at(-1).body;
for (const [key, value] of Object.entries(config.nativeParameters)) assert.deepEqual(wire[key], value);
assert.equal(wire.max_tokens, config.protocol === 'anthropic-messages' ? 128000 : undefined);
assert.equal(wire.max_output_tokens, undefined);
if (config.streaming) assert.equal(result.usage, null, 'missing usage stays unknown');
}
const progress = [];
await assert.rejects(provider.complete({ ...messages, baseUrl, model: 'owned-error', nativeParameters: { max_tokens: 4096 } },
[{ role: 'user', content: 'owned fixture only' }], undefined, undefined, value => progress.push(value), null, []), cause => {
assert.match(cause.message, /owned original upstream rejection/);
assert.equal(cause.diagnostic.upstream.requestId, 'owned-fidelity-400');
return true;
});
assert.equal(requests.length, 4);
assert.equal(progress.find(value => value.consumption)?.consumption.usageStatus, 'unknown');
});