Files
Agent-Workbench/tests/model-fidelity.test.mjs
2026-11-01 22:05:08 +08:00

228 lines
20 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import http from 'node:http';
import { createHash } from 'node:crypto';
import { Provider } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Store } from '../src/server/store.mjs';
import { fixture, compatibleFixture, completion, launchWorkbench, until } from './helpers.mjs';
import { completionBody, actualRequest, outputReservation } from '../src/shared/contracts/model-request.mjs';
import { nativeParameterSettings } from '../src/shared/contracts/native-parameters.mjs';
import { retryAfterMilliseconds, ERROR_CAPTURE_LIMIT } from '../src/core/models/upstream-error.mjs';
const policy = { attempts: 3, delays: [1, 1], timeoutMs: 1000, totalTimeoutMs: 2000 };
const config = { mode: 'compatible', baseUrl: 'http://127.0.0.1:1/v1', model: 'owned-fidelity', maxOutputTokens: 512 };
const sha = value => createHash('sha256').update(value).digest('hex');
const sse = value => 'data: ' + JSON.stringify(value) + '\n\n';
function configure(f, provider, url, extra = {}) {
const hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned fidelity', model: 'owned-fidelity', baseUrl: url,
requiresKey: false, ...extra });
hub.select({ type: 'connection', id: connection.id });
return { hub, connection };
}
async function execute(f, provider) {
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, '读取当前项目;不要修改或执行检查');
await runtime.active.get(f.session.id).promise;
return f.session;
}
async function streamServer(t, bytes, headers = {}) {
let requests = 0;
const server = http.createServer(async (req, res) => { for await (const _ of req) {} requests++;
res.writeHead(200, { 'Content-Type': 'text/event-stream', ...headers }); res.end(bytes); });
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
return { url: 'http://127.0.0.1:' + server.address().port + '/v1', count: () => requests };
}
test('HTTP quota retains native code/type/param/request id, bounded original and allowlisted headers, never repeats that connection', async t => {
const f = fixture(t), original = { error: { code: 'insufficient_quota', type: 'billing_error', param: 'account',
message: 'Owned account quota. Echo owned-private-fidelity-key', api_key: 'unregistered-secret-value' }, request_id: 'body-request', usage: { prompt_tokens: 11, completion_tokens: 7 } };
const model = await compatibleFixture(t, () => ({ status: 429, headers: { 'X-Request-ID': 'upstream-owned-quota', 'Retry-After': '120',
'Set-Cookie': 'private-session=never-keep', 'X-Internal-Secret': 'never-keep' }, body: original }));
const provider = new Provider({ retryPolicy: policy }), { hub, connection } = configure(f, provider, model.baseUrl, { apiKey: 'owned-private-fidelity-key' });
await execute(f, provider);
assert.equal(model.requests.length, 1); assert.equal(f.session.status, 'failed');
const failure = f.session.failure, attempt = hub.view().attempts[0];
assert.equal(failure.origin, 'upstream'); assert.equal(failure.kind, 'quota'); assert.equal(failure.code, 'insufficient_quota');
assert.equal(failure.upstream.type, 'billing_error'); assert.equal(failure.upstream.param, 'account');
assert.equal(failure.upstream.requestId, 'upstream-owned-quota'); assert.notEqual(failure.wireAttemptId, failure.upstream.requestId);
assert.equal(failure.retryAfterMs, 120000); assert.equal(failure.retryable, false);
assert.equal(attempt.diagnostic.upstream.artifactId, attempt.responseArtifactId); assert.equal(attempt.status, 'failed');
const raw = f.store.artifactStore.read(f.store.owned('artifact', attempt.responseArtifactId, f.session.id));
assert.ok(raw.includes('insufficient_quota')); assert.ok(raw.includes('[已隐藏]'));
const publicData = JSON.stringify({ failure, attempt });
for (const secret of ['owned-private-fidelity-key', 'unregistered-secret-value', 'never-keep']) assert.ok(!publicData.includes(secret) && !raw.includes(secret));
assert.equal(hub.status(hub.connection(connection.id)).health.blocked, true);
assert.equal(f.store.core.tool.size, 0); assert.equal(f.store.core.run.get(f.session.runId).usage.unknownRequests, 0);
assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 18);
f.store.close(); const reopened = new Store(f.data); try { assert.deepEqual(reopened.session(f.session.id).failure, failure); } finally { reopened.close(); }
});
test('Retry-After above 90 seconds remains exact and a short local budget cannot trigger an early retry', async () => {
let requests = 0;
const provider = new Provider({ retryPolicy: policy, fetch: async () => { requests++; return new Response(JSON.stringify({ error: { code: 'rate_limit_exceeded', message: 'slow down' } }),
{ status: 429, headers: { 'Retry-After': '180', 'X-Request-ID': 'owned-long-wait' } }); } });
await assert.rejects(provider.complete(config, [], null), error => {
assert.equal(error.diagnostic.retryAfterMs, 180000); assert.equal(error.diagnostic.retryDecision, 'wait_budget');
assert.equal(error.diagnostic.limitReason, 'rate'); assert.equal(error.diagnostic.upstream.retryAfter, '180'); return true;
});
assert.equal(requests, 1);
});
test('Retry-After date, expired date, huge value and invalid header retain explicit semantics', async () => {
const now = Date.parse('2026-11-01T00:00:00Z');
assert.equal(retryAfterMilliseconds('Sun, 01 Nov 2026 00:02:00 GMT', now), 120000);
assert.equal(retryAfterMilliseconds('Sat, 31 Oct 2026 00:00:00 GMT', now), 0);
for (const value of ['999999999999999999999999999999999', 'not-a-date']) {
let requests = 0;
const provider = new Provider({ retryPolicy: policy, fetch: async () => { requests++; return new Response('{"error":{"message":"limited"}}', { status: 429, headers: { 'Retry-After': value } }); } });
await assert.rejects(provider.complete(config, [], null), error => { assert.equal(error.diagnostic.upstream.retryAfter, value);
assert.equal(error.diagnostic.upstream.retryAfterInvalid, true); assert.equal(error.diagnostic.retryable, false); return true; });
assert.equal(requests, 1);
}
});
test('unclassified 429 remains unspecified and every bounded actual retry retains its own original error', async t => {
const f = fixture(t), provider = new Provider({ retryPolicy: { ...policy, attempts: 2 }, fetch: async () => new Response('{"error":{"message":"unspecified limit"}}',
{ status: 429, headers: { 'retry-after': '0' } }) });
f.store.saveConfig({ ...config }); await execute(f, provider);
assert.equal(f.session.failure.limitReason, 'unspecified'); assert.equal(f.session.failure.attempts, 2);
const artifacts = [...f.store.core.artifact.values()].filter(value => value.kind === 'model-http-response');
assert.equal(artifacts.length, 2); assert.equal(f.store.core.run.get(f.session.runId).usage.requests, 2);
});
test('oversized HTTP error saves exactly the bounded prefix and labels its incomplete range', async t => {
const f = fixture(t), original = Buffer.from('owned-error\n' + 'x'.repeat(ERROR_CAPTURE_LIMIT * 3));
const provider = new Provider({ retryPolicy: { ...policy, attempts: 1 }, fetch: async () => new Response(original, { status: 502, headers: { 'X-Request-ID': 'owned-large' } }) });
const { hub } = configure(f, provider, config.baseUrl); await execute(f, provider);
const attempt = hub.view().attempts[0], artifact = f.store.owned('artifact', attempt.responseArtifactId, f.session.id);
assert.equal(artifact.bytes, ERROR_CAPTURE_LIMIT); assert.equal(artifact.sha256, sha(original.subarray(0, ERROR_CAPTURE_LIMIT)));
assert.equal(attempt.upstream.capture.complete, false); assert.equal(attempt.upstream.capture.truncated, true);
assert.deepEqual(attempt.upstream.capture.capturedRange, [0, ERROR_CAPTURE_LIMIT]);
assert.equal(attempt.upstream.rawPreview.length, 8000); assert.equal(attempt.upstream.previewTruncated, true);
});
test('Chat SSE error preserves upstream fields, original failed event, partial tool and reported usage without executing or retrying', async t => {
const f = fixture(t), original = sse({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'partial', type: 'function', function: { name: 'propose_file', arguments: '{"path":' } }] } }] })
+ sse({ error: { code: 'rate_limit_exceeded', type: 'rate_limit_error', message: 'Owned error owned-sse-secret' }, usage: { prompt_tokens: 11, completion_tokens: 3 } });
const model = await streamServer(t, original, { 'x-request-id': 'owned-sse-id' });
const provider = new Provider({ retryPolicy: policy }), { hub } = configure(f, provider, model.url, { apiKey: 'owned-sse-secret', streaming: true, capabilityDeclarations: { streaming: 'supported' } });
await execute(f, provider);
assert.equal(model.count(), 1); assert.equal(f.store.core.tool.size, 0); assert.equal(f.session.changes.length, 0);
assert.equal(f.session.failure.kind, 'rate_limit'); assert.equal(f.session.failure.origin, 'upstream'); assert.equal(f.session.failure.retryDecision, 'stream_not_replayed');
assert.equal(f.session.failure.model, 'owned-fidelity'); assert.equal(f.session.failure.endpoint, model.url); assert.equal(f.session.failure.route, '/chat/completions');
assert.equal(f.session.failure.upstream.requestId, 'owned-sse-id'); assert.ok(!JSON.stringify(f.session.failure).includes('owned-sse-secret'));
const attempt = hub.view().attempts[0]; assert.equal(attempt.status, 'failed'); assert.equal(attempt.stream.terminal, false);
assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 14);
assert.equal(f.session.failure.upstream.artifactId, attempt.responseArtifactId);
assert.equal(f.session.failure.upstream.artifactScope, 'sse-stream');
assert.ok(f.store.artifactStore.read(f.store.owned('artifact', attempt.responseArtifactId, f.session.id)).includes('"id":"partial"'));
});
for (const status of ['failed', 'incomplete']) test('Responses ' + status + ' SSE retains original event/response identity and usage', async t => {
const f = fixture(t), terminal = { id: 'resp-owned', status, usage: { input_tokens: 12, output_tokens: 5 },
...(status === 'failed' ? { error: { code: 'insufficient_quota', type: 'billing_error', message: 'Owned exhausted' } } : { incomplete_details: { reason: 'max_output_tokens' } }) };
const model = await streamServer(t, sse({ type: 'response.created', response: { id: 'resp-owned' }, sequence_number: 1 }) + sse({ type: 'response.' + status, response: terminal, sequence_number: 2 }));
const provider = new Provider({ retryPolicy: policy }), { hub } = configure(f, provider, model.url, { protocol: 'responses', streaming: true, capabilityDeclarations: { streaming: 'supported' } });
await execute(f, provider);
assert.equal(model.count(), 1); assert.equal(f.session.failure.origin, 'upstream'); assert.equal(f.session.failure.upstream.responseId, 'resp-owned');
assert.equal(f.session.failure.upstream.eventType, 'response.' + status); assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 17);
assert.equal(hub.view().attempts[0].status, 'failed'); assert.equal(f.store.core.tool.size, 0);
if (status === 'incomplete') assert.equal(f.session.failure.upstream.reason, 'max_output_tokens');
});
test('unsupported successful Responses output retains original JSON and usage with a local protocol origin', async t => {
const f = fixture(t), original = { id: 'resp-audio', object: 'response', status: 'completed', output: [{ type: 'audio', data: 'owned-data' }], usage: { input_tokens: 21, output_tokens: 3 } };
const model = await compatibleFixture(t, () => ({ body: original })), provider = new Provider({ retryPolicy: policy }), { hub } = configure(f, provider, model.baseUrl, { protocol: 'responses' });
await execute(f, provider);
assert.equal(f.session.failure.origin, 'local'); assert.equal(f.session.failure.kind, 'protocol'); assert.equal(model.requests.length, 1);
const attempt = hub.view().attempts[0]; assert.equal(attempt.status, 'failed');
assert.equal(f.store.artifactStore.read(f.store.owned('artifact', attempt.responseArtifactId, f.session.id)), JSON.stringify(original));
assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 24); assert.equal(f.store.core.tool.size, 0);
});
test('HTTP 200 native error is still an upstream failure rather than a fabricated successful answer', async () => {
const provider = new Provider({ retryPolicy: policy, fetch: async () => new Response('{"error":{"code":"insufficient_quota","message":"owned"}}') });
await assert.rejects(provider.complete(config, [], null), error => error.diagnostic.origin === 'upstream' && error.diagnostic.kind === 'quota' && error.diagnostic.httpStatus === 200);
});
test('JSON-escaped credential echoes are redacted in the original preview and capture as well as parsed fields', async () => {
const key = 'owned-quote-"-and-backslash-\\-key', progress = [];
const provider = new Provider({ retryPolicy: policy, fetch: async () => new Response(JSON.stringify({ error: { message: 'Echo ' + key, code: 'invalid_api_key' } }), { status: 401 }) });
provider.setKey(key);
await assert.rejects(provider.complete(config, [], null, undefined, value => progress.push(value)), error => {
assert.ok(!JSON.stringify(error.diagnostic).includes(JSON.stringify(key).slice(1, -1)));
assert.ok(error.diagnostic.upstream.rawPreview.includes('[已隐藏]')); return true;
});
assert.ok(progress.find(value => value.responseCapture).responseCapture.toString('utf8').includes('[已隐藏]'));
});
test('quota failover uses another independent pool member and never replays a failed stream', async t => {
const f = fixture(t), quota = await compatibleFixture(t, () => ({ status: 429, body: { error: { code: 'insufficient_quota', message: 'Owned empty' } } }));
const ready = await compatibleFixture(t, () => completion([], 'Owned completed'));
const provider = new Provider(), { hub, connection } = configure(f, provider, quota.baseUrl);
const other = hub.upsert({ name: 'Owned other', model: 'owned-other', baseUrl: ready.baseUrl, requiresKey: false });
const pool = hub.pool({ name: 'Owned failover', members: [{ connectionId: connection.id }, { connectionId: other.id }], maxAttempts: 3 }); hub.select({ type: 'pool', id: pool.id });
await execute(f, provider);
assert.equal(quota.requests.length, 1); assert.equal(ready.requests.length, 1); assert.equal(hub.view().attempts.length, 2);
assert.deepEqual(hub.view().attempts.map(value => value.status), ['completed', 'failed']);
assert.equal(f.store.core.run.get(f.session.runId).usage.requests, 2);
});
test('native parameters are isolated by pool member, preserve Responses state and bind the original output reservation', () => {
const a = { id: 'a', model: 'a-model', protocol: 'chat-completions', nativeParameters: { reasoning_effort: 'high', temperature: 0.4, seed: 3 }, outputTokenField: 'max_completion_tokens' };
const b = { id: 'b', model: 'b-model', protocol: 'responses', nativeParameters: { reasoning: { effort: 'low' }, text: { verbosity: 'high' }, parallel_tool_calls: false } };
const c = { id: 'c', model: 'c-model', protocol: 'chat-completions' };
const body = completionBody({ ...config, ...a, modelRouting: { members: [a, b, c] } }, [{ role: 'user', content: 'owned' }], []);
const chat = actualRequest(body, a), responses = actualRequest(body, b), plain = actualRequest(body, c);
assert.equal(chat.max_completion_tokens, 512); assert.equal(chat.max_tokens, undefined); assert.equal(outputReservation(chat), 512);
assert.equal(chat.reasoning_effort, 'high'); assert.equal(chat.seed, 3); assert.deepEqual(responses.reasoning, { effort: 'low' });
assert.equal(responses.temperature, undefined); assert.equal(responses.seed, undefined); assert.deepEqual(responses.include, ['reasoning.encrypted_content']);
assert.equal(responses.store, false); assert.equal(plain.reasoning_effort, undefined); assert.equal(plain.max_tokens, 512);
});
test('unsupported native fields and invalid values reject before dispatch instead of silently dropping or overriding constraints', () => {
for (const [protocol, params, field] of [['chat-completions', { model: 'override' }], ['responses', { tools: [] }], ['responses', { reasoning: { effort: 'high', extra: 1 } }],
['responses', { text: { format: { type: 'json' } } }], ['chat-completions', { temperature: '1' }], ['chat-completions', { top_p: 2 }], ['chat-completions', { seed: 1.5 }],
['chat-completions', { parallel_tool_calls: 'true' }], ['chat-completions', { reasoning_effort: 'invented' }], ['chat-completions', {}, 'max_output_tokens'],
['responses', {}, 'max_tokens'], ['responses', []], ['responses', { unknown: 'x'.repeat(8200) }]]) {
assert.throws(() => nativeParameterSettings(protocol, params, field), error => error.status === 400);
}
});
test('native settings for legacy and registered profiles persist without changing credentials or old records and reject stale edits', async t => {
const f = fixture(t), provider = new Provider(); provider.setKey('owned-legacy-key');
const { hub, connection } = configure(f, provider, config.baseUrl, { apiKey: 'owned-profile-key' });
const oldMessages = JSON.stringify(f.session), credential = provider.credentials.get(connection.id), oldPool = JSON.stringify(hub.state.pools);
hub.setNativeParameters({ id: connection.id, revision: connection.revision, nativeParameters: { reasoning_effort: 'medium' }, outputTokenField: 'max_completion_tokens' });
assert.equal(provider.credentials.get(connection.id), credential); assert.equal(JSON.stringify(f.session), oldMessages); assert.equal(JSON.stringify(hub.state.pools), oldPool);
assert.throws(() => hub.setNativeParameters({ id: connection.id, revision: connection.revision, nativeParameters: {} }), error => error.status === 409);
const legacy = hub.connection('legacy-default'); hub.setNativeParameters({ id: legacy.id, revision: legacy.revision, nativeParameters: { temperature: 0.5 } });
assert.equal(provider.apiKey, 'owned-legacy-key'); assert.deepEqual(f.store.config.nativeParameters, { temperature: 0.5 });
f.store.close(); const reopened = new Store(f.data); try { assert.deepEqual(reopened.config.nativeParameters, { temperature: 0.5 });
assert.deepEqual(new Provider().modelHub(reopened).connection(connection.id).nativeParameters, { reasoning_effort: 'medium' }); } finally { reopened.close(); }
});
test('compiled native settings API saves/restores exact parameters and rejects reserved fields without mutating the profile', async t => {
const f = fixture(t); f.store.close(); const host = await launchWorkbench(t, f.data);
const member = await host.api('/model-center/connections', { name: 'Owned API', model: 'owned-api', protocol: 'responses', baseUrl: config.baseUrl, requiresKey: false });
await host.api('/model-center/native-parameters', { id: member.id, revision: member.revision, nativeParameters: { reasoning: { effort: 'high' }, parallel_tool_calls: false } });
const saved = (await host.api('/model-center')).connections.find(value => value.id === member.id);
assert.deepEqual(saved.nativeParameters, { reasoning: { effort: 'high' }, parallel_tool_calls: false });
await assert.rejects(host.api('/model-center/native-parameters', { id: member.id, revision: saved.revision, nativeParameters: { max_output_tokens: 99999 } }), error => error.status === 400);
assert.deepEqual((await host.api('/model-center')).connections.find(value => value.id === member.id), saved);
});
test('legacy connection test sends its saved native fields and selected output limit field without changing the configuration', async t => {
const f = fixture(t), model = await compatibleFixture(t, () => completion([], 'OK')); f.store.close();
const host = await launchWorkbench(t, f.data), original = (await host.api('/model-center')).connections.find(value => value.id === 'legacy-default');
await host.api('/model-center/native-parameters', { id: original.id, revision: original.revision, nativeParameters: { reasoning_effort: 'low' }, outputTokenField: 'max_completion_tokens' });
const before = await host.api('/model-center');
await host.api('/config/test', { baseUrl: model.baseUrl, model: 'owned-probe' });
assert.equal(model.requests.length, 1); assert.equal(model.requests[0].input.reasoning_effort, 'low');
assert.equal(model.requests[0].input.max_completion_tokens, 64); assert.equal(model.requests[0].input.max_tokens, undefined);
assert.deepEqual(await host.api('/model-center'), before);
});