import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import http from 'node:http'; import { createHash } from 'node:crypto'; import { Provider } from '../src/server/provider.mjs'; import { Runtime } from '../src/server/runtime.mjs'; import { Store } from '../src/server/store.mjs'; import { fixture, compatibleFixture, completion, launchWorkbench, until } from './helpers.mjs'; import { completionBody, actualRequest, outputReservation } from '../src/shared/contracts/model-request.mjs'; import { nativeParameterSettings } from '../src/shared/contracts/native-parameters.mjs'; import { retryAfterMilliseconds, ERROR_CAPTURE_LIMIT } from '../src/core/models/upstream-error.mjs'; const policy = { attempts: 3, delays: [1, 1], timeoutMs: 1000, totalTimeoutMs: 2000 }; const config = { mode: 'compatible', baseUrl: 'http://127.0.0.1:1/v1', model: 'owned-fidelity', maxOutputTokens: 512 }; const sha = value => createHash('sha256').update(value).digest('hex'); const sse = value => 'data: ' + JSON.stringify(value) + '\n\n'; function configure(f, provider, url, extra = {}) { const hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned fidelity', model: 'owned-fidelity', baseUrl: url, requiresKey: false, ...extra }); hub.select({ type: 'connection', id: connection.id }); return { hub, connection }; } async function execute(f, provider) { const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '读取当前项目;不要修改或执行检查'); await runtime.active.get(f.session.id).promise; return f.session; } async function streamServer(t, bytes, headers = {}) { let requests = 0; const server = http.createServer(async (req, res) => { for await (const _ of req) {} requests++; res.writeHead(200, { 'Content-Type': 'text/event-stream', ...headers }); res.end(bytes); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); })); return { url: 'http://127.0.0.1:' + server.address().port + '/v1', count: () => requests }; } test('HTTP quota retains native code/type/param/request id, bounded original and allowlisted headers, never repeats that connection', async t => { const f = fixture(t), original = { error: { code: 'insufficient_quota', type: 'billing_error', param: 'account', message: 'Owned account quota. Echo owned-private-fidelity-key', api_key: 'unregistered-secret-value' }, request_id: 'body-request', usage: { prompt_tokens: 11, completion_tokens: 7 } }; const model = await compatibleFixture(t, () => ({ status: 429, headers: { 'X-Request-ID': 'upstream-owned-quota', 'Retry-After': '120', 'Set-Cookie': 'private-session=never-keep', 'X-Internal-Secret': 'never-keep' }, body: original })); const provider = new Provider({ retryPolicy: policy }), { hub, connection } = configure(f, provider, model.baseUrl, { apiKey: 'owned-private-fidelity-key' }); await execute(f, provider); assert.equal(model.requests.length, 1); assert.equal(f.session.status, 'failed'); const failure = f.session.failure, attempt = hub.view().attempts[0]; assert.equal(failure.origin, 'upstream'); assert.equal(failure.kind, 'quota'); assert.equal(failure.code, 'insufficient_quota'); assert.equal(failure.upstream.type, 'billing_error'); assert.equal(failure.upstream.param, 'account'); assert.equal(failure.upstream.requestId, 'upstream-owned-quota'); assert.notEqual(failure.wireAttemptId, failure.upstream.requestId); assert.equal(failure.retryAfterMs, 120000); assert.equal(failure.retryable, false); assert.equal(attempt.diagnostic.upstream.artifactId, attempt.responseArtifactId); assert.equal(attempt.status, 'failed'); const raw = f.store.artifactStore.read(f.store.owned('artifact', attempt.responseArtifactId, f.session.id)); assert.ok(raw.includes('insufficient_quota')); assert.ok(raw.includes('[已隐藏]')); const publicData = JSON.stringify({ failure, attempt }); for (const secret of ['owned-private-fidelity-key', 'unregistered-secret-value', 'never-keep']) assert.ok(!publicData.includes(secret) && !raw.includes(secret)); assert.equal(hub.status(hub.connection(connection.id)).health.blocked, true); assert.equal(f.store.core.tool.size, 0); assert.equal(f.store.core.run.get(f.session.runId).usage.unknownRequests, 0); assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 18); f.store.close(); const reopened = new Store(f.data); try { assert.deepEqual(reopened.session(f.session.id).failure, failure); } finally { reopened.close(); } }); test('Retry-After above 90 seconds remains exact and a short local budget cannot trigger an early retry', async () => { let requests = 0; const provider = new Provider({ retryPolicy: policy, fetch: async () => { requests++; return new Response(JSON.stringify({ error: { code: 'rate_limit_exceeded', message: 'slow down' } }), { status: 429, headers: { 'Retry-After': '180', 'X-Request-ID': 'owned-long-wait' } }); } }); await assert.rejects(provider.complete(config, [], null), error => { assert.equal(error.diagnostic.retryAfterMs, 180000); assert.equal(error.diagnostic.retryDecision, 'wait_budget'); assert.equal(error.diagnostic.limitReason, 'rate'); assert.equal(error.diagnostic.upstream.retryAfter, '180'); return true; }); assert.equal(requests, 1); }); test('Retry-After date, expired date, huge value and invalid header retain explicit semantics', async () => { const now = Date.parse('2026-11-01T00:00:00Z'); assert.equal(retryAfterMilliseconds('Sun, 01 Nov 2026 00:02:00 GMT', now), 120000); assert.equal(retryAfterMilliseconds('Sat, 31 Oct 2026 00:00:00 GMT', now), 0); for (const value of ['999999999999999999999999999999999', 'not-a-date']) { let requests = 0; const provider = new Provider({ retryPolicy: policy, fetch: async () => { requests++; return new Response('{"error":{"message":"limited"}}', { status: 429, headers: { 'Retry-After': value } }); } }); await assert.rejects(provider.complete(config, [], null), error => { assert.equal(error.diagnostic.upstream.retryAfter, value); assert.equal(error.diagnostic.upstream.retryAfterInvalid, true); assert.equal(error.diagnostic.retryable, false); return true; }); assert.equal(requests, 1); } }); test('unclassified 429 remains unspecified and every bounded actual retry retains its own original error', async t => { const f = fixture(t), provider = new Provider({ retryPolicy: { ...policy, attempts: 2 }, fetch: async () => new Response('{"error":{"message":"unspecified limit"}}', { status: 429, headers: { 'retry-after': '0' } }) }); f.store.saveConfig({ ...config }); await execute(f, provider); assert.equal(f.session.failure.limitReason, 'unspecified'); assert.equal(f.session.failure.attempts, 2); const artifacts = [...f.store.core.artifact.values()].filter(value => value.kind === 'model-http-response'); assert.equal(artifacts.length, 2); assert.equal(f.store.core.run.get(f.session.runId).usage.requests, 2); }); test('oversized HTTP error saves exactly the bounded prefix and labels its incomplete range', async t => { const f = fixture(t), original = Buffer.from('owned-error\n' + 'x'.repeat(ERROR_CAPTURE_LIMIT * 3)); const provider = new Provider({ retryPolicy: { ...policy, attempts: 1 }, fetch: async () => new Response(original, { status: 502, headers: { 'X-Request-ID': 'owned-large' } }) }); const { hub } = configure(f, provider, config.baseUrl); await execute(f, provider); const attempt = hub.view().attempts[0], artifact = f.store.owned('artifact', attempt.responseArtifactId, f.session.id); assert.equal(artifact.bytes, ERROR_CAPTURE_LIMIT); assert.equal(artifact.sha256, sha(original.subarray(0, ERROR_CAPTURE_LIMIT))); assert.equal(attempt.upstream.capture.complete, false); assert.equal(attempt.upstream.capture.truncated, true); assert.deepEqual(attempt.upstream.capture.capturedRange, [0, ERROR_CAPTURE_LIMIT]); assert.equal(attempt.upstream.rawPreview.length, 8000); assert.equal(attempt.upstream.previewTruncated, true); }); test('Chat SSE error preserves upstream fields, original failed event, partial tool and reported usage without executing or retrying', async t => { const f = fixture(t), original = sse({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'partial', type: 'function', function: { name: 'propose_file', arguments: '{"path":' } }] } }] }) + sse({ error: { code: 'rate_limit_exceeded', type: 'rate_limit_error', message: 'Owned error owned-sse-secret' }, usage: { prompt_tokens: 11, completion_tokens: 3 } }); const model = await streamServer(t, original, { 'x-request-id': 'owned-sse-id' }); const provider = new Provider({ retryPolicy: policy }), { hub } = configure(f, provider, model.url, { apiKey: 'owned-sse-secret', streaming: true, capabilityDeclarations: { streaming: 'supported' } }); await execute(f, provider); assert.equal(model.count(), 1); assert.equal(f.store.core.tool.size, 0); assert.equal(f.session.changes.length, 0); assert.equal(f.session.failure.kind, 'rate_limit'); assert.equal(f.session.failure.origin, 'upstream'); assert.equal(f.session.failure.retryDecision, 'stream_not_replayed'); assert.equal(f.session.failure.model, 'owned-fidelity'); assert.equal(f.session.failure.endpoint, model.url); assert.equal(f.session.failure.route, '/chat/completions'); assert.equal(f.session.failure.upstream.requestId, 'owned-sse-id'); assert.ok(!JSON.stringify(f.session.failure).includes('owned-sse-secret')); const attempt = hub.view().attempts[0]; assert.equal(attempt.status, 'failed'); assert.equal(attempt.stream.terminal, false); assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 14); assert.equal(f.session.failure.upstream.artifactId, attempt.responseArtifactId); assert.equal(f.session.failure.upstream.artifactScope, 'sse-stream'); assert.ok(f.store.artifactStore.read(f.store.owned('artifact', attempt.responseArtifactId, f.session.id)).includes('"id":"partial"')); }); for (const status of ['failed', 'incomplete']) test('Responses ' + status + ' SSE retains original event/response identity and usage', async t => { const f = fixture(t), terminal = { id: 'resp-owned', status, usage: { input_tokens: 12, output_tokens: 5 }, ...(status === 'failed' ? { error: { code: 'insufficient_quota', type: 'billing_error', message: 'Owned exhausted' } } : { incomplete_details: { reason: 'max_output_tokens' } }) }; const model = await streamServer(t, sse({ type: 'response.created', response: { id: 'resp-owned' }, sequence_number: 1 }) + sse({ type: 'response.' + status, response: terminal, sequence_number: 2 })); const provider = new Provider({ retryPolicy: policy }), { hub } = configure(f, provider, model.url, { protocol: 'responses', streaming: true, capabilityDeclarations: { streaming: 'supported' } }); await execute(f, provider); assert.equal(model.count(), 1); assert.equal(f.session.failure.origin, 'upstream'); assert.equal(f.session.failure.upstream.responseId, 'resp-owned'); assert.equal(f.session.failure.upstream.eventType, 'response.' + status); assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 17); assert.equal(hub.view().attempts[0].status, 'failed'); assert.equal(f.store.core.tool.size, 0); if (status === 'incomplete') assert.equal(f.session.failure.upstream.reason, 'max_output_tokens'); }); test('unsupported successful Responses output retains original JSON and usage with a local protocol origin', async t => { const f = fixture(t), original = { id: 'resp-audio', object: 'response', status: 'completed', output: [{ type: 'audio', data: 'owned-data' }], usage: { input_tokens: 21, output_tokens: 3 } }; const model = await compatibleFixture(t, () => ({ body: original })), provider = new Provider({ retryPolicy: policy }), { hub } = configure(f, provider, model.baseUrl, { protocol: 'responses' }); await execute(f, provider); assert.equal(f.session.failure.origin, 'local'); assert.equal(f.session.failure.kind, 'protocol'); assert.equal(model.requests.length, 1); const attempt = hub.view().attempts[0]; assert.equal(attempt.status, 'failed'); assert.equal(f.store.artifactStore.read(f.store.owned('artifact', attempt.responseArtifactId, f.session.id)), JSON.stringify(original)); assert.equal(f.store.core.run.get(f.session.runId).usage.totalTokens, 24); assert.equal(f.store.core.tool.size, 0); }); test('HTTP 200 native error is still an upstream failure rather than a fabricated successful answer', async () => { const provider = new Provider({ retryPolicy: policy, fetch: async () => new Response('{"error":{"code":"insufficient_quota","message":"owned"}}') }); await assert.rejects(provider.complete(config, [], null), error => error.diagnostic.origin === 'upstream' && error.diagnostic.kind === 'quota' && error.diagnostic.httpStatus === 200); }); test('JSON-escaped credential echoes are redacted in the original preview and capture as well as parsed fields', async () => { const key = 'owned-quote-"-and-backslash-\\-key', progress = []; const provider = new Provider({ retryPolicy: policy, fetch: async () => new Response(JSON.stringify({ error: { message: 'Echo ' + key, code: 'invalid_api_key' } }), { status: 401 }) }); provider.setKey(key); await assert.rejects(provider.complete(config, [], null, undefined, value => progress.push(value)), error => { assert.ok(!JSON.stringify(error.diagnostic).includes(JSON.stringify(key).slice(1, -1))); assert.ok(error.diagnostic.upstream.rawPreview.includes('[已隐藏]')); return true; }); assert.ok(progress.find(value => value.responseCapture).responseCapture.toString('utf8').includes('[已隐藏]')); }); test('quota failover uses another independent pool member and never replays a failed stream', async t => { const f = fixture(t), quota = await compatibleFixture(t, () => ({ status: 429, body: { error: { code: 'insufficient_quota', message: 'Owned empty' } } })); const ready = await compatibleFixture(t, () => completion([], 'Owned completed')); const provider = new Provider(), { hub, connection } = configure(f, provider, quota.baseUrl); const other = hub.upsert({ name: 'Owned other', model: 'owned-other', baseUrl: ready.baseUrl, requiresKey: false }); const pool = hub.pool({ name: 'Owned failover', members: [{ connectionId: connection.id }, { connectionId: other.id }], maxAttempts: 3 }); hub.select({ type: 'pool', id: pool.id }); await execute(f, provider); assert.equal(quota.requests.length, 1); assert.equal(ready.requests.length, 1); assert.equal(hub.view().attempts.length, 2); assert.deepEqual(hub.view().attempts.map(value => value.status), ['completed', 'failed']); assert.equal(f.store.core.run.get(f.session.runId).usage.requests, 2); }); test('native parameters are isolated by pool member, preserve Responses state and bind the original output reservation', () => { const a = { id: 'a', model: 'a-model', protocol: 'chat-completions', nativeParameters: { reasoning_effort: 'high', temperature: 0.4, seed: 3 }, outputTokenField: 'max_completion_tokens' }; const b = { id: 'b', model: 'b-model', protocol: 'responses', nativeParameters: { reasoning: { effort: 'low' }, text: { verbosity: 'high' }, parallel_tool_calls: false } }; const c = { id: 'c', model: 'c-model', protocol: 'chat-completions' }; const body = completionBody({ ...config, ...a, modelRouting: { members: [a, b, c] } }, [{ role: 'user', content: 'owned' }], []); const chat = actualRequest(body, a), responses = actualRequest(body, b), plain = actualRequest(body, c); assert.equal(chat.max_completion_tokens, 512); assert.equal(chat.max_tokens, undefined); assert.equal(outputReservation(chat), 512); assert.equal(chat.reasoning_effort, 'high'); assert.equal(chat.seed, 3); assert.deepEqual(responses.reasoning, { effort: 'low' }); assert.equal(responses.temperature, undefined); assert.equal(responses.seed, undefined); assert.deepEqual(responses.include, ['reasoning.encrypted_content']); assert.equal(responses.store, false); assert.equal(plain.reasoning_effort, undefined); assert.equal(plain.max_tokens, 512); }); test('unsupported native fields and invalid values reject before dispatch instead of silently dropping or overriding constraints', () => { for (const [protocol, params, field] of [['chat-completions', { model: 'override' }], ['responses', { tools: [] }], ['responses', { reasoning: { effort: 'high', extra: 1 } }], ['responses', { text: { format: { type: 'json' } } }], ['chat-completions', { temperature: '1' }], ['chat-completions', { top_p: 2 }], ['chat-completions', { seed: 1.5 }], ['chat-completions', { parallel_tool_calls: 'true' }], ['chat-completions', { reasoning_effort: 'invented' }], ['chat-completions', {}, 'max_output_tokens'], ['responses', {}, 'max_tokens'], ['responses', []], ['responses', { unknown: 'x'.repeat(8200) }]]) { assert.throws(() => nativeParameterSettings(protocol, params, field), error => error.status === 400); } }); test('native settings for legacy and registered profiles persist without changing credentials or old records and reject stale edits', async t => { const f = fixture(t), provider = new Provider(); provider.setKey('owned-legacy-key'); const { hub, connection } = configure(f, provider, config.baseUrl, { apiKey: 'owned-profile-key' }); const oldMessages = JSON.stringify(f.session), credential = provider.credentials.get(connection.id), oldPool = JSON.stringify(hub.state.pools); hub.setNativeParameters({ id: connection.id, revision: connection.revision, nativeParameters: { reasoning_effort: 'medium' }, outputTokenField: 'max_completion_tokens' }); assert.equal(provider.credentials.get(connection.id), credential); assert.equal(JSON.stringify(f.session), oldMessages); assert.equal(JSON.stringify(hub.state.pools), oldPool); assert.throws(() => hub.setNativeParameters({ id: connection.id, revision: connection.revision, nativeParameters: {} }), error => error.status === 409); const legacy = hub.connection('legacy-default'); hub.setNativeParameters({ id: legacy.id, revision: legacy.revision, nativeParameters: { temperature: 0.5 } }); assert.equal(provider.apiKey, 'owned-legacy-key'); assert.deepEqual(f.store.config.nativeParameters, { temperature: 0.5 }); f.store.close(); const reopened = new Store(f.data); try { assert.deepEqual(reopened.config.nativeParameters, { temperature: 0.5 }); assert.deepEqual(new Provider().modelHub(reopened).connection(connection.id).nativeParameters, { reasoning_effort: 'medium' }); } finally { reopened.close(); } }); test('compiled native settings API saves/restores exact parameters and rejects reserved fields without mutating the profile', async t => { const f = fixture(t); f.store.close(); const host = await launchWorkbench(t, f.data); const member = await host.api('/model-center/connections', { name: 'Owned API', model: 'owned-api', protocol: 'responses', baseUrl: config.baseUrl, requiresKey: false }); await host.api('/model-center/native-parameters', { id: member.id, revision: member.revision, nativeParameters: { reasoning: { effort: 'high' }, parallel_tool_calls: false } }); const saved = (await host.api('/model-center')).connections.find(value => value.id === member.id); assert.deepEqual(saved.nativeParameters, { reasoning: { effort: 'high' }, parallel_tool_calls: false }); await assert.rejects(host.api('/model-center/native-parameters', { id: member.id, revision: saved.revision, nativeParameters: { max_output_tokens: 99999 } }), error => error.status === 400); assert.deepEqual((await host.api('/model-center')).connections.find(value => value.id === member.id), saved); }); test('legacy connection test sends its saved native fields and selected output limit field without changing the configuration', async t => { const f = fixture(t), model = await compatibleFixture(t, () => completion([], 'OK')); f.store.close(); const host = await launchWorkbench(t, f.data), original = (await host.api('/model-center')).connections.find(value => value.id === 'legacy-default'); await host.api('/model-center/native-parameters', { id: original.id, revision: original.revision, nativeParameters: { reasoning_effort: 'low' }, outputTokenField: 'max_completion_tokens' }); const before = await host.api('/model-center'); await host.api('/config/test', { baseUrl: model.baseUrl, model: 'owned-probe' }); assert.equal(model.requests.length, 1); assert.equal(model.requests[0].input.reasoning_effort, 'low'); assert.equal(model.requests[0].input.max_completion_tokens, 64); assert.equal(model.requests[0].input.max_tokens, undefined); assert.deepEqual(await host.api('/model-center'), before); });