fix: preserve upstream model errors and native request parameters

This commit is contained in:
Agent Workbench Dev
2026-11-01 22:05:08 +08:00
parent 02cefd0498
commit 97c8dde70d
35 changed files with 816 additions and 75 deletions
+31 -9
View File
@@ -6,6 +6,8 @@ import { visionProbe } from './vision-probe.mjs';
import { modelProtocol, capabilityDeclarations, capabilityView, capabilityRejections } from '../../shared/contracts/model-capabilities.mjs';
import { actualRequest, requestVariants } from '../../shared/contracts/model-request.mjs';
import { modelWaitSettings } from './request-deadline.mjs';
import { sanitizeModelText } from './upstream-error.mjs';
import { nativeParameterSettings } from '../../shared/contracts/native-parameters.mjs';
const legacyId = 'legacy-default';
const stamp = () => new Date().toISOString();
@@ -75,7 +77,8 @@ export class ModelHub {
connections() {
return [{ id: legacyId, name: '原模型连接', brand: '兼容服务', protocol: 'chat-completions', baseUrl: this.store.config.baseUrl,
model: this.store.config.model, enabled: true, requiresKey: false, revision: this.store.config.configRevision ?? 1,
contextLimit: this.store.config.contextLimit, maxOutputTokens: this.store.config.maxOutputTokens, concurrency: 1, source: 'legacy' }, ...this.state.connections];
contextLimit: this.store.config.contextLimit, maxOutputTokens: this.store.config.maxOutputTokens, concurrency: 1, source: 'legacy',
...(this.store.config.nativeParameters ? { nativeParameters: this.store.config.nativeParameters, outputTokenField: this.store.config.outputTokenField } : {}) }, ...this.state.connections];
}
credential(connection) { return connection.id === legacyId ? { key: this.provider.apiKey, revision: this.provider.legacyCredentialRevision ?? 1 } : this.provider.credentials.get(connection.id); }
vault() { return this.credentialVault ??= new CredentialVault(this.store, this.provider); }
@@ -166,6 +169,8 @@ export class ModelHub {
if (this.state.connections.length >= 100 && !previous) throw error('最多保存100个模型连接。');
if (previous && input.revision !== previous.revision) throw error('模型连接已修改,请刷新后重试。', 409);
const protocol = modelProtocol(input.protocol ?? previous?.protocol ?? 'chat-completions');
const native = Object.hasOwn(input, 'nativeParameters') || Object.hasOwn(input, 'outputTokenField') || previous?.nativeParameters
? nativeParameterSettings(protocol, input.nativeParameters ?? previous?.nativeParameters ?? {}, input.outputTokenField ?? previous?.outputTokenField) : null;
const declarations = capabilityDeclarations(input.capabilityDeclarations, previous?.capabilityDeclarations);
if (input.streaming !== undefined && typeof input.streaming !== 'boolean') throw error('流式选项必须为开启或关闭。');
const streaming = input.streaming ?? previous?.streaming ?? false;
@@ -173,7 +178,7 @@ export class ModelHub {
const name = label(input.name), model = label(input.model, 200);
if (!name || !model) throw error('请填写连接名称和实际模型名称。');
const connection = { id: previous?.id ?? randomUUID(), name, brand: label(input.brand || '自定义', 40), protocol, model,
capabilitySchema: 1, capabilityDeclarations: declarations, streaming,
capabilitySchema: 1, capabilityDeclarations: declarations, streaming, ...(native ?? {}),
baseUrl: modelEndpoint(input.baseUrl), enabled: input.enabled !== false, requiresKey: input.requiresKey !== false,
contextLimit: bounded(input.contextLimit, 16000, 1000, 200000), maxOutputTokens: bounded(input.maxOutputTokens, 2048, 256, 16000),
concurrency: bounded(input.concurrency, 1, 1, 8), revision: (previous?.revision ?? 0) + 1,
@@ -191,6 +196,14 @@ export class ModelHub {
}
return this.status(connection);
}
setNativeParameters(input) {
const connection = this.connection(input.id);
if (connection.revision !== input.revision) throw error('连接已变化,请刷新后核对原生参数。', 409);
const settings = nativeParameterSettings(connection.protocol, input.nativeParameters, input.outputTokenField);
if (connection.id === legacyId) this.save(settings);
else this.upsert({ ...connection, ...settings });
return { center: this.view(), config: { ...this.store.config, hasKey: this.provider.hasKey() } };
}
toggle(id, enabled) {
if (id === legacyId) throw error('原连接不能停用,请选择离线演示或其他连接。');
const value = this.connection(id); value.enabled = enabled === true; value.revision++; value.updatedAt = stamp(); this.save(); return this.status(value);
@@ -230,6 +243,7 @@ export class ModelHub {
credentialRevision: this.credential(this.connection(value.connectionId))?.revision ?? 0 }));
if (!members.some(value => this.status(value).status === 'configured' || this.status(value).status === 'cooling')) throw error('没有可用模型连接,请核对启用状态、临时凭据和故障。');
return { ...config, baseUrl: members[0].baseUrl, model: members[0].model, protocol: modelProtocol(members[0].protocol), streaming: members[0].streaming === true,
nativeParameters: members[0].nativeParameters ?? {}, outputTokenField: members[0].outputTokenField,
contextLimit: Math.min(config.contextLimit, ...members.map(value => value.contextLimit)), maxOutputTokens: Math.min(config.maxOutputTokens, ...members.map(value => value.maxOutputTokens)),
modelRouting: { type: selection.type, id: selection.id, revision: pool?.revision ?? members[0].revision,
strategy: pool?.strategy ?? 'round-robin', sticky: pool?.sticky ?? true, maxAttempts: pool?.maxAttempts ?? 1, queueTimeoutMs: pool?.queueTimeoutMs ?? 15000, members } };
@@ -310,6 +324,8 @@ export class ModelHub {
}
this.attempts.push(attempt); this.saveAttempts();
const progress = event => {
if (event.wireAttemptId) attempt.wireAttemptId = event.wireAttemptId;
if (event.upstreamResponse) attempt.upstreamResponse = event.upstreamResponse;
if (event.consumption) {
attempt.usage = event.consumption.usage; attempt.usageStatus = event.consumption.usageStatus;
onProgress?.({ consumption: { ...event.consumption, wireAttemptId: event.consumption.attemptId, attemptId: attempt.id,
@@ -317,16 +333,21 @@ export class ModelHub {
this.saveAttempts();
return;
}
if (event.streamCapture !== undefined) {
attempt.stream = event.stream;
if (event.streamCapture !== undefined || event.responseCapture !== undefined) {
if (event.stream) attempt.stream = event.stream;
if (plan?.sessionId) {
const session = this.store.session(plan.sessionId), run = this.store.owned('run', plan.runId, session.id);
const raw = event.streamCapture, decoded = raw.toString('utf8'), redacted = this.provider.redact(decoded);
const raw = event.streamCapture ?? event.responseCapture, decoded = raw.toString('utf8'), redacted = sanitizeModelText(decoded, value => this.provider.redact(value));
const captured = redacted === decoded ? raw : Buffer.from(redacted, 'utf8');
const artifact = this.store.artifactStore.write(session, run, captured, { kind: 'model-stream-response', mediaType: 'text/event-stream' });
const artifact = this.store.artifactStore.write(session, run, captured, { kind: event.streamCapture !== undefined ? 'model-stream-response' : 'model-http-response',
mediaType: event.streamCapture !== undefined ? 'text/event-stream' : 'text/plain' });
this.store.core.artifact.set(artifact.id, artifact); attempt.responseArtifactId = artifact.id;
attempt.responseSha256 = artifact.sha256; attempt.responseRedacted = redacted !== decoded; this.store.save(session);
attempt.responseSha256 = artifact.sha256; attempt.responseRedacted = redacted !== decoded || event.upstream?.capture?.redacted === true; this.store.save(session);
if (event.upstream) Object.assign(event.upstream, { artifactId: artifact.id, artifactSha256: artifact.sha256,
artifactScope: event.streamCapture !== undefined ? 'sse-stream' : 'http-body', sessionId: session.id });
if (event.upstream?.capture) Object.assign(event.upstream.capture, { retention: 'artifact', storedBytes: artifact.bytes });
}
if (event.upstream) attempt.upstream = event.upstream;
return;
}
if (event.streaming) attempt.partial = { sequence: event.streamSequence, bytes: event.streamBytes, textCharacters: event.textCharacters, textPreview: this.provider.redact(event.textPreview ?? '').slice(0, 8000), toolFragments: event.toolFragments };
@@ -340,6 +361,7 @@ export class ModelHub {
return result;
} catch (cause) {
const diagnostic = cause.diagnostic;
if (diagnostic) attempt.diagnostic = diagnostic;
attempt.status = signal?.aborted ? 'interrupted' : diagnostic?.outcome === 'unknown' ? 'unknown' : 'failed';
attempt.reason = this.provider.redact(cause.message).slice(0, 600); attempt.httpStatus = diagnostic?.status ?? diagnostic?.httpStatus ?? null;
const health = this.health.get(this.identity(member)) ?? {};
@@ -348,14 +370,14 @@ export class ModelHub {
// the attempt, release its lease, and leave the connection eligible.
this.health.set(this.identity(member), { ...health, state: 'interrupted', lastInterruptedAt: stamp() });
} else {
const blocked = [401, 403].includes(attempt.httpStatus);
const blocked = [401, 403].includes(attempt.httpStatus) || diagnostic?.kind === 'quota';
const failedAt = Date.now();
this.health.set(this.identity(member), { state: blocked ? 'needs_attention' : 'failed', blocked, lastFailureAt: new Date(failedAt).toISOString(),
failures: (health.failures ?? 0) + 1, cooldownUntil: failedAt + (diagnostic?.retryAfterMs ?? Math.min(60000, 1000 * 2 ** Math.min(health.failures ?? 0, 5))), reason: attempt.reason });
}
lastError = cause; excluded.add(member.id); this.sticky.delete(runId);
signal?.throwIfAborted();
if (!diagnostic?.retryable || count === routing.maxAttempts) throw cause;
if ((!diagnostic?.retryable && !diagnostic?.failoverAllowed) || count === routing.maxAttempts) throw cause;
onProgress?.({ phase: 'retrying', attempt: count, maxAttempts: routing.maxAttempts, connectionId: member.id, modelAttemptId: attempt.id,
diagnostic, waitMs: 0, switching: true });
} finally { attempt.finishedAt = stamp(); lease.release(); this.saveAttempts(); }
+20 -6
View File
@@ -1,14 +1,17 @@
import { createHash } from 'node:crypto';
import { protocolFailure, normalizeProtocolResponse, validateModelResponse, reportedUsage } from './protocols.mjs';
import { httpFailure } from '../../server/diagnostics.mjs';
import { upstreamCapture } from './upstream-error.mjs';
// Only a fully terminated and validated stream produces a dispatchable message.
// Progress is cumulative, bounded and explicitly not a tool invocation.
export async function readModelStream(response, config, signal, onProgress, tools, onBytes) {
export async function readModelStream(response, config, signal, onProgress, tools, onBytes, redact = String) {
if (!response.headers.get('content-type')?.toLowerCase().includes('text/event-stream') || !response.body) throw protocolFailure('服务未返回所选协议的事件流,请核对流式配置。');
const reader = response.body.getReader(), decoder = new TextDecoder('utf-8', { fatal: true }), hash = createHash('sha256');
const chunks = [];
let buffer = '', data = [], bytes = 0, frames = 0, sequence = -1, text = '', done = false, terminal = null, usage = null, finish = null;
let lastProgress = 0, responseId = null, validated = false, progressTimer = null, progressFailure = null;
let upstreamFailure = null;
const calls = new Map(), responseCalls = new Map();
const progress = force => {
if (!force && Date.now() - lastProgress < 100) {
@@ -30,7 +33,18 @@ export async function readModelStream(response, config, signal, onProgress, tool
if (raw === '[DONE]') { if (config.protocol === 'responses') throw protocolFailure('Responses缺少正式终结事件。'); done = true; return; }
let event;
try { event = JSON.parse(raw); } catch { throw protocolFailure('模型事件流包含无效JSON。'); }
if (event.error || event.type === 'error') throw protocolFailure('模型流返回错误,已收到片段保留且不会自动重发。', 'stream_interrupted');
if (event.usage) usage = event.usage;
const failUpstream = () => {
usage = event.response?.usage ?? usage;
const { details } = upstreamCapture({ raw: Buffer.from(raw, 'utf8'), event, scope: 'sse-event', status: response.status, headers: response.headers }, redact);
details.responseId ??= responseId;
upstreamFailure = httpFailure(response.status, (details.message ?? details.reason ?? '上游事件 ' + event.type).slice(0, 600), { upstream: details });
if (event.type === 'response.incomplete') {
upstreamFailure.diagnostic.kind = 'incomplete'; upstreamFailure.diagnostic.title = '上游模型响应未完整结束';
}
throw upstreamFailure;
};
if (event.error || event.type === 'error') failUpstream();
if (config.protocol === 'responses') {
if (event.sequence_number !== undefined) {
if (!Number.isSafeInteger(event.sequence_number) || event.sequence_number <= sequence) throw protocolFailure('Responses事件序列重复或逆序。');
@@ -54,8 +68,7 @@ export async function readModelStream(response, config, signal, onProgress, tool
if (!event.response || responseId && event.response.id !== responseId) throw protocolFailure('Responses终结身份与创建身份不一致。');
terminal = event.response;
} else if (['response.failed', 'response.incomplete', 'response.cancelled'].includes(event.type)) {
usage = event.response?.usage ?? usage;
throw protocolFailure('Responses流未完整结束,工具未执行。', 'stream_interrupted');
failUpstream();
}
} else {
if (!Array.isArray(event.choices)) throw protocolFailure('Chat流缺少choices数组。');
@@ -135,7 +148,8 @@ export async function readModelStream(response, config, signal, onProgress, tool
} finally {
if (progressTimer) clearTimeout(progressTimer);
try { await reader.cancel(); } catch {} reader.releaseLock();
onProgress?.({ streamCapture: Buffer.concat(chunks), reportedUsage: reportedUsage(config, { usage: terminal?.usage ?? usage }), stream: { bytes, capturedBytes: chunks.reduce((sum, item) => sum + item.length, 0), frames,
sha256: hash.digest('hex'), terminal: validated, terminalSeen: !!terminal || done, protocol: config.protocol ?? 'chat-completions' } });
onProgress?.({ streamCapture: Buffer.concat(chunks), ...(upstreamFailure ? { upstream: upstreamFailure.diagnostic.upstream } : {}),
reportedUsage: reportedUsage(config, { usage: terminal?.usage ?? usage }), stream: { bytes, capturedBytes: chunks.reduce((sum, item) => sum + item.length, 0), frames,
sha256: hash.digest('hex'), terminal: validated, terminalSeen: !!terminal || done || !!upstreamFailure, protocol: config.protocol ?? 'chat-completions' } });
}
}
+3 -3
View File
@@ -1,5 +1,5 @@
import { ModelRequestError } from '../../server/diagnostics.mjs';
export const protocolFailure = (message, kind = 'protocol') => new ModelRequestError({ kind, source: 'model_connection', stage: 'model_request',
export const protocolFailure = (message, kind = 'protocol') => new ModelRequestError({ kind, source: 'model_connection', origin: 'local', stage: 'model_request',
title: '模型响应未通过完整性核对', message, retryable: false, occurredAt: new Date().toISOString() });
export const protocolRoute = config => config.protocol === 'responses' ? '/responses' : '/chat/completions';
@@ -55,9 +55,9 @@ export function normalizeProtocolResponse(config, value) {
for (const content of item.content) {
if (content.type === 'output_text' && typeof content.text === 'string') texts.push(content.text);
else if (content.type === 'refusal' && typeof content.refusal === 'string') texts.push(content.refusal);
else throw protocolFailure('Responses包含当前尚未接入的输出内容类型。');
else throw protocolFailure('Responses包含当前尚未接入的输出内容类型:' + String(content?.type ?? '未标注').slice(0, 100) + '。');
}
} else if (item.type !== 'reasoning') throw protocolFailure('Responses包含当前尚未接入的服务端工具或输出类型。');
} else if (item.type !== 'reasoning') throw protocolFailure('Responses包含当前尚未接入的服务端工具或输出类型:' + String(item.type ?? '未标注').slice(0, 100) + '。');
}
const raw = value.usage, usage = raw ? { prompt_tokens: raw.input_tokens, completion_tokens: raw.output_tokens,
...(raw.total_tokens !== undefined ? { total_tokens: raw.total_tokens } : {}), raw, protocol: 'responses' } : null;
+89
View File
@@ -0,0 +1,89 @@
import { createHash } from 'node:crypto';
export const ERROR_CAPTURE_LIMIT = 64 * 1024;
const PREVIEW_LIMIT = 8000;
const allowedHeaders = ['content-type', 'retry-after', 'date', 'x-request-id', 'request-id', 'requestid', 'x-amzn-requestid',
'anthropic-request-id', 'x-ratelimit-limit-requests', 'x-ratelimit-remaining-requests', 'x-ratelimit-reset-requests',
'x-ratelimit-limit-tokens', 'x-ratelimit-remaining-tokens', 'x-ratelimit-reset-tokens'];
export function sanitizeModelText(value, redact = String) {
return redact(String(value))
.replace(/("(?:api[_-]?key|authorization|access[_-]?token|refresh[_-]?token|password|secret|cookie|set-cookie)"\s*:\s*")((?:\\.|[^"\\])*)(")/gi, '$1[已隐藏]$3')
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [已隐藏]');
}
export function retryAfterMilliseconds(value, now = Date.now()) {
if (!value) return null;
const milliseconds = /^\d+$/.test(value) ? Number(value) * 1000 : Date.parse(value) - now;
return Number.isSafeInteger(milliseconds) ? Math.max(0, milliseconds) : null;
}
// Captures are bounded independently of the generated-response limit. An
// interrupted or truncated prefix never claims to be the complete response.
export async function readErrorResponse(response, signal) {
if (!response.body) return { raw: Buffer.alloc(0), receivedBytes: 0, truncated: false };
const reader = response.body.getReader(), parts = [];
let receivedBytes = 0, capturedBytes = 0, truncated = false, interrupted = false;
try {
while (true) {
signal?.throwIfAborted();
const { done, value } = await reader.read();
if (done) break;
receivedBytes += value.length;
const keep = Math.min(value.length, ERROR_CAPTURE_LIMIT - capturedBytes);
if (keep) { parts.push(Buffer.from(value.subarray(0, keep))); capturedBytes += keep; }
if (receivedBytes > ERROR_CAPTURE_LIMIT) { truncated = true; break; }
}
} catch (cause) {
interrupted = true;
} finally {
try { await reader.cancel(); } catch {}
reader.releaseLock();
}
return { raw: Buffer.concat(parts), receivedBytes, truncated, ...(interrupted ? { interrupted: true } : {}) };
}
export function upstreamCapture({ raw, receivedBytes = raw.length, truncated = false, interrupted = false, status,
headers, event, scope = 'http-body' }, redact = String) {
const bounded = raw.subarray(0, ERROR_CAPTURE_LIMIT), decoded = bounded.toString('utf8');
const text = sanitizeModelText(decoded, redact), changed = text !== decoded;
const capture = changed ? Buffer.from(text, 'utf8') : Buffer.from(bounded);
let parsed = event;
if (!parsed) { try { parsed = JSON.parse(decoded); } catch {} }
const original = parsed?.response?.error ?? parsed?.error;
const fields = original && typeof original === 'object' ? original : parsed;
const shortenedFields = [];
const field = (name, value, limit = 512) => {
if (typeof value !== 'string' && typeof value !== 'number') return null;
const clean = sanitizeModelText(value, redact);
if (clean.length > limit) shortenedFields.push(name);
return clean.slice(0, limit);
};
const diagnosticHeaders = {};
for (const name of allowedHeaders) {
const value = headers?.get(name);
if (value !== null && value !== undefined) diagnosticHeaders[name] = field('header:' + name, value, 1024);
}
const retryAfter = diagnosticHeaders['retry-after'] ?? null;
const retryAfterMs = retryAfterMilliseconds(retryAfter);
const details = {
schema: 1, scope, httpStatus: status,
code: field('code', fields?.code, 160), type: field('type', fields?.type ?? (parsed?.type === 'error' ? null : parsed?.type), 160),
message: field('message', fields?.message ?? (typeof original === 'string' ? original : null), 4096),
param: field('param', fields?.param), reason: field('reason', parsed?.response?.incomplete_details?.reason ?? parsed?.incomplete_details?.reason),
requestId: diagnosticHeaders['x-request-id'] ?? diagnosticHeaders['request-id'] ?? diagnosticHeaders.requestid
?? diagnosticHeaders['anthropic-request-id'] ?? diagnosticHeaders['x-amzn-requestid']
?? field('requestId', parsed?.request_id ?? fields?.request_id ?? parsed?.requestId),
responseId: field('responseId', parsed?.response?.id ?? (parsed?.object === 'response' ? parsed.id : null)),
eventType: scope === 'sse-event' ? field('eventType', parsed?.type) : null,
headers: diagnosticHeaders, retryAfter, retryAfterMs,
...(retryAfter && retryAfterMs === null ? { retryAfterInvalid: true } : {}),
rawPreview: text.slice(0, PREVIEW_LIMIT), previewTruncated: text.length > PREVIEW_LIMIT,
capture: { scope, receivedBytes, capturedBytes: bounded.length, storedBytes: Buffer.byteLength(text.slice(0, PREVIEW_LIMIT), 'utf8'), retention: 'preview-only',
complete: !truncated && !interrupted && raw.length <= ERROR_CAPTURE_LIMIT,
truncated: truncated || raw.length > ERROR_CAPTURE_LIMIT, interrupted, capturedRange: [0, bounded.length],
redacted: changed, sha256: createHash('sha256').update(capture).digest('hex') },
...(shortenedFields.length ? { shortenedFields } : {}),
};
return { capture, details };
}
+1
View File
@@ -12,6 +12,7 @@ export const DEFAULT_CONFIG = {
export function cleanConfig(value = {}) {
const result = { ...DEFAULT_CONFIG };
for (const key of Object.keys(result)) if (Object.hasOwn(value, key)) result[key] = value[key];
for (const key of ['nativeParameters', 'outputTokenField']) if (Object.hasOwn(value, key)) result[key] = structuredClone(value[key]);
return result;
}