fix: preserve upstream model errors and native request parameters

This commit is contained in:
Agent Workbench Dev
2026-11-01 22:05:08 +08:00
parent 02cefd0498
commit 97c8dde70d
35 changed files with 816 additions and 75 deletions
+4 -1
View File
@@ -17,6 +17,7 @@ import { hostAuthority } from '../core/desktop/host-authority.mjs';
import { companionGateway } from './companion-gateway.mjs';
import { inspectOperations } from './operations.mjs';
import { validateModelWaitSettings } from '../core/models/request-deadline.mjs';
import { nativeParameterSettings } from '../shared/contracts/native-parameters.mjs';
export function createSessionPublisher(clients) {
return session => {
@@ -173,6 +174,7 @@ return async (request, response) => {
return json(response, { center: hub.view(), config: { ...store.config, hasKey: provider.hasKey() } });
}
if (route === '/api/model-center/credentials') return json(response, hub.manageCredential(input));
if (route === '/api/model-center/native-parameters') return json(response, hub.setNativeParameters(input));
if (route === '/api/model-center/connections') return json(response, hub.upsert(input), 201);
if (route === '/api/model-center/pools') return json(response, hub.pool(input), 201);
if (route === '/api/model-center/select') return json(response, { center: hub.select(input), config: { ...store.config, hasKey: provider.hasKey() } });
@@ -731,7 +733,8 @@ return async (request, response) => {
const config = { ...store.config, baseUrl: String(input.baseUrl || store.config.baseUrl), model: String(input.model || store.config.model), mode: 'compatible' };
provider.validate(config);
const startedAt = Date.now();
const result = await provider.request(config, '/chat/completions', { model: config.model, stream: false, max_tokens: 64,
const native = nativeParameterSettings('chat-completions', config.nativeParameters ?? {}, config.outputTokenField);
const result = await provider.request(config, '/chat/completions', { model: config.model, stream: false, [native.outputTokenField]: 64, ...native.nativeParameters,
messages: [{ role: 'user', content: 'Connection check. Reply with OK.' }] }, undefined,
Object.hasOwn(input, 'apiKey') ? String(input.apiKey).trim() : provider.apiKey);
if (!result.choices?.[0]?.message) throw new Error('模型服务响应中缺少 choices[0].message。');
+21 -10
View File
@@ -1,6 +1,6 @@
import { randomUUID, createHash } from 'node:crypto';
export const APP_VERSION = '1.20.2';
export const APP_VERSION = '1.20.3';
export function executionEnvironment() {
const windows = process.platform === 'win32';
@@ -22,7 +22,8 @@ export class ModelRequestError extends Error {
}
export function connectionFailure(error, { endpoint, model, route, attempt, redact = String, timedOut = false, timeout = null }) {
if (error instanceof ModelRequestError) return { endpoint, model, route, ...error.diagnostic, attempts: attempt };
if (error instanceof ModelRequestError) return { ...error.diagnostic, endpoint: error.diagnostic.endpoint ?? endpoint,
model: error.diagnostic.model ?? model, route: error.diagnostic.route ?? route, attempts: attempt };
const causes = [error, error?.cause, ...(error?.cause?.errors ?? [])];
const code = causes.map(item => item?.code).find(value => typeof value === 'string' && /^[A-Z0-9_]{2,60}$/.test(value)) ?? null;
const kind = timedOut || causes.some(item => item?.name === 'TimeoutError') ? 'timeout'
@@ -31,7 +32,7 @@ export function connectionFailure(error, { endpoint, model, route, attempt, reda
const title = kind === 'timeout' ? '模型服务响应超时' : kind === 'dns' ? '模型服务地址解析失败'
: kind === 'certificate' ? '模型服务证书校验失败' : '模型服务连接失败';
return {
id: randomUUID(), source: 'model_connection', stage: 'model_request', kind, code,
id: randomUUID(), source: 'model_connection', origin: 'local', stage: 'model_request', kind, code,
endpoint, model, route, attempts: attempt, occurredAt: new Date().toISOString(),
retryable: kind !== 'certificate', title,
message: title + (code ? '(' + code + ')' : '') + '。' + (timeout ? timeout.kind === 'idle'
@@ -45,17 +46,27 @@ export function connectionFailure(error, { endpoint, model, route, attempt, reda
};
}
export function httpFailure(status, detail, { endpoint, model, route, attempt }) {
const kind = status === 401 || status === 403 ? 'authentication' : status === 429 ? 'rate_limit' : 'service';
export function httpFailure(status, detail, { endpoint, model, route, attempt, upstream, attemptId } = {}) {
const codes = [upstream?.code, upstream?.type].filter(value => typeof value === 'string').map(value => value.toLowerCase());
const quota = codes.some(value => ['insufficient_quota', 'billing_hard_limit_reached', 'billing_limit_reached', 'usage_limit_reached',
'credit_balance_too_low', 'insufficient_balance', 'exceeded_quota'].includes(value));
const limited = status === 429 || codes.some(value => ['rate_limit_exceeded', 'rate_limit_error', 'requests_limit_exceeded', 'tokens_limit_exceeded'].includes(value));
const kind = quota ? 'quota' : status === 401 || status === 403 ? 'authentication' : limited ? 'rate_limit' : 'service';
const title = kind === 'authentication' ? '模型服务拒绝了身份验证'
: kind === 'rate_limit' ? '模型服务请求达到限额' : '模型服务暂时无法处理请求';
: kind === 'quota' ? '模型服务额度或消费限制' : kind === 'rate_limit' ? '模型服务请求达到限额' : '模型服务暂时无法处理请求';
return new ModelRequestError({
id: randomUUID(), source: 'model_connection', stage: 'model_request', kind, httpStatus: status,
id: randomUUID(), source: 'model_connection', origin: 'upstream', stage: 'model_request', kind, httpStatus: status,
endpoint, model, route, attempts: attempt, occurredAt: new Date().toISOString(),
retryable: [408, 429, 500, 502, 503, 504].includes(status), title,
message: title + '(HTTP ' + status + ')。' + (detail ? ' ' + detail : ''), detail,
retryable: !quota && !upstream?.retryAfterInvalid && (limited || [408, 500, 502, 503, 504].includes(status)),
...(quota ? { failoverAllowed: true } : {}), title,
...(upstream ? { upstream, upstreamRequestId: upstream.requestId, code: upstream.code, type: upstream.type,
...(upstream.retryAfterMs !== null ? { retryAfterMs: upstream.retryAfterMs } : {}),
limitReason: quota ? 'quota' : limited ? codes.some(value => /rate_limit|requests_limit|tokens_limit/.test(value)) ? 'rate' : 'unspecified' : null } : {}),
...(attemptId ? { wireAttemptId: attemptId } : {}),
message: title + (status >= 400 ? '(HTTP ' + status + ')' : '(上游返回错误内容)') + '。' + (detail ? ' ' + detail : ''), detail,
suggestion: kind === 'authentication' ? '在模型连接中核对 API Key、模型权限和接口地址。'
: kind === 'rate_limit' ? '等待服务限额恢复后重试;同时检查账户额度。' : '稍后重试,或检查模型服务状态。',
: quota ? '核对上游账户余额、消费上限或用量限制。该线路不会自动重复请求;模型池可按规则选择其他可用成员。'
: kind === 'rate_limit' ? '依据上游错误码核对限制原因;有 Retry-After 时按原等待时间处理,原因未给出时保持未确定。' : '稍后重试,或检查模型服务状态。',
});
}
+54 -22
View File
@@ -7,6 +7,7 @@ import { ModelHub } from '../core/models/model-hub.mjs';
import { normalizeProtocolResponse, protocolRoute, validateModelResponse, reportedUsage } from '../core/models/protocols.mjs';
import { readModelStream } from '../core/models/model-stream.mjs';
import { modelWaitSettings, requestDeadline } from '../core/models/request-deadline.mjs';
import { readErrorResponse, upstreamCapture } from '../core/models/upstream-error.mjs';
export const engineToolDefinitions = [
{ type: 'function', function: { name: 'inspect_engine_checks', description: '读取当前Task已经保存的外部引擎连接核验:真实安装版本、启动完成信号、SDK会话、进程关闭、原报告工件和能力限制。不会启动引擎或发送prompt;完整原记录通过已有read_artifact读取。', parameters: { type: 'object', properties: {}, additionalProperties: false } } },
@@ -121,7 +122,10 @@ export class Provider {
prepareConfig(config, store) { return this.modelHub(store).prepare(config); }
redact(value, key = this.apiKey) {
let text = String(value);
for (const secret of [key, this.apiKey, ...[...(this.credentials?.values() ?? [])].map(value => value.key)].filter(Boolean).sort((a, b) => b.length - a.length)) text = text.replaceAll(secret, '[已隐藏]');
for (const secret of [key, this.apiKey, ...[...(this.credentials?.values() ?? [])].map(value => value.key)].filter(Boolean).sort((a, b) => b.length - a.length)) {
const encoded = JSON.stringify(secret).slice(1, -1);
text = text.replaceAll(encoded, '[已隐藏]').replaceAll(secret, '[已隐藏]');
}
return text;
}
validate(config) {
@@ -148,14 +152,31 @@ export class Provider {
attempts = attempt;
const timeout = requestDeadline({ signal, timeoutMs: Math.max(1, Math.min(policy.timeoutMs, deadline - Date.now())), idleTimeoutMs: policy.idleTimeoutMs });
const requestSignal = timeout.signal;
const context = { endpoint: base, model: config.model, route, attempt };
onProgress?.({ phase: 'running', attempt, maxAttempts: policy.attempts, endpoint: base, route });
let readingStream = false, streamTerminalSeen = false, dispatched = false, usage = null, outcome = 'unknown';
const receipt = { attemptId: randomUUID(), endpoint: base, model: config.model, protocol: config.protocol ?? 'chat-completions', startedAt: new Date().toISOString() };
const context = { endpoint: base, model: config.model, route, attempt, attemptId: receipt.attemptId };
onProgress?.({ phase: 'running', attempt, maxAttempts: policy.attempts, endpoint: base, route, wireAttemptId: receipt.attemptId });
let responseDetails = null;
const captureResponse = (response, input) => {
const captured = upstreamCapture({ ...input, status: response.status, headers: response.headers }, value => this.redact(value, key));
responseDetails = captured.details;
onProgress?.({ responseCapture: captured.capture, upstream: responseDetails, attempt, maxAttempts: policy.attempts, wireAttemptId: receipt.attemptId });
return responseDetails;
};
try {
dispatched = true;
const response = await (this.fetcher ?? fetch)(base + route, { method: body ? 'POST' : 'GET', headers,
body: body ? JSON.stringify(body) : undefined, signal: requestSignal });
const metadata = upstreamCapture({ raw: Buffer.alloc(0), status: response.status, headers: response.headers }, value => this.redact(value, key)).details;
receipt.upstreamRequestId = metadata.requestId; receipt.httpStatus = response.status;
onProgress?.({ phase: 'running', attempt, maxAttempts: policy.attempts, upstreamResponse: { httpStatus: response.status, requestId: metadata.requestId, headers: metadata.headers } });
if (!response.ok) {
const received = await readErrorResponse(response, requestSignal);
if (!received.truncated && !received.interrupted) { try { usage = reportedUsage(config, JSON.parse(received.raw.toString('utf8'))); } catch {} }
const upstream = captureResponse(response, received);
requestSignal.throwIfAborted();
throw httpFailure(response.status, (upstream.message ?? upstream.rawPreview).slice(0, 600), { ...context, upstream });
}
if (response.ok && body?.stream === true) {
readingStream = true;
timeout.beginStream();
@@ -163,31 +184,37 @@ export class Provider {
if (progress.reportedUsage) usage = progress.reportedUsage;
if (progress.stream?.terminalSeen) streamTerminalSeen = true;
onProgress?.({ ...progress, attempt, maxAttempts: policy.attempts });
}, body.tools, length => timeout.receive(length));
}, body.tools, length => timeout.receive(length), value => this.redact(value, key));
const value = options.transformResponse ? options.transformResponse(streamed.value) : streamed.value;
outcome = 'completed';
this.lastRequest = { source: 'model_connection', endpoint: base, route, model: config.model, status: 'completed', attempts: attempt, durationMs: Date.now() - startedAt, stream: streamed.evidence };
onProgress?.({ phase: 'completed', attempt, maxAttempts: policy.attempts, durationMs: Date.now() - startedAt, stream: streamed.evidence });
return streamed.value;
return value;
}
const text = await boundedResponse(response);
try { usage = reportedUsage(config, JSON.parse(text)); } catch {}
if (!response.ok) {
let message = text;
try { message = JSON.parse(text).error?.message ?? text; } catch {}
const failure = httpFailure(response.status, this.redact(message, key).slice(0, 600), context);
const retryAfter = response.headers.get('retry-after');
if (retryAfter) {
const milliseconds = /^\d+$/.test(retryAfter) ? Number(retryAfter) * 1000 : Date.parse(retryAfter) - Date.now();
if (Number.isFinite(milliseconds)) failure.diagnostic.retryAfterMs = Math.max(0, Math.min(milliseconds, 90000));
}
throw failure;
}
let result;
try { result = JSON.parse(text); } catch {
const upstream = captureResponse(response, { raw: Buffer.from(text, 'utf8') });
throw new ModelRequestError({ ...context, id: randomUUID(), source: 'model_connection', stage: 'model_request',
origin: 'local', upstream,
kind: 'protocol', title: '模型接口响应格式不兼容', message: '模型服务没有返回有效 JSON,请核对兼容接口地址。',
attempts: attempt, retryable: false, occurredAt: new Date().toISOString(), suggestion: '填写服务实际提供的 Chat Completions 接口根地址。' });
}
if (result.error || config.protocol === 'responses' && ['failed', 'incomplete', 'cancelled'].includes(result.status)) {
const upstream = captureResponse(response, { raw: Buffer.from(text, 'utf8') });
const failure = httpFailure(response.status, (upstream.message ?? upstream.reason ?? upstream.rawPreview).slice(0, 600), { ...context, upstream });
if (result.status === 'incomplete') { failure.diagnostic.kind = 'incomplete'; failure.diagnostic.title = '上游模型响应未完整结束'; }
throw failure;
}
if (options.transformResponse) {
try { result = options.transformResponse(result); }
catch (cause) {
const upstream = captureResponse(response, { raw: Buffer.from(text, 'utf8') });
if (cause.diagnostic) cause.diagnostic.upstream = upstream;
throw cause;
}
}
this.lastRequest = { source: 'model_connection', endpoint: base, route, model: config.model, status: 'completed', attempts: attempt, durationMs: Date.now() - startedAt };
outcome = 'completed';
onProgress?.({ phase: 'completed', attempt, maxAttempts: policy.attempts, durationMs: Date.now() - startedAt });
@@ -196,11 +223,18 @@ export class Provider {
outcome = signal?.aborted ? 'interrupted' : readingStream && !streamTerminalSeen ? 'unknown' : 'failed';
signal?.throwIfAborted();
let diagnostic = connectionFailure(error, { ...context, timedOut: !!timeout.timeout, timeout: timeout.timeout, redact: value => this.redact(value, key) });
if (readingStream) diagnostic = { ...diagnostic, retryable: false, outcome: streamTerminalSeen ? 'failed' : 'unknown', kind: timeout.timeout ? 'timeout' : streamTerminalSeen || diagnostic.kind === 'protocol' ? 'protocol' : 'stream_interrupted',
message: diagnostic.message + (streamTerminalSeen ? ' 服务流已终结,响应校验未通过;原流保留,不自动重发。' : ' 已接收的流片段保留,请求不会自动重发。') };
diagnostic = { ...diagnostic, origin: diagnostic.origin ?? 'local', wireAttemptId: receipt.attemptId, upstreamRequestId: diagnostic.upstreamRequestId ?? receipt.upstreamRequestId,
...(responseDetails && !diagnostic.upstream ? { upstream: responseDetails } : {}) };
if (readingStream) diagnostic = { ...diagnostic, retryable: false, failoverAllowed: false, outcome: streamTerminalSeen ? 'failed' : 'unknown',
kind: diagnostic.origin === 'upstream' ? diagnostic.kind : timeout.timeout ? 'timeout' : streamTerminalSeen || diagnostic.kind === 'protocol' ? 'protocol' : 'stream_interrupted',
...(diagnostic.origin === 'upstream' ? { suggestion: '可查看原失败事件、上游请求 ID 和已保存的响应。核对服务状态后手动续接;该流不会自动重发。' } : {}),
message: diagnostic.message + (diagnostic.origin === 'upstream' ? ' 已保存上游失败事件与原流,工具未执行,不会自动重发。'
: streamTerminalSeen ? ' 服务流已终结,响应校验未通过;原流保留,不自动重发。' : ' 已接收的流片段保留,请求不会自动重发。') };
const remaining = deadline - Date.now();
const waitMs = diagnostic.retryAfterMs ?? policy.delays[Math.min(attempt - 1, policy.delays.length - 1)] ?? 500;
const retry = diagnostic.retryable && attempt < policy.attempts && remaining > waitMs;
diagnostic.retryDecision = retry ? 'retry_after_wait' : readingStream ? 'stream_not_replayed' : !diagnostic.retryable ? 'not_retryable' : attempt >= policy.attempts ? 'attempt_limit' : 'wait_budget';
diagnostic.remainingWaitBudgetMs = Math.max(0, remaining);
this.lastRequest = { ...diagnostic, status: retry ? 'retrying' : 'failed', durationMs: Date.now() - startedAt };
onProgress?.({ phase: retry ? 'retrying' : 'failed', attempt, maxAttempts: policy.attempts,
diagnostic, waitMs: retry ? waitMs : 0, durationMs: Date.now() - startedAt });
@@ -232,10 +266,8 @@ export class Provider {
if (config.mode === 'demo') return this.demo(messages, project);
this.validate(config);
const execute = async (selected, key, input, progress) => {
const value = normalizeProtocolResponse(selected, await this.request(selected, protocolRoute(selected), input, signal, key, progress, { ...(config.modelRouting ? { attempts: 1 } : {}), ...requestPolicy }));
if (!value.choices?.[0]?.message || value.choices[0].finish_reason === 'length') throw new ModelRequestError({ kind: 'protocol', source: 'model_connection',
stage: 'model_request', title: '模型响应不完整', message: '模型响应缺少完整message或达到输出上限,未执行工具。', retryable: false, occurredAt: new Date().toISOString() });
return validateModelResponse(value, tools);
return this.request(selected, protocolRoute(selected), input, signal, key, progress, { ...(config.modelRouting ? { attempts: 1 } : {}), ...requestPolicy,
transformResponse: raw => validateModelResponse(normalizeProtocolResponse(selected, raw), tools) });
};
const result = config.modelRouting && this.hub ? await this.hub.complete(config, body, signal, onProgress, requestPlan, execute)
: await execute(config, this.apiKey, body, onProgress);
+13 -1
View File
@@ -1,6 +1,7 @@
import { spawn, execFile } from 'node:child_process';
import path from 'node:path';
import { randomUUID } from 'node:crypto';
import { sanitizeModelText } from '../core/models/upstream-error.mjs';
import { StringDecoder } from 'node:string_decoder';
import { setImmediate as yieldExecution } from 'node:timers/promises';
import { safePath, listFiles, readLines, searchFiles, proposeFile, proposePatch } from './files.mjs';
@@ -317,6 +318,17 @@ export class Runtime {
if (run.worker) { planned.record.workerUsageAccounted = true; this.store.save(session); }
return;
}
if (progress.responseCapture !== undefined || progress.streamCapture !== undefined) {
const raw = progress.responseCapture ?? progress.streamCapture;
const decoded = raw.toString('utf8'), redacted = sanitizeModelText(decoded, value => this.provider.redact(value));
const captured = redacted === decoded ? raw : Buffer.from(redacted, 'utf8');
const artifact = this.store.artifactStore.write(session, run.record, captured, { kind: progress.streamCapture !== undefined ? 'model-stream-response' : 'model-http-response', mediaType: 'text/plain' });
this.store.core.artifact.set(artifact.id, artifact);
if (progress.upstream) Object.assign(progress.upstream, { artifactId: artifact.id, artifactSha256: artifact.sha256,
artifactScope: progress.streamCapture !== undefined ? 'sse-stream' : 'http-body', sessionId: session.id });
if (progress.upstream?.capture) Object.assign(progress.upstream.capture, { retention: 'artifact', storedBytes: artifact.bytes });
this.store.save(session); return;
}
if (!requestEvent) requestEvent = this.store.event(session, { kind: externalEngine ? 'engine' : 'model', runId: run.id,
source: externalEngine ? 'controlled_engine' : 'model_connection', label: externalEngine ? '受控引擎执行' : '请求模型服务', phase: 'running', startedAt: now(), contextPlanId: planned.record.id });
if (progress.engineRunId) requestEvent.engineRunId = progress.engineRunId;
@@ -472,7 +484,7 @@ export class Runtime {
this.store.message(session, 'system', budget?.message ?? '任务已停止,执行记录已保存。');
} else {
session.status = 'failed';
const diagnostic = error.diagnostic ?? { source: 'workbench', stage: 'agent_loop', kind: 'runtime',
const diagnostic = error.diagnostic ?? { source: 'workbench', origin: 'local', stage: 'agent_loop', kind: 'runtime',
title: '工作台任务需要处理', message: this.provider.redact(error.message ?? String(error)), occurredAt: now() };
session.lastFailure = { ...diagnostic, runId: run.id };
const message = this.store.message(session, 'system', diagnostic.message);