fix: isolate model transport and stop replaying unknown requests
This commit is contained in:
@@ -21,25 +21,45 @@ export class ModelRequestError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export function connectionFailure(error, { endpoint, model, route, attempt, redact = String, timedOut = false, timeout = null }) {
|
||||
export function connectionFailure(error, { endpoint, model, route, attempt, redact = String, timedOut = false, timeout = null,method='POST',responseReceived=false,transport }) {
|
||||
if (error instanceof ModelRequestError) return { ...error.diagnostic, endpoint: error.diagnostic.endpoint ?? endpoint,
|
||||
model: error.diagnostic.model ?? model, route: error.diagnostic.route ?? route, attempts: attempt };
|
||||
const causes = [error, error?.cause, ...(error?.cause?.errors ?? [])];
|
||||
const causes=[],leaves=[],seen=new Set();let causesComplete=true;
|
||||
const visit=(value,depth=0)=>{
|
||||
if(!value||typeof value!=='object'||seen.has(value)||depth>4||causes.length>=16){causesComplete=false;return;}
|
||||
seen.add(value);causes.push(value);
|
||||
const children=[...(value.cause?[value.cause]:[]),...(Array.isArray(value.errors)?value.errors:[])];
|
||||
if(!children.length)leaves.push(value);
|
||||
if(children.length>8)causesComplete=false;
|
||||
for(const child of children.slice(0,8))visit(child,depth+1);
|
||||
};visit(error);
|
||||
const code = causes.map(item => item?.code).find(value => typeof value === 'string' && /^[A-Z0-9_]{2,60}$/.test(value)) ?? null;
|
||||
const kind = timedOut || causes.some(item => item?.name === 'TimeoutError') ? 'timeout'
|
||||
const connectTimeout=causes.some(item=>item.code==='UND_ERR_CONNECT_TIMEOUT'||item.code==='ETIMEDOUT'&&item.syscall==='connect');
|
||||
const kind = timedOut || connectTimeout || causes.some(item => item?.name === 'TimeoutError') ? 'timeout'
|
||||
: ['ENOTFOUND', 'EAI_AGAIN'].includes(code) ? 'dns'
|
||||
: /CERT|TLS|SSL/.test(code ?? '') ? 'certificate' : 'network';
|
||||
const title = kind === 'timeout' ? '模型服务响应超时' : kind === 'dns' ? '模型服务地址解析失败'
|
||||
const title = connectTimeout ? '模型服务连接超时' : kind === 'timeout' ? '模型服务响应超时' : kind === 'dns' ? '模型服务地址解析失败'
|
||||
: kind === 'certificate' ? '模型服务证书校验失败' : '模型服务连接失败';
|
||||
const coded=causes.filter(item=>typeof item.code==='string'&&/^[A-Z0-9_]{2,60}$/.test(item.code));
|
||||
const preSend=item=>['ENOTFOUND','EAI_AGAIN','ECONNREFUSED','EHOSTUNREACH','ENETUNREACH','UND_ERR_CONNECT_TIMEOUT'].includes(item.code)
|
||||
||item.code==='ETIMEDOUT'&&item.syscall==='connect'||/CERT|TLS|SSL/.test(item.code??'');
|
||||
const beforeSend=!responseReceived&&!timedOut&&causesComplete&&coded.length>0&&coded.every(preSend)&&leaves.every(preSend);
|
||||
const unknown=method==='POST'&&!beforeSend;
|
||||
const network={stage:responseReceived?'response_body':kind==='dns'?'dns':kind==='certificate'?'tls':connectTimeout||beforeSend?'connect':timedOut?'waiting_response':'unknown',
|
||||
replaySafety:beforeSend?'before_model_send':unknown?'unknown_after_dispatch':'read_only_request',responseReceived,causesComplete,
|
||||
...(transport?{transport}:{}),causes:coded.map(item=>({code:item.code,...(typeof item.syscall==='string'&&/^[a-z_]{1,40}$/.test(item.syscall)?{syscall:item.syscall}:{}),
|
||||
...(item.family===4||item.family===6?{family:item.family}:{})}))};
|
||||
return {
|
||||
id: randomUUID(), source: 'model_connection', origin: 'local', stage: 'model_request', kind, code,
|
||||
endpoint, model, route, attempts: attempt, occurredAt: new Date().toISOString(),
|
||||
retryable: kind !== 'certificate', title,
|
||||
retryable: kind !== 'certificate'&&!unknown,failoverAllowed:false,outcome:unknown?'unknown':'failed',network,title,
|
||||
message: title + (code ? '(' + code + ')' : '') + '。' + (timeout ? timeout.kind === 'idle'
|
||||
? '流式响应连续 ' + Math.round(timeout.limitMs / 1000) + ' 秒未收到新数据。'
|
||||
: '已达到本次模型响应 ' + Math.round(timeout.limitMs / 1000) + ' 秒等待上限。' : '') + '项目文件和已完成的工具记录已保留。',
|
||||
: '已达到本次模型响应 ' + Math.round(timeout.limitMs / 1000) + ' 秒等待上限。' : '') + (unknown?'原POST结果未知,不会自动重发或切池。':'')+'项目文件和已完成的工具记录已保留。',
|
||||
...(timeout ? { timeout } : {}),
|
||||
suggestion: kind === 'certificate' ? '核对服务地址和证书配置。'
|
||||
: connectTimeout ? '核对网络、DNS、IPv4/IPv6候选及代理;这是建立连接时超时,增加模型响应等待时限不能修复连接。'
|
||||
: unknown ? '核对原请求与上游状态后再显式续接;未知用量保持未知,已完成工具不得重放。'
|
||||
: timeout ? '在模型中心的“等待时限”调整设置,或核对服务状态后手动续接;任务总预算仍有效。'
|
||||
: '可以重试上次模型请求;若持续失败,在模型连接中检查服务地址和连接状态。',
|
||||
detail: redact(kind === 'network' ? '网络请求未取得完整模型响应。' : title),
|
||||
|
||||
+29
-14
@@ -7,7 +7,8 @@ import { ModelHub } from '../core/models/model-hub.mjs';
|
||||
import { normalizeProtocolResponse, protocolRoute, validateModelResponse, reportedUsage } from '../core/models/protocols.mjs';
|
||||
import { readModelStream } from '../core/models/model-stream.mjs';
|
||||
import { modelWaitSettings, requestDeadline } from '../core/models/request-deadline.mjs';
|
||||
import { readErrorResponse, upstreamCapture } from '../core/models/upstream-error.mjs';
|
||||
import { ModelTransport } from '../core/models/transport.mjs';
|
||||
import { ERROR_CAPTURE_LIMIT, readErrorResponse, upstreamCapture } from '../core/models/upstream-error.mjs';
|
||||
|
||||
export const engineToolDefinitions = [
|
||||
{ type: 'function', function: { name: 'inspect_engine_checks', description: '读取当前Task已经保存的外部引擎连接核验:真实安装版本、启动完成信号、SDK会话、进程关闭、原报告工件和能力限制。不会启动引擎或发送prompt;完整原记录通过已有read_artifact读取。', parameters: { type: 'object', properties: {}, additionalProperties: false } } },
|
||||
@@ -88,23 +89,31 @@ export function projectContext(project) {
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
async function boundedResponse(response, maximum = 4 * 1024 * 1024) {
|
||||
async function boundedResponse(response, captureIncomplete, maximum = 4 * 1024 * 1024) {
|
||||
if (!response.body) return '';
|
||||
const reader = response.body.getReader();
|
||||
const parts = []; let length = 0;
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
length += value.length;
|
||||
if (length > maximum) { await reader.cancel(); throw new ModelRequestError({ source: 'model_connection', stage: 'model_request', kind: 'protocol',
|
||||
title: '模型响应超过读取上限', message: '模型服务的响应超过 4 MB 读取上限。', retryable: false, occurredAt: new Date().toISOString() }); }
|
||||
parts.push(value);
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
length += value.length;
|
||||
parts.push(value);
|
||||
if (length > maximum) throw new ModelRequestError({ source: 'model_connection', stage: 'model_request', kind: 'protocol',
|
||||
title: '模型响应超过读取上限', message: '模型服务的响应超过 4 MB 读取上限。', retryable: false, occurredAt: new Date().toISOString() });
|
||||
}
|
||||
return Buffer.concat(parts).toString('utf8');
|
||||
} catch (cause) {
|
||||
captureIncomplete?.({raw:Buffer.concat(parts).subarray(0,ERROR_CAPTURE_LIMIT),receivedBytes:length,truncated:length>ERROR_CAPTURE_LIMIT,interrupted:true});
|
||||
throw cause;
|
||||
} finally {
|
||||
try { await reader.cancel(); } catch {}
|
||||
reader.releaseLock();
|
||||
}
|
||||
return Buffer.concat(parts).toString('utf8');
|
||||
}
|
||||
|
||||
export class Provider {
|
||||
constructor(options = {}) { this.apiKey = process.env.AGENT_WORKBENCH_API_KEY || ''; this.fetcher = options.fetch; this.retryPolicy = options.retryPolicy; }
|
||||
constructor(options = {}) { this.apiKey = process.env.AGENT_WORKBENCH_API_KEY || ''; this.fetcher = options.fetch; this.retryPolicy = options.retryPolicy; this.transport=options.transport; }
|
||||
hasKey() { return !!this.apiKey; }
|
||||
setKey(value) { this.apiKey = String(value ?? '').trim(); }
|
||||
requestOperation(signal) {
|
||||
@@ -121,6 +130,7 @@ export class Provider {
|
||||
const operations = [...(this.requestControllers ?? [])];
|
||||
for (const operation of operations) operation.controller.abort(new Error('用户停止执行宿主。'));
|
||||
await Promise.all(operations.map(operation => operation.done));
|
||||
const transport=this.transport;this.transport=null;await transport?.close();
|
||||
}
|
||||
resumeRequests() { this.requestsClosing = false; }
|
||||
modelHub(store) { return this.hub ??= new ModelHub(store, this); }
|
||||
@@ -170,7 +180,9 @@ export class Provider {
|
||||
};
|
||||
try {
|
||||
dispatched = true;
|
||||
const response = await (this.fetcher ?? fetch)(base + route, { method: body ? 'POST' : 'GET', headers,
|
||||
const transport=this.fetcher?null:(this.transport??=new ModelTransport());
|
||||
if(transport)onProgress?.({phase:'running',transport:transport.profile,attempt,maxAttempts:policy.attempts,wireAttemptId:receipt.attemptId});
|
||||
const response = await (this.fetcher??transport.fetch.bind(transport))(base + route, { method: body ? 'POST' : 'GET', headers,
|
||||
body: body ? JSON.stringify(body) : undefined, signal: requestSignal });
|
||||
const metadata = upstreamCapture({ raw: Buffer.alloc(0), status: response.status, headers: response.headers }, value => this.redact(value, key)).details;
|
||||
receipt.upstreamRequestId = metadata.requestId; receipt.httpStatus = response.status;
|
||||
@@ -180,6 +192,7 @@ export class Provider {
|
||||
if (!received.truncated && !received.interrupted) { try { usage = reportedUsage(config, JSON.parse(received.raw.toString('utf8'))); } catch {} }
|
||||
const upstream = captureResponse(response, received);
|
||||
requestSignal.throwIfAborted();
|
||||
if(received.interrupted)throw new TypeError('模型错误响应在读取完成前中断。');
|
||||
throw httpFailure(response.status, (upstream.message ?? upstream.rawPreview).slice(0, 600), { ...context, upstream });
|
||||
}
|
||||
if (response.ok && body?.stream === true) {
|
||||
@@ -196,7 +209,7 @@ export class Provider {
|
||||
onProgress?.({ phase: 'completed', attempt, maxAttempts: policy.attempts, durationMs: Date.now() - startedAt, stream: streamed.evidence });
|
||||
return value;
|
||||
}
|
||||
const text = await boundedResponse(response);
|
||||
const text = await boundedResponse(response,input=>captureResponse(response,input));
|
||||
try { usage = reportedUsage(config, JSON.parse(text)); } catch {}
|
||||
let result;
|
||||
try { result = JSON.parse(text); } catch {
|
||||
@@ -227,7 +240,8 @@ export class Provider {
|
||||
} catch (error) {
|
||||
outcome = signal?.aborted ? 'interrupted' : readingStream && !streamTerminalSeen ? 'unknown' : 'failed';
|
||||
signal?.throwIfAborted();
|
||||
let diagnostic = connectionFailure(error, { ...context, timedOut: !!timeout.timeout, timeout: timeout.timeout, redact: value => this.redact(value, key) });
|
||||
let diagnostic = connectionFailure(error, { ...context, method:body?'POST':'GET',responseReceived:receipt.httpStatus!==undefined,
|
||||
transport:this.transport?.profile,timedOut: !!timeout.timeout, timeout: timeout.timeout, redact: value => this.redact(value, key) });
|
||||
diagnostic = { ...diagnostic, origin: diagnostic.origin ?? 'local', wireAttemptId: receipt.attemptId, upstreamRequestId: diagnostic.upstreamRequestId ?? receipt.upstreamRequestId,
|
||||
...(responseDetails && !diagnostic.upstream ? { upstream: responseDetails } : {}) };
|
||||
if (readingStream) diagnostic = { ...diagnostic, retryable: false, failoverAllowed: false, outcome: streamTerminalSeen ? 'failed' : 'unknown',
|
||||
@@ -235,6 +249,7 @@ export class Provider {
|
||||
...(diagnostic.origin === 'upstream' ? { suggestion: '可查看原失败事件、上游请求 ID 和已保存的响应。核对服务状态后手动续接;该流不会自动重发。' } : {}),
|
||||
message: diagnostic.message + (diagnostic.origin === 'upstream' ? ' 已保存上游失败事件与原流,工具未执行,不会自动重发。'
|
||||
: streamTerminalSeen ? ' 服务流已终结,响应校验未通过;原流保留,不自动重发。' : ' 已接收的流片段保留,请求不会自动重发。') };
|
||||
outcome=diagnostic.outcome??outcome;
|
||||
const remaining = deadline - Date.now();
|
||||
const waitMs = diagnostic.retryAfterMs ?? policy.delays[Math.min(attempt - 1, policy.delays.length - 1)] ?? 500;
|
||||
const retry = diagnostic.retryable && attempt < policy.attempts && remaining > waitMs;
|
||||
|
||||
Reference in New Issue
Block a user