From e73bca94a04263b09d0b3927e0ae7444f4626c0d Mon Sep 17 00:00:00 2001 From: Agent Workbench Dev Date: Mon, 2 Nov 2026 20:11:21 +0800 Subject: [PATCH] fix: isolate model transport and stop replaying unknown requests --- docs/LIVE-EXECUTION-REPAIR-PLAN.md | 2 +- docs/QA-LIVE-NETWORK-B.md | 22 +++++ findings.md | 5 ++ package-lock.json | 5 +- package.json | 3 +- progress.md | 3 + src/core/models/transport.mjs | 17 ++++ src/server/diagnostics.mjs | 32 ++++++-- src/server/provider.mjs | 43 ++++++---- task_plan.md | 11 +++ tests/diagnostics.test.mjs | 25 ++++-- tests/model-transport-http.test.mjs | 39 +++++++++ tests/model-transport.test.mjs | 123 ++++++++++++++++++++++++++++ 13 files changed, 299 insertions(+), 31 deletions(-) create mode 100644 docs/QA-LIVE-NETWORK-B.md create mode 100644 src/core/models/transport.mjs create mode 100644 tests/model-transport-http.test.mjs create mode 100644 tests/model-transport.test.mjs diff --git a/docs/LIVE-EXECUTION-REPAIR-PLAN.md b/docs/LIVE-EXECUTION-REPAIR-PLAN.md index b049d698..13d86e79 100644 --- a/docs/LIVE-EXECUTION-REPAIR-PLAN.md +++ b/docs/LIVE-EXECUTION-REPAIR-PLAN.md @@ -1,6 +1,6 @@ # 真实执行可靠性修复:下一切片 -起点为 [真实模型/Purge原样本结果](LIVE-MODEL-PURGE-RESULTS.md)。六例未交付,当前优先修复阻止工程闭环的基础问题;[1.25要求与交付复核](PLAN-1.25.md)和后续产品路线继续保留。A已完成源码修复和两协议16K工程链路验收,见[QA-LIVE-CONTEXT-A](QA-LIVE-CONTEXT-A.md);尚未部署正式,B–E按下表继续,不代表真实模型或Purge收益已经通过。 +起点为 [真实模型/Purge原样本结果](LIVE-MODEL-PURGE-RESULTS.md)。六例未交付,当前优先修复阻止工程闭环的基础问题;[1.25要求与交付复核](PLAN-1.25.md)和后续产品路线继续保留。A已完成源码修复和两协议16K工程链路验收,见[QA-LIVE-CONTEXT-A](QA-LIVE-CONTEXT-A.md);B已完成所属连接/未知POST/原片段修复,见[QA-LIVE-NETWORK-B](QA-LIVE-NETWORK-B.md)。尚未部署正式,C–E按下表继续,不代表真实模型或Purge收益已经通过。 ## 执行顺序 diff --git a/docs/QA-LIVE-NETWORK-B.md b/docs/QA-LIVE-NETWORK-B.md new file mode 100644 index 00000000..cd8553bd --- /dev/null +++ b/docs/QA-LIVE-NETWORK-B.md @@ -0,0 +1,22 @@ +# 真实执行修复 B:模型连接与未知结果 + +本切片修复原 `UND_ERR_CONNECT_TIMEOUT` 约十秒连接失败的诊断及请求重放边界。正式 1.24 服务、Provider 和 Key 未变;全部请求来自独立数据/端口、自有 HTTP 服务。本轮真实模型请求为 0,尚不能宣称原真实线路已恢复或自主工程交付通过。 + +## 最终行为 + +- 固定生产依赖 Undici 7.30.1。Provider 拥有所属 dispatcher,启用双栈候选和 250ms 候选间隔,连接上限 10s;不更改全局 DNS、不关闭 IPv6。HTTP(S) 环境代理和 NO_PROXY 生效,代理自身也使用所属连接策略,诊断仅保存存在位。 +- 连接超时、响应等待和正文中断分别记录。完整错误链明确证明未发送模型正文时才可按原预算重试;已发送后 reset、等待超时、正文断裂,或原因不完整时记录 unknown,不自动重发或切池。完整 HTTP 429/503 等原重试策略保持。 +- 非流式正文中断也封存最多 64KiB 原片段、完整性标记和上游请求 ID;未知用量保持未知。片段不执行工具,也不充当完整响应。 +- 停止先取消所属请求,再关闭所属 dispatcher;保留 Provider/凭据对象,显式恢复可重建连接。续接保留原 Task、工具账本和已完成命令,不重放副作用;继续尊重连接冷却。 + +## 验证与原失败 + +相关源码回归 94/94 通过:诊断、真实本机双栈/代理/绕过/取消、未知 POST/正文、池路由、协议、SSE、原错误、请求计划。TypeScript、桌面/手机前端、服务端及桌面构建通过。原 engine-worker 14/14 通过;最终编译宿主 HTTP 200/503 正文中断及冷却后显式续接 2/2 通过。 + +原诊断首轮 9/12 保留:旧测试错误地把 ECONNRESET 当安全自动重试。现在明确覆盖未知停止/显式续接不重放已批准计数命令;另用明确 ECONNREFUSED 保留安全退避/耗尽测试,修正后 12/12。新增编译宿主首轮合计 14/16、状态复核 0/2 均保留:失败是验证器在已有冷却结束前立刻续接,并非再次自动请求;最终按原冷却收据等待后两组通过。原 14 项不重复运行、不重写旧结果。 + +A 的 621 份 / 25,521,387 字节原封存再次离线通过,manifest SHA-256 仍为 `f0bfd70ba570ed48391511a9279e8857b3f1ab9c3319d738d355d1b03548b9f5`,646 条工件记录及 20 次自有请求 / 10 个有界视图保持。封存详见 `live-network-b-evidence/manifest.json`,其中明确区分原失败与最终相关日志,不把固定 HTTP 结果记为真实模型或 Purge 收益。 + +## 回顾与下一步 + +单看 `fetch failed` 无法判断鉴权或生成超时;连接候选、原错误阶段和上游请求身份必须一起判断。网络异常之后的响应是否发生无法凭空确定,可靠恢复必须保留 unknown 和原工具结果。当前仍需 C 的实际模型身份/已知用量预算校准,随后 D 的原版插件隔离与 E 的新真实批次;原六例失败不回写,1.25 产品路线继续保留。 diff --git a/findings.md b/findings.md index 30394096..ab5912ac 100644 --- a/findings.md +++ b/findings.md @@ -909,3 +909,8 @@ I轮实际Edge/Electron各7组、手机真实配对TLS网关观察2组,共16 ### 1.24 里程碑结论 当前读取、SHA、要求修订与显式来源能够证明采用的文件及归属,不能单独证明全部业务正确。无变更交付必须保留实际修改/检查/Git和原失败的区别;新增验收器也必须同时覆盖源码和真正便携/安装目录。最终证据按A原完整716/B严格输入与日志绑定/C完整界面及B实际安装分别归属,避免把测试来源混成新一轮成功。 +# 真实执行 B:连接与未知结果接续 + +原UND_ERR_CONNECT_TIMEOUT属于建连阶段。Provider现在拥有固定Undici7.30.1 dispatcher,双栈候选在请求内选择,不更改全局DNS或关闭IPv6;代理连接自身也使用同一候选/超时配置,环境代理与NO_PROXY只输出存在位。POST的ECONNRESET或已发送后等待超时不能证明上游未处理,应保存unknown、禁止自动重试/切池。只有完整、有界、全部明确预发送的错误链可以安全重试,截断/环形/含未知叶节点均不能据已知部分推断。 + +诊断旧测试9/12原失败保持在.qa/live-network-diagnostics.tap;修正的是“reset可安全重试”的过时假设。新增显式续接不重放已批准命令断言,明确ECONNREFUSED继续覆盖重试耗尽/退避停止,12/12通过。HTTP正文中途断裂现保存64KiB以内原片段/请求ID,完整性标为interrupted,不把缺失usage填零。 diff --git a/package-lock.json b/package-lock.json index c4f1e7a9..450e604f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,8 @@ "react": "^19.2.0", "react-dom": "^19.2.0", "reflect-metadata": "0.2.2", - "smol-toml": "1.9.1" + "smol-toml": "1.9.1", + "undici": "7.30.1" }, "devDependencies": { "@types/node": "^24.0.0", @@ -2484,9 +2485,7 @@ "version": "7.30.1", "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", - "dev": true, "license": "MIT", - "optional": true, "engines": { "node": ">=20.18.1" } diff --git a/package.json b/package.json index 7941332d..7f4dba5d 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,8 @@ "react": "^19.2.0", "react-dom": "^19.2.0", "reflect-metadata": "0.2.2", - "smol-toml": "1.9.1" + "smol-toml": "1.9.1", + "undici": "7.30.1" }, "devDependencies": { "@types/node": "^24.0.0", diff --git a/progress.md b/progress.md index 8d5c283c..2dce3379 100644 --- a/progress.md +++ b/progress.md @@ -1393,3 +1393,6 @@ A独立源码12批合计716/716、0失败/取消/跳过/待办;实际Electron 先导实测收口:520份原件/11208828字节及清单已封存,15份实际来源身份、20请求/19原SSE用量、四拒绝输入、同源条件和6份Git基线72文件离线验证通过,常见敏感格式扫描0命中。实际监督4次检查批准、0应用/续接/Git确认/实现提示;验收通过的是证据真实性及状态保持,不是自主工程成功。最终正式状态相对恢复后的完整当前快照再次逐行/全导出/工件/模型中心完全一致,仍1.24/PID36552/原池/hasKey=true、无活动。报告、修复优先方案、NEXT-STEPS和AGENTS已同步,保留1.25后续路线,按本次范围本地Git,无远端推送或正式载荷变更。 提交预检首次指出原TAP中的空格行,原始日志不修剪;只为本版证据TAP设binary属性后重验diff检查。532个所属暂存文件及521份证据/manifest原字节批量Git对象核对通过;四份普通Markdown仅正常CRLF→LF,封存原件不发生换行转换。 +# 真实执行 B 接续:网络请求归属与重放边界 + +继续沿用planning-with-files和原修复方案;未启动子代理,未操作正式宿主或Key,未发起真实模型请求。A原实现与证据保持。旧诊断9/12原失败已保留,未知reset停止并显式续接不重放计数命令、明确拒绝连接重试/退避停止的修正后12/12通过。新增正文断裂原片段/请求ID、错误链不完整拒绝重放、模型池不切换和dispatcher显式复用,正在定向验证。工具恢复时一次多文件读取输出截断,随后限定短段,截断内容不作完整审查结论。 diff --git a/src/core/models/transport.mjs b/src/core/models/transport.mjs new file mode 100644 index 00000000..5c452ecf --- /dev/null +++ b/src/core/models/transport.mjs @@ -0,0 +1,17 @@ +import { Agent,EnvHttpProxyAgent,fetch } from 'undici'; + +export class ModelTransport { + constructor({environment=process.env,lookup,connectTimeoutMs=10000}={}){ + const setting=(lower,upper)=>environment[lower]??environment[upper]??''; + this.proxy={httpProxy:setting('http_proxy','HTTP_PROXY'),httpsProxy:setting('https_proxy','HTTPS_PROXY'),noProxy:setting('no_proxy','NO_PROXY')}; + this.connect={autoSelectFamily:true,autoSelectFamilyAttemptTimeout:250,timeout:connectTimeoutMs,...(lookup?{lookup}:{})}; + this.profile={schemaVersion:1,implementation:'undici-7.30.1',autoSelectFamily:true,familyAttemptTimeoutMs:250,connectTimeoutMs, + httpProxyConfigured:!!this.proxy.httpProxy,httpsProxyConfigured:!!this.proxy.httpsProxy,noProxyConfigured:!!this.proxy.noProxy}; + } + fetch(url,options){ + this.dispatcher??=this.profile.httpProxyConfigured||this.profile.httpsProxyConfigured + ?new EnvHttpProxyAgent({...this.proxy,connect:this.connect,proxyTls:this.connect,requestTls:this.connect}):new Agent({connect:this.connect}); + return fetch(url,{...options,dispatcher:this.dispatcher}); + } + async close(){const dispatcher=this.dispatcher;this.dispatcher=null;await dispatcher?.close();} +} diff --git a/src/server/diagnostics.mjs b/src/server/diagnostics.mjs index cf34075d..2972d207 100644 --- a/src/server/diagnostics.mjs +++ b/src/server/diagnostics.mjs @@ -21,25 +21,45 @@ export class ModelRequestError extends Error { } } -export function connectionFailure(error, { endpoint, model, route, attempt, redact = String, timedOut = false, timeout = null }) { +export function connectionFailure(error, { endpoint, model, route, attempt, redact = String, timedOut = false, timeout = null,method='POST',responseReceived=false,transport }) { if (error instanceof ModelRequestError) return { ...error.diagnostic, endpoint: error.diagnostic.endpoint ?? endpoint, model: error.diagnostic.model ?? model, route: error.diagnostic.route ?? route, attempts: attempt }; - const causes = [error, error?.cause, ...(error?.cause?.errors ?? [])]; + const causes=[],leaves=[],seen=new Set();let causesComplete=true; + const visit=(value,depth=0)=>{ + if(!value||typeof value!=='object'||seen.has(value)||depth>4||causes.length>=16){causesComplete=false;return;} + seen.add(value);causes.push(value); + const children=[...(value.cause?[value.cause]:[]),...(Array.isArray(value.errors)?value.errors:[])]; + if(!children.length)leaves.push(value); + if(children.length>8)causesComplete=false; + for(const child of children.slice(0,8))visit(child,depth+1); + };visit(error); const code = causes.map(item => item?.code).find(value => typeof value === 'string' && /^[A-Z0-9_]{2,60}$/.test(value)) ?? null; - const kind = timedOut || causes.some(item => item?.name === 'TimeoutError') ? 'timeout' + const connectTimeout=causes.some(item=>item.code==='UND_ERR_CONNECT_TIMEOUT'||item.code==='ETIMEDOUT'&&item.syscall==='connect'); + const kind = timedOut || connectTimeout || causes.some(item => item?.name === 'TimeoutError') ? 'timeout' : ['ENOTFOUND', 'EAI_AGAIN'].includes(code) ? 'dns' : /CERT|TLS|SSL/.test(code ?? '') ? 'certificate' : 'network'; - const title = kind === 'timeout' ? '模型服务响应超时' : kind === 'dns' ? '模型服务地址解析失败' + const title = connectTimeout ? '模型服务连接超时' : kind === 'timeout' ? '模型服务响应超时' : kind === 'dns' ? '模型服务地址解析失败' : kind === 'certificate' ? '模型服务证书校验失败' : '模型服务连接失败'; + const coded=causes.filter(item=>typeof item.code==='string'&&/^[A-Z0-9_]{2,60}$/.test(item.code)); + const preSend=item=>['ENOTFOUND','EAI_AGAIN','ECONNREFUSED','EHOSTUNREACH','ENETUNREACH','UND_ERR_CONNECT_TIMEOUT'].includes(item.code) + ||item.code==='ETIMEDOUT'&&item.syscall==='connect'||/CERT|TLS|SSL/.test(item.code??''); + const beforeSend=!responseReceived&&!timedOut&&causesComplete&&coded.length>0&&coded.every(preSend)&&leaves.every(preSend); + const unknown=method==='POST'&&!beforeSend; + const network={stage:responseReceived?'response_body':kind==='dns'?'dns':kind==='certificate'?'tls':connectTimeout||beforeSend?'connect':timedOut?'waiting_response':'unknown', + replaySafety:beforeSend?'before_model_send':unknown?'unknown_after_dispatch':'read_only_request',responseReceived,causesComplete, + ...(transport?{transport}:{}),causes:coded.map(item=>({code:item.code,...(typeof item.syscall==='string'&&/^[a-z_]{1,40}$/.test(item.syscall)?{syscall:item.syscall}:{}), + ...(item.family===4||item.family===6?{family:item.family}:{})}))}; return { id: randomUUID(), source: 'model_connection', origin: 'local', stage: 'model_request', kind, code, endpoint, model, route, attempts: attempt, occurredAt: new Date().toISOString(), - retryable: kind !== 'certificate', title, + retryable: kind !== 'certificate'&&!unknown,failoverAllowed:false,outcome:unknown?'unknown':'failed',network,title, message: title + (code ? '(' + code + ')' : '') + '。' + (timeout ? timeout.kind === 'idle' ? '流式响应连续 ' + Math.round(timeout.limitMs / 1000) + ' 秒未收到新数据。' - : '已达到本次模型响应 ' + Math.round(timeout.limitMs / 1000) + ' 秒等待上限。' : '') + '项目文件和已完成的工具记录已保留。', + : '已达到本次模型响应 ' + Math.round(timeout.limitMs / 1000) + ' 秒等待上限。' : '') + (unknown?'原POST结果未知,不会自动重发或切池。':'')+'项目文件和已完成的工具记录已保留。', ...(timeout ? { timeout } : {}), suggestion: kind === 'certificate' ? '核对服务地址和证书配置。' + : connectTimeout ? '核对网络、DNS、IPv4/IPv6候选及代理;这是建立连接时超时,增加模型响应等待时限不能修复连接。' + : unknown ? '核对原请求与上游状态后再显式续接;未知用量保持未知,已完成工具不得重放。' : timeout ? '在模型中心的“等待时限”调整设置,或核对服务状态后手动续接;任务总预算仍有效。' : '可以重试上次模型请求;若持续失败,在模型连接中检查服务地址和连接状态。', detail: redact(kind === 'network' ? '网络请求未取得完整模型响应。' : title), diff --git a/src/server/provider.mjs b/src/server/provider.mjs index f8f638c1..a7358666 100644 --- a/src/server/provider.mjs +++ b/src/server/provider.mjs @@ -7,7 +7,8 @@ import { ModelHub } from '../core/models/model-hub.mjs'; import { normalizeProtocolResponse, protocolRoute, validateModelResponse, reportedUsage } from '../core/models/protocols.mjs'; import { readModelStream } from '../core/models/model-stream.mjs'; import { modelWaitSettings, requestDeadline } from '../core/models/request-deadline.mjs'; -import { readErrorResponse, upstreamCapture } from '../core/models/upstream-error.mjs'; +import { ModelTransport } from '../core/models/transport.mjs'; +import { ERROR_CAPTURE_LIMIT, readErrorResponse, upstreamCapture } from '../core/models/upstream-error.mjs'; export const engineToolDefinitions = [ { type: 'function', function: { name: 'inspect_engine_checks', description: '读取当前Task已经保存的外部引擎连接核验:真实安装版本、启动完成信号、SDK会话、进程关闭、原报告工件和能力限制。不会启动引擎或发送prompt;完整原记录通过已有read_artifact读取。', parameters: { type: 'object', properties: {}, additionalProperties: false } } }, @@ -88,23 +89,31 @@ export function projectContext(project) { ].join('\n'); } -async function boundedResponse(response, maximum = 4 * 1024 * 1024) { +async function boundedResponse(response, captureIncomplete, maximum = 4 * 1024 * 1024) { if (!response.body) return ''; const reader = response.body.getReader(); const parts = []; let length = 0; - while (true) { - const { value, done } = await reader.read(); - if (done) break; - length += value.length; - if (length > maximum) { await reader.cancel(); throw new ModelRequestError({ source: 'model_connection', stage: 'model_request', kind: 'protocol', - title: '模型响应超过读取上限', message: '模型服务的响应超过 4 MB 读取上限。', retryable: false, occurredAt: new Date().toISOString() }); } - parts.push(value); + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + length += value.length; + parts.push(value); + if (length > maximum) throw new ModelRequestError({ source: 'model_connection', stage: 'model_request', kind: 'protocol', + title: '模型响应超过读取上限', message: '模型服务的响应超过 4 MB 读取上限。', retryable: false, occurredAt: new Date().toISOString() }); + } + return Buffer.concat(parts).toString('utf8'); + } catch (cause) { + captureIncomplete?.({raw:Buffer.concat(parts).subarray(0,ERROR_CAPTURE_LIMIT),receivedBytes:length,truncated:length>ERROR_CAPTURE_LIMIT,interrupted:true}); + throw cause; + } finally { + try { await reader.cancel(); } catch {} + reader.releaseLock(); } - return Buffer.concat(parts).toString('utf8'); } export class Provider { - constructor(options = {}) { this.apiKey = process.env.AGENT_WORKBENCH_API_KEY || ''; this.fetcher = options.fetch; this.retryPolicy = options.retryPolicy; } + constructor(options = {}) { this.apiKey = process.env.AGENT_WORKBENCH_API_KEY || ''; this.fetcher = options.fetch; this.retryPolicy = options.retryPolicy; this.transport=options.transport; } hasKey() { return !!this.apiKey; } setKey(value) { this.apiKey = String(value ?? '').trim(); } requestOperation(signal) { @@ -121,6 +130,7 @@ export class Provider { const operations = [...(this.requestControllers ?? [])]; for (const operation of operations) operation.controller.abort(new Error('用户停止执行宿主。')); await Promise.all(operations.map(operation => operation.done)); + const transport=this.transport;this.transport=null;await transport?.close(); } resumeRequests() { this.requestsClosing = false; } modelHub(store) { return this.hub ??= new ModelHub(store, this); } @@ -170,7 +180,9 @@ export class Provider { }; try { dispatched = true; - const response = await (this.fetcher ?? fetch)(base + route, { method: body ? 'POST' : 'GET', headers, + const transport=this.fetcher?null:(this.transport??=new ModelTransport()); + if(transport)onProgress?.({phase:'running',transport:transport.profile,attempt,maxAttempts:policy.attempts,wireAttemptId:receipt.attemptId}); + const response = await (this.fetcher??transport.fetch.bind(transport))(base + route, { method: body ? 'POST' : 'GET', headers, body: body ? JSON.stringify(body) : undefined, signal: requestSignal }); const metadata = upstreamCapture({ raw: Buffer.alloc(0), status: response.status, headers: response.headers }, value => this.redact(value, key)).details; receipt.upstreamRequestId = metadata.requestId; receipt.httpStatus = response.status; @@ -180,6 +192,7 @@ export class Provider { if (!received.truncated && !received.interrupted) { try { usage = reportedUsage(config, JSON.parse(received.raw.toString('utf8'))); } catch {} } const upstream = captureResponse(response, received); requestSignal.throwIfAborted(); + if(received.interrupted)throw new TypeError('模型错误响应在读取完成前中断。'); throw httpFailure(response.status, (upstream.message ?? upstream.rawPreview).slice(0, 600), { ...context, upstream }); } if (response.ok && body?.stream === true) { @@ -196,7 +209,7 @@ export class Provider { onProgress?.({ phase: 'completed', attempt, maxAttempts: policy.attempts, durationMs: Date.now() - startedAt, stream: streamed.evidence }); return value; } - const text = await boundedResponse(response); + const text = await boundedResponse(response,input=>captureResponse(response,input)); try { usage = reportedUsage(config, JSON.parse(text)); } catch {} let result; try { result = JSON.parse(text); } catch { @@ -227,7 +240,8 @@ export class Provider { } catch (error) { outcome = signal?.aborted ? 'interrupted' : readingStream && !streamTerminalSeen ? 'unknown' : 'failed'; signal?.throwIfAborted(); - let diagnostic = connectionFailure(error, { ...context, timedOut: !!timeout.timeout, timeout: timeout.timeout, redact: value => this.redact(value, key) }); + let diagnostic = connectionFailure(error, { ...context, method:body?'POST':'GET',responseReceived:receipt.httpStatus!==undefined, + transport:this.transport?.profile,timedOut: !!timeout.timeout, timeout: timeout.timeout, redact: value => this.redact(value, key) }); diagnostic = { ...diagnostic, origin: diagnostic.origin ?? 'local', wireAttemptId: receipt.attemptId, upstreamRequestId: diagnostic.upstreamRequestId ?? receipt.upstreamRequestId, ...(responseDetails && !diagnostic.upstream ? { upstream: responseDetails } : {}) }; if (readingStream) diagnostic = { ...diagnostic, retryable: false, failoverAllowed: false, outcome: streamTerminalSeen ? 'failed' : 'unknown', @@ -235,6 +249,7 @@ export class Provider { ...(diagnostic.origin === 'upstream' ? { suggestion: '可查看原失败事件、上游请求 ID 和已保存的响应。核对服务状态后手动续接;该流不会自动重发。' } : {}), message: diagnostic.message + (diagnostic.origin === 'upstream' ? ' 已保存上游失败事件与原流,工具未执行,不会自动重发。' : streamTerminalSeen ? ' 服务流已终结,响应校验未通过;原流保留,不自动重发。' : ' 已接收的流片段保留,请求不会自动重发。') }; + outcome=diagnostic.outcome??outcome; const remaining = deadline - Date.now(); const waitMs = diagnostic.retryAfterMs ?? policy.delays[Math.min(attempt - 1, policy.delays.length - 1)] ?? 500; const retry = diagnostic.retryable && attempt < policy.attempts && remaining > waitMs; diff --git a/task_plan.md b/task_plan.md index 7aa1309d..c7e3c081 100644 --- a/task_plan.md +++ b/task_plan.md @@ -1,5 +1,16 @@ # Agent Workbench:核心代理能力研究与产品计划 +## 当前修复B/C:连接阶段诊断与实际模型身份 + +A已提交0a58c0d/6257bba并封存。继续先完成网络与报告中的确定问题,再校准后续预算和原版插件隔离;不请求真实模型,不改正式Provider。原GPT失败是UND_ERR_CONNECT_TIMEOUT、约10秒连接阶段,而非300秒响应等待;当前没有HTTP(S)_PROXY环境设置,不能推断是Key或模型拒绝。 + +1. [complete] 固定Node24.22.1/Undici7.30.1;原失败明确连接阶段,当前HTTP(S)_PROXY/NO_PROXY仅存在位核对、均无设置。连接/响应等待/正文中断分开,不导出代理或密钥。 +2. [complete] Provider所属双栈与代理/NO_PROXY连接策略;未知POST不自动重发/切池,明确完整预发送错误链才可重试。HTTP正文中断保留有界原片段、上游请求ID与未知消费;取消后关闭所属dispatcher,可显式重建。 +3. [complete] 自有双栈/代理/绕过、未知POST/正文、取消复用与工具不重放包含在94/94相关回归;构建通过。engine-worker原14/14和最终编译宿主200/503正文中断、冷却后显式续接2/2分别通过。A原621份/646工件/20夹具请求离线核对保持。 +4. [in_progress] 修正真实验收器实际模型身份来源,区分零请求、混合路由与未知用量;按范围本地Git和回顾,不把诊断连通当真实模型鉴权通过。C身份纯函数四项通过,尚未正式接入验收器或预算校准。 + +B原94/94、构建和A封存通过。编译/引擎首轮14/16与状态复核0/2保留:两项是验收在原连接冷却结束前立刻续接,最终按原cooldownUntil等待后2/2,未重复已通过14项。B见docs/QA-LIVE-NETWORK-B.md。一次猜测budget.mjs不存在已改按rg实际定位execution.mjs;文档新增尾空行已清理。 + ## 当前修复:真实执行上下文与可诊断容量(1.24.1切片) 接续b67b4f2,分支fix/live-execution-context-1.24.1。用户继续授权执行已记录修复方案;先A上下文,再B连接/C预算身份,不开启子代理。正式1.24/PID36552/hasKey=true、无活动,原池保持;开发与验证使用隔离数据/端口,先离线复现,不续跑原六例。 diff --git a/tests/diagnostics.test.mjs b/tests/diagnostics.test.mjs index 69692a21..4394e330 100644 --- a/tests/diagnostics.test.mjs +++ b/tests/diagnostics.test.mjs @@ -11,6 +11,7 @@ import { fixture, compatibleFixture, call, completion, until, assertToolPairs } const policy = { attempts: 3, delays: [1, 2], timeoutMs: 1000, totalTimeoutMs: 3000 }; const response = result => new Response(JSON.stringify(result), { headers: { 'Content-Type': 'application/json' } }); const disconnected = () => new TypeError('fetch failed', { cause: Object.assign(new Error('connection reset'), { code: 'ECONNRESET' }) }); +const refused = () => new TypeError('fetch failed', { cause: Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' }) }); const configure = f => f.store.saveConfig({ mode: 'compatible', baseUrl: 'http://127.0.0.1:1/v1', model: 'fixture-agent' }); test('a question about the previous workbench error receives local error context and current change states', async t => { @@ -41,7 +42,7 @@ test('a question about the previous workbench error receives local error context assert.ok(toolDefinitions.some(tool => tool.function.name === 'inspect_session')); }); -test('a transient model failure after an approved command retries only the model request', async t => { +test('an unknown model request stops, and explicit resume never replays the approved command', async t => { const f = fixture(t); configure(f); fs.writeFileSync(path.join(f.projectRoot, 'counter.mjs'), "import fs from 'node:fs'; const previous = fs.existsSync('count.txt') ? Number(fs.readFileSync('count.txt','utf8')) : 0; fs.writeFileSync('count.txt', String(previous + 1));\n"); let requests = 0; @@ -56,26 +57,38 @@ test('a transient model failure after an approved command retries only the model await until(() => f.session.status === 'awaiting_approval'); runtime.decide(f.session.id, f.session.approvals[0].id, true); await done; assert.equal(fs.readFileSync(path.join(f.projectRoot, 'count.txt'), 'utf8'), '1'); + assert.equal(requests, 2); assert.equal(f.session.status, 'failed'); + assert.equal(f.session.failure.outcome, 'unknown'); + assert.equal(f.session.failure.retryable, false); + assert.equal(f.session.failure.failoverAllowed, false); + assert.equal(f.session.failure.attempts, 1); + const failedEvent = f.session.events.filter(event => event.kind === 'model').at(-1); + assert.equal(failedEvent.phase, 'failed'); assert.equal(failedEvent.diagnostic.outcome, 'unknown'); + const userTurns = f.session.providerMessages.filter(message => message.role === 'user').length; + runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise; assert.equal(requests, 3); assert.equal(f.session.status, 'completed'); + assert.equal(fs.readFileSync(path.join(f.projectRoot, 'count.txt'), 'utf8'), '1'); + assert.equal(f.session.providerMessages.filter(message => message.role === 'user').length, userTurns); assert.equal(f.session.approvals.length, 1); assert.equal(f.session.events.filter(event => event.tool === 'run_command').length, 1); - assert.equal(f.session.events.filter(event => event.kind === 'model').at(-1).attempts, 2); + assert.equal(f.session.events.filter(event => event.kind === 'model').at(-1).attempts, 1); + assert.ok(f.session.events.includes(failedEvent)); assertToolPairs(f.session.providerMessages); }); -test('exhausted model retries preserve tool results, and explicit resume adds no new user turn or replay', async t => { +test('exhausted refused connections preserve tool results, and explicit resume adds no new user turn or replay', async t => { const f = fixture(t); configure(f); let requests = 0, fail = true; const provider = new Provider({ retryPolicy: { ...policy, attempts: 2 }, fetch: async (_url, input) => { const messages = JSON.parse(input.body).messages; assertToolPairs(messages); if (++requests === 1) return response(completion([call('read', 'read_file', { path: 'README.md' })])); - if (fail) throw disconnected(); + if (fail) throw refused(); assert.equal(messages.at(-1).role, 'tool'); return response(completion([], 'Recovered')); } }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, 'Read the fixture.'); await runtime.active.get(f.session.id).promise; assert.equal(f.session.status, 'failed'); - assert.equal(f.session.failure.source, 'model_connection'); assert.equal(f.session.failure.code, 'ECONNRESET'); + assert.equal(f.session.failure.source, 'model_connection'); assert.equal(f.session.failure.code, 'ECONNREFUSED'); assert.equal(f.session.failure.attempts, 2); const userTurns = f.session.providerMessages.filter(message => message.role === 'user').length; fail = false; runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise; @@ -87,7 +100,7 @@ test('exhausted model retries preserve tool results, and explicit resume adds no test('stopping during model backoff cancels further requests and clears the running request state', async t => { const f = fixture(t); configure(f); let requests = 0; - const provider = new Provider({ retryPolicy: { ...policy, delays: [500, 500] }, fetch: async () => { requests++; throw disconnected(); } }); + const provider = new Provider({ retryPolicy: { ...policy, delays: [500, 500] }, fetch: async () => { requests++; throw refused(); } }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, 'test'); await until(() => f.session.events.some(event => event.phase === 'retrying')); await runtime.stop(f.session.id); diff --git a/tests/model-transport-http.test.mjs b/tests/model-transport-http.test.mjs new file mode 100644 index 00000000..9dd7bc97 --- /dev/null +++ b/tests/model-transport-http.test.mjs @@ -0,0 +1,39 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { fixture,launchWorkbench,completion,call,until } from './helpers.mjs'; + +for(const status of [200,503])test('compiled host retains interrupted HTTP '+status+' and explicitly resumes after the recorded cooldown',async t=>{ + const f=fixture(t);let requests=0,recovered=false; + const server=http.createServer(async(request,response)=>{ + const parts=[];for await(const chunk of request)parts.push(chunk); + const body=JSON.parse(Buffer.concat(parts));requests++; + response.writeHead(recovered?200:status,{'Content-Type':'application/json','X-Request-ID':'owned-compiled-partial-'+status}); + if(!recovered){response.write('{"error":{"message":"owned interrupted');setTimeout(()=>response.destroy(),20);return;} + response.end(JSON.stringify(body.messages.some(message=>message.role==='tool')?completion([],'README已读取,文件保持。') + :completion([call('owned-read','read_file',{path:'README.md'})]))); + }); + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + t.after(()=>new Promise(resolve=>{server.closeAllConnections();server.close(resolve);})); + f.store.close();const host=await launchWorkbench(t,f.data); + const connection=await host.api('/model-center/connections',{name:'Owned compiled transport',model:'owned-compiled', + baseUrl:'http://127.0.0.1:'+server.address().port+'/v1',requiresKey:false}); + await host.api('/model-center/select',{type:'connection',id:connection.id}); + const route='/sessions/'+f.session.id;let state; + await host.api(route+'/messages',{content:'分析README,不修改文件,不检查。'}); + await until(async()=>{state=await host.api(route);return state.status==='failed';},15000); + assert.equal(requests,1);assert.equal(state.failure.outcome,'unknown');assert.equal(state.failure.retryable,false); + assert.equal(state.failure.upstreamRequestId,'owned-compiled-partial-'+status); + assert.equal(state.failure.upstream.capture.interrupted,true);assert.equal(state.execution.run.consumption.length,1); + assert.equal(state.execution.run.consumption[0].usageStatus,'unknown');assert.equal(state.execution.task.usage.requests,1); + const taskId=state.taskId,userTurns=state.messages.filter(message=>message.role==='user').length; + const center=await host.api('/model-center'),cooldown=center.connections.find(value=>value.id===connection.id).health.cooldownUntil; + assert.ok(Number.isFinite(cooldown));await until(()=>Date.now()>=cooldown);assert.equal(requests,1); + recovered=true;await host.api(route+'/resume',{}); + await until(async()=>{state=await host.api(route);return !['running','awaiting_approval','stopping'].includes(state.status);},15000); + assert.equal(state.status,'completed',JSON.stringify({status:state.status,failure:state.failure?.message,stopReason:state.execution.run.stopReason,requests})); + assert.equal(state.taskId,taskId);assert.equal(requests,3); + assert.equal(state.messages.filter(message=>message.role==='user').length,userTurns); + assert.equal(state.events.filter(event=>event.tool==='read_file'&&event.phase==='completed').length,1); + assert.equal(state.execution.task.usage.requests,3);assert.equal(state.execution.task.usage.unknownRequests,1); +}); diff --git a/tests/model-transport.test.mjs b/tests/model-transport.test.mjs new file mode 100644 index 00000000..236957b6 --- /dev/null +++ b/tests/model-transport.test.mjs @@ -0,0 +1,123 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import net from 'node:net'; +import { createHash } from 'node:crypto'; +import { ModelTransport } from '../src/core/models/transport.mjs'; +import { Provider } from '../src/server/provider.mjs'; +import { connectionFailure } from '../src/server/diagnostics.mjs'; +import { Runtime } from '../src/server/runtime.mjs'; +import { fixture,compatibleFixture,completion,until } from './helpers.mjs'; + +const config=baseUrl=>({mode:'compatible',baseUrl,model:'owned-transport',maxOutputTokens:128}); +const policy={attempts:3,delays:[1,1],timeoutMs:1500,totalTimeoutMs:4000}; +const lookup=(_hostname,options,callback)=>{ + assert.equal(options.all,true);callback(null,[{address:'::1',family:6},{address:'127.0.0.1',family:4}]); +}; + +test('owned IPv4 service works after a failed IPv6 candidate without a second model POST or global DNS change',async t=>{ + const model=await compatibleFixture(t,()=>completion([],'Owned dual-stack result')); + const url=new URL(model.baseUrl);url.hostname='owned-transport.invalid'; + const transport=new ModelTransport({environment:{},lookup}),provider=new Provider({transport,retryPolicy:policy}),events=[]; + t.after(()=>provider.abortRequests());const original=net.getDefaultAutoSelectFamily(); + const result=await provider.complete(config(url.href),[],null,null,event=>events.push(event)); + assert.equal(result.message.content,'Owned dual-stack result');assert.equal(model.requests.length,1); + assert.equal(net.getDefaultAutoSelectFamily(),original);assert.ok(events.some(event=>event.transport?.autoSelectFamily)); + assert.equal(provider.lastRequest.attempts,1); +}); + +test('environment HTTP proxy tunnels the owned request and NO_PROXY sends a separate direct request',async t=>{ + const model=await compatibleFixture(t,()=>completion([],'Owned proxy result')),target=new URL(model.baseUrl),sockets=new Set();let tunnels=0; + const proxy=http.createServer();proxy.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));}); + proxy.on('connect',(request,socket,head)=>{ + assert.equal(request.url,target.host);tunnels++; + const upstream=net.connect({host:target.hostname,port:Number(target.port)},()=>{socket.write('HTTP/1.1 200 Connection Established\r\n\r\n');if(head.length)upstream.write(head);socket.pipe(upstream);upstream.pipe(socket);}); + upstream.on('error',()=>socket.destroy());socket.on('error',()=>upstream.destroy());socket.on('close',()=>upstream.destroy()); + }); + await new Promise(resolve=>proxy.listen(0,'127.0.0.1',resolve)); + t.after(async()=>{for(const socket of sockets)socket.destroy();await new Promise(resolve=>proxy.close(resolve));}); + const environment={HTTP_PROXY:'http://owned-proxy.invalid:'+proxy.address().port}; + const through=new Provider({transport:new ModelTransport({environment,lookup}),retryPolicy:policy});t.after(()=>through.abortRequests()); + assert.equal((await through.complete(config(model.baseUrl),[],null)).message.content,'Owned proxy result');assert.equal(tunnels,1); + const bypass=new Provider({transport:new ModelTransport({environment:{...environment,NO_PROXY:'127.0.0.1'}}),retryPolicy:policy});t.after(()=>bypass.abortRequests()); + await bypass.complete(config(model.baseUrl),[],null);assert.equal(tunnels,1);assert.equal(model.requests.length,2); + assert.equal(JSON.stringify(through.transport.profile).includes(environment.HTTP_PROXY),false); +}); + +test('original connect timeout has a bounded nested cause and does not masquerade as response generation timeout',()=>{ + const error=new TypeError('fetch failed',{cause:Object.assign(new Error('socket'),{code:'UND_ERR_CONNECT_TIMEOUT'})}); + const value=connectionFailure(error,{model:'owned',attempt:1}); + assert.equal(value.kind,'timeout');assert.equal(value.network.stage,'connect');assert.equal(value.network.replaySafety,'before_model_send'); + assert.equal(value.retryable,true);assert.equal(value.outcome,'failed');assert.equal(value.title,'模型服务连接超时'); + assert.match(value.suggestion,/增加模型响应等待时限不能修复连接/); + const unreachable=Object.assign(new Error('owned'),{code:'ENETUNREACH',family:6,syscall:'connect'}); + const aggregate=new TypeError('fetch',{cause:new AggregateError([unreachable,Object.assign(new Error('owned'),{code:'ECONNREFUSED',family:4})])}); + const bounded=connectionFailure(aggregate,{attempt:1});assert.equal(bounded.retryable,true);assert.equal(bounded.network.causes.length,2); + unreachable.cause=unreachable; + assert.equal(connectionFailure(aggregate,{attempt:1}).retryable,false,'Incomplete or cyclic causes cannot establish safe replay.'); + for(const errors of [[Object.assign(new Error('refused'),{code:'ECONNREFUSED'}),new Error('unknown failure')], + [...Array.from({length:8},()=>Object.assign(new Error('refused'),{code:'ECONNREFUSED'})),Object.assign(new Error('reset'),{code:'ECONNRESET'})]]){ + const unknown=connectionFailure(new AggregateError(errors),{attempt:1}); + assert.equal(unknown.outcome,'unknown');assert.equal(unknown.retryable,false); + } +}); + +test('POST accepted then reset is unknown, is charged once and never automatically retried',async t=>{ + const sockets=new Set();let posts=0; + const server=http.createServer(async(request,response)=>{for await(const _chunk of request){}posts++;response.socket.destroy();}); + server.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));}); + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + t.after(async()=>{for(const socket of sockets)socket.destroy();await new Promise(resolve=>server.close(resolve));}); + const provider=new Provider({transport:new ModelTransport({environment:{}}),retryPolicy:policy}),events=[];t.after(()=>provider.abortRequests()); + await assert.rejects(provider.complete(config('http://127.0.0.1:'+server.address().port+'/v1'),[],null,null,event=>events.push(event)),error=>{ + assert.equal(error.diagnostic.outcome,'unknown');assert.equal(error.diagnostic.retryable,false);assert.equal(error.diagnostic.failoverAllowed,false);return true; + }); + assert.equal(posts,1);const receipts=events.filter(event=>event.consumption);assert.equal(receipts.length,1);assert.equal(receipts[0].consumption.usageStatus,'unknown'); + assert.equal(receipts[0].consumption.outcome,'unknown'); +}); + +test('partial HTTP bodies retain original request id and bytes, and an unknown pool attempt never switches model',async t=>{ + const f=fixture(t),prefix=Buffer.from('{"choices":[{"message":{"content":"owned partial 中文');let posts=0; + const server=http.createServer(async(request,response)=>{ + for await(const _chunk of request){}posts++; + response.writeHead(200,{'Content-Type':'application/json','X-Request-ID':'owned-partial-http'});response.write(prefix); + setTimeout(()=>response.destroy(),20); + }); + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + t.after(()=>new Promise(resolve=>{server.closeAllConnections();server.close(resolve);})); + const backup=await compatibleFixture(t,()=>completion([],'Must not be called')); + const provider=new Provider({transport:new ModelTransport({environment:{}}),retryPolicy:policy}),hub=provider.modelHub(f.store); + const first=hub.upsert({name:'owned partial',model:'actual-partial',baseUrl:'http://127.0.0.1:'+server.address().port+'/v1',requiresKey:false}); + const second=hub.upsert({name:'owned backup',model:'actual-backup',baseUrl:backup.baseUrl,requiresKey:false}); + const pool=hub.pool({name:'owned unknown pool',members:[{connectionId:first.id},{connectionId:second.id}]});hub.select({type:'pool',id:pool.id}); + const runtime=new Runtime(f.store,provider);t.after(()=>runtime.stopAll()); + runtime.start(f.session.id,'读取项目,不修改、不检查');await runtime.active.get(f.session.id).promise; + assert.equal(f.session.status,'failed');assert.equal(posts,1);assert.equal(backup.requests.length,0); + const failure=f.session.failure,attempt=hub.view().attempts[0],run=f.store.core.run.get(f.session.runId); + assert.equal(hub.view().attempts.length,1);assert.equal(attempt.status,'unknown'); + assert.equal(failure.network.stage,'response_body');assert.equal(failure.outcome,'unknown');assert.equal(failure.upstreamRequestId,'owned-partial-http'); + assert.equal(failure.upstream.capture.complete,false);assert.equal(failure.upstream.capture.interrupted,true); + const captured=f.store.artifactStore.read(f.store.owned('artifact',attempt.responseArtifactId,f.session.id)); + assert.equal(captured,prefix.toString('utf8'));assert.equal(attempt.responseSha256,createHash('sha256').update(prefix).digest('hex')); + assert.equal(run.consumption.length,1);assert.equal(run.consumption[0].outcome,'unknown');assert.equal(run.consumption[0].usageStatus,'unknown'); + assert.equal(run.usage.requests,1);assert.equal(run.usage.unknownRequests,1);assert.equal(f.store.core.tool.size,0); +}); + +test('response wait timeout after an accepted POST is unknown; cancellation closes the owned socket and permits explicit reuse',async t=>{ + let posts=0,closed=0;const sockets=new Set(); + const server=http.createServer(async(request,_response)=>{for await(const _chunk of request){}posts++;request.socket.once('close',()=>closed++);}); + server.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));}); + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + t.after(async()=>{for(const socket of sockets)socket.destroy();await new Promise(resolve=>server.close(resolve));}); + const provider=new Provider({transport:new ModelTransport({environment:{}}),retryPolicy:{...policy,timeoutMs:100,totalTimeoutMs:100}});t.after(()=>provider.abortRequests()); + const endpoint=config('http://127.0.0.1:'+server.address().port+'/v1'); + await assert.rejects(provider.complete(endpoint,[],null),error=>error.diagnostic.outcome==='unknown'&&error.diagnostic.network.stage==='waiting_response'); + assert.equal(posts,1);assert.equal(provider.lastRequest.attempts,1); + provider.retryPolicy=policy;const controller=new AbortController(),pending=provider.complete(endpoint,[],null,controller.signal); + await until(()=>posts===2);controller.abort(new Error('Owned user cancellation'));await assert.rejects(pending,/Owned user cancellation/); + await provider.abortRequests();assert.equal(provider.activeRequests,0);assert.ok(closed>=1);assert.equal(provider.transport,null);provider.resumeRequests(); + assert.equal(posts,2); + const working=await compatibleFixture(t,()=>completion([],'Explicit reuse works')); + assert.equal((await provider.complete(config(working.baseUrl),[],null)).message.content,'Explicit reuse works'); + assert.equal(working.requests.length,1); +});