feat: add bounded connection network diagnostics

This commit is contained in:
Agent Workbench Dev
2026-11-03 23:11:33 +08:00
parent dca6cbdb2c
commit 54dbe3d6fc
30 changed files with 683 additions and 40 deletions
+2
View File
@@ -10,6 +10,7 @@ import { modelWaitSettings } from './request-deadline.mjs';
import { sanitizeModelText } from './upstream-error.mjs';
import { nativeParameterSettings } from '../../shared/contracts/native-parameters.mjs';
import { modelIdentitySettings } from '../../shared/contracts/model-identity.mjs';
import { NetworkDiagnostics } from './network-diagnostics.mjs';
const legacyId = 'legacy-default';
const stamp = () => new Date().toISOString();
@@ -129,6 +130,7 @@ export class ModelHub {
: this.connections().find(value => value.id === selected.id)?.name;
return { ...selected, name: name ?? '原模型连接' };
}
networkDiagnostics() { return this.networkChecks ??= new NetworkDiagnostics(this); }
connection(id) { const value = this.connections().find(item => item.id === id); if (!value) throw error('模型连接不存在。', 404); return value; }
async probe(id, scope = 'text') {
if (!['text', 'tools', 'streaming', 'vision'].includes(scope)) throw error('请选择文本、工具调用、流式或图像检测。');
+81
View File
@@ -0,0 +1,81 @@
import { randomUUID } from 'node:crypto';
import { ModelTransport } from './transport.mjs';
import { probeNetwork } from './network-probe.mjs';
const problem = (message, status = 409) => Object.assign(new Error(message), { status });
const terminal = new Set(['passed', 'partial', 'failed', 'cancelled', 'interrupted']);
export class NetworkDiagnostics {
constructor(hub, options = {}) {
this.hub = hub; this.options = options; this.active = new Map();
this.jobs = structuredClone(hub.store.storage.meta('modelNetworkDiagnostics', []));
let changed = false;
for (const job of this.jobs) if (!terminal.has(job.status)) {
job.status = 'interrupted'; job.finishedAt = new Date().toISOString(); job.code = 'HOST_RESTARTED';
for (const trial of job.trials) if (['running', 'pending'].includes(trial.status)) trial.status = 'interrupted';
if (job.stages.dns?.status === 'running') job.stages.dns.status = 'interrupted';
changed = true;
}
if (changed) this.save();
}
save() { this.hub.store.storage.transaction(() => this.hub.store.storage.setMeta('modelNetworkDiagnostics', this.jobs)); }
view() {
return { jobs: this.jobs.map(job => {
const connection = this.hub.connections().find(row => row.id === job.connectionId);
return { ...structuredClone(job), stale: !connection || connection.revision !== job.connectionRevision,
currentConnectionRevision: connection?.revision ?? null };
}).reverse(), active: this.active.size, retained: 30, httpRequests: 0, modelRequests: 0 };
}
start(input) {
if (!input || Object.keys(input).length !== 2 || Object.keys(input).some(key => !['id', 'revision'].includes(key)) || !Number.isSafeInteger(input.revision)) throw problem('连接诊断需要所选连接及其原修订。', 400);
if (this.closing || this.hub.provider.requestsClosing) throw problem('执行宿主正在停止连接诊断。');
if (this.active.size || this.hub.probing || this.hub.provider.activeRequests) throw problem('请等待正在执行的模型请求或连接诊断结束。');
const connection = this.hub.connection(input.id);
if (connection.revision !== input.revision) throw problem('连接已变化,请刷新后重新选择诊断。');
const url = new URL(connection.baseUrl);
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) throw problem('连接地址无法用于网络诊断。', 400);
const provider = this.hub.provider, customTransport = !!provider.fetcher;
const transport = provider.transport ??= new ModelTransport();
const profile = structuredClone(transport.profile);
const target = { endpoint: connection.baseUrl, origin: url.origin, host: url.hostname.replace(/^\[|\]$/g, ''),
port: Number(url.port || (url.protocol === 'https:' ? 443 : 80)), scheme: url.protocol };
const budgetMs = this.options.budgetMs ?? Math.min(12000, profile.connectTimeoutMs + 2000);
const job = { schemaVersion: 1, id: randomUUID(), connectionId: connection.id, connectionRevision: connection.revision,
connectionName: connection.name, status: 'running', startedAt: new Date().toISOString(), finishedAt: null,
target, transport: profile, route: { kind: 'direct-endpoint-only', proxyConfigured: profile.httpProxyConfigured || profile.httpsProxyConfigured,
noProxyConfigured: profile.noProxyConfigured, customTransport, modelApiRouteVerified: false },
budgetMs, stages: {}, trials: [], elapsedMs: 0, httpRequests: 0, modelRequests: 0, credentialAccess: false };
const previous = this.jobs;
this.jobs = [...this.jobs.slice(-29), job];
try { this.save(); } catch (error) { this.jobs = previous; throw error; }
const controller = new AbortController(), operation = { job, controller, promise: null };
this.active.set(job.id, operation);
const timer = setTimeout(() => controller.abort({ code: 'DIAGNOSTIC_DEADLINE' }), budgetMs);
operation.promise = Promise.resolve().then(async () => {
const result = await probeNetwork(target, profile, { ...this.options,
...(transport.connect?.lookup ? { lookup: transport.connect.lookup } : {}), signal: controller.signal,
update: result => { Object.assign(job, result); this.save(); } });
Object.assign(job, result);
if (controller.signal.aborted) { job.status = controller.signal.reason?.code === 'DIAGNOSTIC_DEADLINE' ? 'failed' : 'cancelled'; job.code = controller.signal.reason?.code; }
else { const trials = job.trials.filter(row => row.status !== 'skipped'); const passed = trials.filter(row => row.status === 'passed').length;
job.status = job.stages.dns.status !== 'passed' || !passed ? 'failed' : passed === trials.length ? 'passed' : 'partial'; }
}).catch(() => { job.status = 'failed'; job.code = 'DIAGNOSTIC_INTERNAL_ERROR'; }).finally(() => {
clearTimeout(timer); job.finishedAt = new Date().toISOString();
try { this.save(); } finally { this.active.delete(job.id); }
});
operation.promise.catch(() => {});
return { job: this.view().jobs.find(row => row.id === job.id) };
}
async stop(id, input) {
if (!input || Object.keys(input).length) throw problem('停止诊断不接受额外参数。', 400);
const job = this.jobs.find(row => row.id === id); if (!job) throw problem('连接诊断记录不存在。', 404);
const operation = this.active.get(id);
operation?.controller.abort({ code: 'DIAGNOSTIC_CANCELLED' }); await operation?.promise;
return { job: this.view().jobs.find(row => row.id === id) };
}
async stopAll() {
this.closing = true;
const operations = [...this.active.values()];
for (const operation of operations) operation.controller.abort({ code: 'HOST_STOPPED' });
await Promise.all(operations.map(operation => operation.promise));
}
}
+89
View File
@@ -0,0 +1,89 @@
import dns from 'node:dns';
import net from 'node:net';
import tls from 'node:tls';
import { performance } from 'node:perf_hooks';
const elapsed = start => Math.max(0, Math.round(performance.now() - start));
const code = error => /^[A-Z0-9_]{1,80}$/.test(error?.code ?? '') ? error.code : 'NETWORK_CHECK_FAILED';
const cancelled = signal => Object.assign(new Error('连接诊断已停止。'), { code: signal.reason?.code ?? 'DIAGNOSTIC_CANCELLED' });
export async function probeNetwork(target, profile, { signal, update, lookup = dns.lookup, ca, stageTimeoutMs = profile.connectTimeoutMs } = {}) {
const started = performance.now(), stages = { dns: { status: 'running', startedAt: new Date().toISOString() } };
const trials = ['auto', 'ipv4', 'ipv6'].map(mode => ({ mode, status: 'pending', stage: 'dns', attempts: [] }));
let updateError;
const publish = () => { try { update?.(structuredClone({ stages, trials, elapsedMs: elapsed(started) })); } catch (error) { updateError ??= error; } };
const lookupStart = performance.now(); let addresses;
publish();
try {
addresses = await new Promise((resolve, reject) => {
let done = false;
const finish = (error, value) => { if (done) return; done = true; signal?.removeEventListener('abort', abort); error ? reject(error) : resolve(value); };
const abort = () => finish(cancelled(signal));
if (signal?.aborted) { abort(); return; }
signal?.addEventListener('abort', abort, { once: true });
const family = net.isIP(target.host);
if (family) { finish(null, [{ address: target.host, family }]); return; }
try { lookup(target.host, { all: true, verbatim: true }, (error, rows) => finish(error, rows)); }
catch (error) { finish(error); }
});
addresses = addresses.filter(row => [4, 6].includes(row.family) && net.isIP(row.address) === row.family).slice(0, 32);
if (!addresses.length) throw Object.assign(new Error('没有可用地址。'), { code: 'DNS_NO_ADDRESSES' });
stages.dns = { ...stages.dns, status: 'passed', elapsedMs: elapsed(lookupStart), addresses };
} catch (error) {
stages.dns = { ...stages.dns, status: signal?.aborted ? 'cancelled' : 'failed', elapsedMs: elapsed(lookupStart), code: code(error) };
for (const trial of trials) { trial.status = signal?.aborted ? 'cancelled' : 'skipped'; trial.code = 'DNS_NOT_COMPLETED'; }
publish(); return { stages, trials, elapsedMs: elapsed(started) };
}
publish();
if (updateError) throw updateError;
await Promise.all(trials.map(trial => new Promise(resolve => {
const candidates = trial.mode === 'auto' ? addresses : addresses.filter(row => row.family === (trial.mode === 'ipv4' ? 4 : 6));
if (!candidates.length) { trial.status = 'skipped'; trial.code = 'ADDRESS_FAMILY_UNAVAILABLE'; publish(); resolve(); return; }
const began = performance.now(); let tcp, secure, timer, finished = false, tcpConnected = false, tlsStarted;
const finish = (status, error) => {
if (finished) return; finished = true; clearTimeout(timer); signal?.removeEventListener('abort', abort);
if (trial.stage === 'tcp') trial.tcpMs = elapsed(began); else if (tlsStarted !== undefined) trial.tlsMs = elapsed(tlsStarted);
trial.status = status; trial.elapsedMs = elapsed(began); if (error) trial.code = code(error);
const sockets = [...new Set([secure, tcp].filter(Boolean))];
for (const socket of sockets) socket.destroy();
// A final state means owned sockets have actually closed, including cancellation.
Promise.all(sockets.map(socket => socket.closed ? null : new Promise(done => socket.once('close', done))))
.then(() => { publish(); resolve(); });
};
const abort = () => finish('cancelled', cancelled(signal));
if (signal?.aborted) { abort(); return; }
signal?.addEventListener('abort', abort, { once: true });
trial.status = 'running'; trial.stage = 'tcp'; publish();
timer = setTimeout(() => finish('failed', { code: trial.stage === 'tcp' ? 'TCP_CONNECT_TIMEOUT' : 'TLS_HANDSHAKE_TIMEOUT' }), stageTimeoutMs);
const frozenLookup = (_host, options, callback) => options.all ? callback(null, candidates)
: callback(null, candidates[0].address, candidates[0].family);
try {
tcp = net.createConnection({ host: target.host, port: target.port, lookup: frozenLookup,
autoSelectFamily: profile.autoSelectFamily, autoSelectFamilyAttemptTimeout: profile.familyAttemptTimeoutMs });
tcp.on('error', error => finish('failed', error));
tcp.on('connectionAttempt', (address, port, family) => { if (trial.attempts.length < 32) trial.attempts.push({ address, port, family }); });
tcp.on('connectionAttemptFailed', (address, _port, _family, error) => {
const attempt = trial.attempts.findLast(row => row.address === address); if (attempt) attempt.code = code(error);
});
tcp.once('connect', () => {
if (finished) return; tcpConnected = true;
trial.tcpMs = elapsed(began); trial.address = tcp.remoteAddress; trial.family = tcp.remoteFamily;
if (target.scheme === 'http:') { trial.tls = 'not_applicable'; finish('passed'); return; }
trial.stage = 'tls'; tlsStarted = performance.now(); publish();
try {
secure = tls.connect({ socket: tcp, host: target.host, servername: net.isIP(target.host) ? undefined : target.host,
rejectUnauthorized: true, ...(ca ? { ca } : {}) });
secure.on('error', error => finish('failed', error));
secure.once('secureConnect', () => {
trial.certificateAuthorized = secure.authorized === true; trial.tlsProtocol = secure.getProtocol();
trial.certificateFingerprint256 = secure.getPeerCertificate().fingerprint256;
finish(secure.authorized ? 'passed' : 'failed', secure.authorized ? null : { code: 'TLS_CERTIFICATE_REJECTED' });
});
} catch (error) { finish('failed', error); }
});
tcp.once('close', () => { if (!finished) finish('failed', { code: tcpConnected ? 'TLS_CONNECTION_CLOSED' : 'TCP_CONNECTION_CLOSED' }); });
} catch (error) { finish('failed', error); }
})));
if (updateError) throw updateError;
return { stages, trials, elapsedMs: elapsed(started) };
}