import { randomUUID, createHash } from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; import { ModelRequestError } from '../../server/diagnostics.mjs'; import { CredentialVault } from './credential-vault.mjs'; import { visionProbe } from './vision-probe.mjs'; import { modelProtocol, capabilityDeclarations, capabilityView, capabilityRejections } from '../../shared/contracts/model-capabilities.mjs'; import { actualRequest, requestVariants, requestImages } from '../../shared/contracts/model-request.mjs'; import { messagesAuthSettings, validateProtocolEndpoint, modelRequestUrl } from '../../shared/contracts/model-endpoint.mjs'; import { modelWaitSettings } from './request-deadline.mjs'; import { sanitizeModelText } from './upstream-error.mjs'; import { nativeParameterSettings } from '../../shared/contracts/native-parameters.mjs'; import { modelIdentitySettings } from '../../shared/contracts/model-identity.mjs'; import { NetworkDiagnostics } from './network-diagnostics.mjs'; const legacyId = 'legacy-default'; const stamp = () => new Date().toISOString(); const error = (message, status = 400) => Object.assign(new Error(message), { status }); const bounded = (value, fallback, min, max) => Number.isFinite(Number(value)) ? Math.max(min, Math.min(max, Math.trunc(Number(value)))) : fallback; const label = (value, maximum = 100) => String(value ?? '').trim().slice(0, maximum); export function modelEndpoint(value) { let url; try { url = new URL(String(value)); } catch { throw error('请输入有效的模型接口根地址。'); } if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash || url.toString().length > 1000) throw error('模型地址只支持不含账号、查询参数的 http/https 地址。'); return url.toString().replace(/\/+$/, ''); } export class ModelHub { constructor(store, provider) { this.store = store; this.provider = provider; this.state = store.storage.meta('modelRegistry', { schema: 1, revision: 0, selected: { type: 'legacy', id: legacyId }, connections: [], pools: [], imports: {} }); this.health = new Map(); this.leases = new Map(); this.cursors = new Map(); this.sticky = new Map(); this.waiting = 0; this.attempts = store.storage.meta('modelAttempts', []); // A process exit leaves a request outcome unknown; never replay it here. let recovered = false; for (const attempt of this.attempts) if (attempt.status === 'running') { attempt.status = 'unknown'; attempt.finishedAt = stamp(); attempt.reason = '宿主重启,请求结果与用量待核对。'; recovered = true; } if (recovered) this.saveAttempts(); provider.credentials ??= new Map(); if (store.storage.meta('modelCredentials', null)) this.vault().restore(this); } save(config = null) { this.state.revision++; if (this.editingRegistry) { if (config) throw error('批量连接导入不能修改当前运行模式。'); return; } const persist = () => this.store.storage.setMeta('modelRegistry', this.state); try { if (config) this.store.saveConfig(config, { beforeSave: persist }); else this.store.storage.transaction(persist); } catch (cause) { // Keep runtime routing consistent with the authoritative committed state. this.state = this.store.storage.meta('modelRegistry', { schema: 1, revision: 0, selected: { type: 'legacy', id: legacyId }, connections: [], pools: [], imports: {} }); throw cause; } } editRegistry(action) { if (this.editingRegistry) return action(); const before = structuredClone(this.state), credentials = new Map(this.provider.credentials); const originalKey = this.provider.apiKey, legacyRevision = this.provider.legacyCredentialRevision; const vaultIssues = this.credentialVault ? new Map(this.credentialVault.issues) : null; this.editingRegistry = true; try { return this.store.storage.transaction(() => { const result = action(); this.store.storage.setMeta('modelRegistry', this.state); return result; }); } catch (cause) { this.state = before; this.provider.credentials.clear(); for (const [id, value] of credentials) this.provider.credentials.set(id, value); this.provider.apiKey = originalKey; this.provider.legacyCredentialRevision = legacyRevision; this.credentialVault?.reload(); if (vaultIssues) this.credentialVault.issues = vaultIssues; throw cause; } finally { this.editingRegistry = false; } } saveAttempts() { this.attempts = this.attempts.slice(-1000); this.store.storage.transaction(() => this.store.storage.setMeta('modelAttempts', this.attempts)); } connections() { return [{ id: legacyId, name: '原模型连接', brand: '兼容服务', protocol: 'chat-completions', baseUrl: this.store.config.baseUrl, model: this.store.config.model, enabled: true, requiresKey: false, revision: this.store.config.configRevision ?? 1, contextLimit: this.store.config.contextLimit, maxOutputTokens: this.store.config.maxOutputTokens, concurrency: 1, source: 'legacy', ...(this.store.config.nativeParameters ? { nativeParameters: this.store.config.nativeParameters, outputTokenField: this.store.config.outputTokenField } : {}) }, ...this.state.connections]; } credential(connection) { return connection.id === legacyId ? { key: this.provider.apiKey, revision: this.provider.legacyCredentialRevision ?? 1 } : this.provider.credentials.get(connection.id); } vault() { return this.credentialVault ??= new CredentialVault(this.store, this.provider); } manageCredential(input) { const connection = this.connection(input.id); if (input.revision !== connection.revision) throw error('模型连接已修改,请刷新凭据保存页面。', 409); return this.editRegistry(() => { const vault = this.vault(), previous = vault.state.entries[connection.id]; if (input.action === 'save') { const revision = (this.credential(connection)?.revision ?? 0) + 1, record = vault.prepare(connection, input.apiKey, revision); if (connection.id === legacyId) { this.provider.setKey(input.apiKey); this.provider.legacyCredentialRevision = revision; } else this.provider.credentials.set(connection.id, { key: input.apiKey.trim(), revision }); vault.stage(connection.id, record); } else if (input.action === 'delete' || input.action === 'forget') { vault.stage(connection.id, null); if (input.action === 'forget') { if (connection.id === legacyId) { this.provider.setKey(''); this.provider.legacyCredentialRevision = (this.provider.legacyCredentialRevision ?? 1) + 1; } else this.provider.credentials.set(connection.id, { key: '', revision: (this.credential(connection)?.revision ?? 0) + 1 }); } } else if (input.action === 'disable' || input.action === 'enable') { if (!previous) throw error('该连接没有本机保存块。'); vault.stage(connection.id, { ...previous, enabled: input.action === 'enable' }); if (input.action === 'enable') vault.restore(this); } else throw error('凭据保存操作无效。'); this.save(); return { credential: vault.status(connection), center: this.view(), config: { ...this.store.config, hasKey: this.provider.hasKey() } }; }); } identity(connection) { return connection.id + ':' + connection.revision + ':' + (this.credential(connection)?.revision ?? 0); } status(connection) { const credential = this.credential(connection), health = this.health.get(this.identity(connection)); const ready = connection.enabled && (!connection.requiresKey || !!credential?.key); return { ...connection, hasKey: !!credential?.key, credentialRevision: credential?.revision ?? 0, credentialStorage: 'memory', ...(this.credentialVault ? { savedCredential: this.credentialVault.status(connection) } : {}), status: !connection.enabled ? 'disabled' : !ready ? 'needs_key' : health?.blocked ? 'needs_attention' : health?.cooldownUntil > Date.now() ? 'cooling' : 'configured', activeRequests: this.leases.get(connection.id) ?? 0, health: health ?? { state: 'unverified' }, streaming: connection.streaming === true, capabilitySchema: 1, capabilityDeclarations: capabilityDeclarations(undefined, connection.capabilityDeclarations), ...(this.state.capabilityResults?.[connection.id] ? { capabilityEvidence: this.state.capabilityResults[connection.id] } : {}), capabilities: capabilityView({ ...connection, capabilityEvidence: this.state.capabilityResults?.[connection.id] }, credential?.revision ?? 0) }; } view() { return { revision: this.state.revision, selected: this.state.selected, waitSettings: modelWaitSettings(this.store.config), connections: this.connections().map(value => this.status(value)), pools: this.state.pools, imports: Object.entries(this.state.imports).map(([id, value]) => ({ id, ...value })), attempts: this.attempts.slice(-100).reverse(), waiting: this.waiting, storage: { credentials: this.credentialVault ? 'memory-or-windows-user-dpapi' : 'memory', restart: '临时凭据在宿主退出后需要重新填写;已明确本机加密保存的连接按用户与原配置加载。', attemptsRetained: 1000 } }; } selectionView() { const selected = this.state.selected; const name = selected.type === 'pool' ? this.state.pools.find(value => value.id === selected.id)?.name : this.connections().find(value => value.id === selected.id)?.name; return { ...selected, name: name ?? '原模型连接' }; } networkDiagnostics() { return this.networkChecks ??= new NetworkDiagnostics(this); } connection(id) { const value = this.connections().find(item => item.id === id); if (!value) throw error('模型连接不存在。', 404); return value; } async probe(id, scope = 'text') { if (!['text', 'tools', 'streaming', 'vision'].includes(scope)) throw error('请选择文本、工具调用、流式或图像检测。'); if (this.probing) throw error('已有模型能力检测正在执行,请等待结束。', 409); this.probing = true; try { const connection = { ...this.connection(id) }, member = { ...connection, credentialRevision: this.credential(connection)?.revision ?? 0 }; if (!this.available(member)) throw error('该连接不可用,请核对凭据、冷却或停用状态。'); member.streaming = scope === 'streaming'; const nonce = randomUUID(), tools = scope === 'tools' ? [{ type: 'function', function: { name: 'workbench_capability_echo', description: 'Return the provided nonce in this capability test. No files or commands are available.', parameters: { type: 'object', properties: { nonce: { type: 'string' } }, required: ['nonce'], additionalProperties: false } } }] : []; const config = { ...this.store.config, mode: 'compatible', model: connection.model, baseUrl: connection.baseUrl, protocol: modelProtocol(connection.protocol), maxOutputTokens: 256, capabilityProbe: true, modelRouting: { type: 'connection', id: connection.id, revision: connection.revision, strategy: 'round-robin', sticky: false, maxAttempts: 1, queueTimeoutMs: 1000, members: [member] } }; const visual=scope==='vision'?visionProbe():null; if(visual)config.inputImages=[visual.image]; let attemptId; const result = await this.provider.complete(config, [{ role: 'user', content: visual?.prompt ?? (scope === 'tools' ? 'Call workbench_capability_echo with nonce ' + nonce + '. Do not reply with plain text.' : 'Reply OK.') }], null, undefined, event => { if (event.modelAttemptId) attemptId = event.modelAttemptId; }, null, tools, { attempts: 1 }); if (scope === 'tools') { const calls = result.message.tool_calls; if (calls?.length !== 1 || calls[0].function.name !== 'workbench_capability_echo' || JSON.parse(calls[0].function.arguments).nonce !== nonce) throw error('服务没有返回要求的原工具调用;未标为已验证。'); } else if (!result.message.content.trim() || result.message.tool_calls?.length) throw error('服务没有返回纯文本响应;未标为已验证。'); if(visual&&result.message.content.trim().toUpperCase().replace(/\s/g,'')!==visual.expected)throw error('服务未正确识别合成图像的四色顺序,原请求保留,未标为视觉已验证。'); if (this.connection(id).revision !== connection.revision || (this.credential(connection)?.revision ?? 0) !== member.credentialRevision) throw error('检测期间连接已变化,原结果保留但不更新能力结论。', 409); this.editRegistry(() => { this.state.capabilityResults ??= {}; const previous = this.state.capabilityResults[id]; const results = previous?.connectionRevision === connection.revision && previous.credentialRevision === member.credentialRevision ? { ...previous.results } : {}; results[scope] = 'verified'; const attempt = this.attempts.find(value => value.id === attemptId); if (!attempt || attempt.connectionId !== id || attempt.status !== 'completed') throw error('检测原请求账本不完整,未更新能力结论。'); this.state.capabilityResults[id] = { schema: 1, connectionRevision: connection.revision, credentialRevision: member.credentialRevision, results, checkedAt: stamp(), protocol: modelProtocol(connection.protocol), attemptId: attempt?.id, source: 'explicit-model-probe', scope: 'Only the requested synthetic capability; no project files, commands or real-model engineering benefit.' }; this.save(); }); return { ok: true, usage: result.usage, scope, center: this.view() }; } finally { this.probing = false; } } upsert(input) { if (!this.editingRegistry) return this.editRegistry(() => this.upsert(input)); if (input.id === legacyId) throw error('原连接请通过模型连接设置修改。'); const previous = input.id ? this.connection(input.id) : null; if (this.state.connections.length >= 100 && !previous) throw error('最多保存100个模型连接。'); if (previous && input.revision !== previous.revision) throw error('模型连接已修改,请刷新后重试。', 409); const protocol = modelProtocol(input.protocol ?? previous?.protocol ?? 'chat-completions'); const native = Object.hasOwn(input, 'nativeParameters') || Object.hasOwn(input, 'outputTokenField') || previous?.nativeParameters ? nativeParameterSettings(protocol, input.nativeParameters ?? previous?.nativeParameters ?? {}, input.outputTokenField ?? previous?.outputTokenField) : null; const declarations = capabilityDeclarations(input.capabilityDeclarations, previous?.capabilityDeclarations); if (input.streaming !== undefined && typeof input.streaming !== 'boolean') throw error('流式选项必须为开启或关闭。'); const streaming = input.streaming ?? previous?.streaming ?? false; if (streaming && declarations.streaming !== 'supported') throw error('请先声明服务支持流式输出,再开启流式请求。'); const name = label(input.name), model = label(input.model, 200); if (!name || !model) throw error('请填写连接名称和实际模型名称。'); const connection = { id: previous?.id ?? randomUUID(), name, brand: label(input.brand || '自定义', 40), protocol, model, capabilitySchema: 1, capabilityDeclarations: declarations, streaming, ...(native ?? {}), ...modelIdentitySettings(input, previous ?? {}), ...messagesAuthSettings(protocol, input.messagesAuth ?? previous?.messagesAuth), baseUrl: validateProtocolEndpoint(protocol, modelEndpoint(input.baseUrl)), enabled: input.enabled !== false, requiresKey: input.requiresKey !== false, contextLimit: bounded(input.contextLimit, 16000, 1000, 200000), maxOutputTokens: bounded(input.maxOutputTokens, 2048, 256, 16000), concurrency: bounded(input.concurrency, 1, 1, 8), revision: (previous?.revision ?? 0) + 1, source: previous?.source ?? 'manual', ...(previous?.importOrigin ? { importOrigin: previous.importOrigin, importRevision: previous.importRevision } : {}), createdAt: previous?.createdAt ?? stamp(), updatedAt: stamp() }; if (Object.hasOwn(input, 'apiKey')) { if (typeof input.apiKey !== 'string' || input.apiKey.length > 16000) throw error('模型凭据格式无效。'); } this.state.connections = [...this.state.connections.filter(item => item.id !== connection.id), connection]; this.save(); if (Object.hasOwn(input, 'apiKey')) this.provider.credentials.set(connection.id, { key: input.apiKey.trim(), revision: (this.provider.credentials.get(connection.id)?.revision ?? 0) + 1 }); if (input.persistKey === true) { const vault = this.vault(), record = vault.prepare(connection, input.apiKey, this.credential(connection)?.revision ?? 1); vault.stage(connection.id, record); } return this.status(connection); } setNativeParameters(input) { const connection = this.connection(input.id); if (connection.revision !== input.revision) throw error('连接已变化,请刷新后核对原生参数。', 409); const settings = nativeParameterSettings(connection.protocol, input.nativeParameters, input.outputTokenField); if (connection.id === legacyId) this.save(settings); else this.upsert({ ...connection, ...settings }); return { center: this.view(), config: { ...this.store.config, hasKey: this.provider.hasKey() } }; } toggle(id, enabled) { if (id === legacyId) throw error('原连接不能停用,请选择离线演示或其他连接。'); const value = this.connection(id); value.enabled = enabled === true; value.revision++; value.updatedAt = stamp(); this.save(); return this.status(value); } pool(input) { const previous = input.id ? this.state.pools.find(value => value.id === input.id) : null; if (input.id && !previous) throw error('模型池不存在。', 404); if (previous && input.revision !== previous.revision) throw error('模型池已修改,请刷新后重试。', 409); if (!previous && this.state.pools.length >= 32) throw error('最多保存32个模型池。'); if (!label(input.name) || !Array.isArray(input.members) || !input.members.length || input.members.length > 32) throw error('模型池需要名称和1–32个成员。'); const seen = new Set(); const members = input.members.map(value => { this.connection(value.connectionId); if (seen.has(value.connectionId)) throw error('同一连接不能重复加入模型池。'); seen.add(value.connectionId); return { connectionId: value.connectionId, weight: bounded(value.weight, 1, 1, 10) }; }); if (!['round-robin', 'weighted'].includes(input.strategy || 'round-robin')) throw error('请选择轮询或权重策略。'); const value = { id: previous?.id ?? randomUUID(), name: label(input.name), members, strategy: input.strategy || 'round-robin', sticky: input.sticky !== false, maxAttempts: bounded(input.maxAttempts, 3, 1, 8), queueTimeoutMs: bounded(input.queueTimeoutMs, 15000, 100, 60000), revision: (previous?.revision ?? 0) + 1 }; this.state.pools = [...this.state.pools.filter(item => item.id !== value.id), value]; this.save(); return value; } select(input, config = null) { if (input.type === 'connection') this.connection(input.id); else if (input.type === 'pool') { if (!this.state.pools.some(item => item.id === input.id)) throw error('模型池不存在。'); } else if (input.type !== 'legacy') throw error('模型选择无效。'); this.state.selected = { type: input.type, id: input.type === 'legacy' ? legacyId : input.id }; // Selection and mode must commit together; the original endpoint/key pair // stays intact and a Run alone receives its immutable routing snapshot. this.save(config ?? (input.type !== 'legacy' ? { mode: 'compatible' } : null)); return this.view(); } prepare(config, selection = this.state.selected) { if (config.mode === 'demo' || selection.type === 'legacy') return config; const pool = selection.type === 'pool' ? this.state.pools.find(value => value.id === selection.id) : null; if (selection.type === 'pool' && !pool) throw error('选中的模型池不存在。'); const members = (pool?.members ?? [{ connectionId: selection.id, weight: 1 }]).map(value => ({ ...this.connection(value.connectionId), capabilityEvidence: this.state.capabilityResults?.[value.connectionId], weight: value.weight, credentialRevision: this.credential(this.connection(value.connectionId))?.revision ?? 0 })); if (!members.some(value => this.status(value).status === 'configured' || this.status(value).status === 'cooling')) throw error('没有可用模型连接,请核对启用状态、临时凭据和故障。'); return { ...config, ...messagesAuthSettings(members[0].protocol, members[0].messagesAuth), baseUrl: members[0].baseUrl, model: members[0].model, protocol: modelProtocol(members[0].protocol), streaming: members[0].streaming === true, nativeParameters: members[0].nativeParameters ?? {}, outputTokenField: members[0].outputTokenField, contextLimit: Math.min(config.contextLimit, ...members.map(value => value.contextLimit)), maxOutputTokens: Math.min(config.maxOutputTokens, ...members.map(value => value.maxOutputTokens)), modelRouting: { type: selection.type, id: selection.id, revision: pool?.revision ?? members[0].revision, strategy: pool?.strategy ?? 'round-robin', sticky: pool?.sticky ?? true, maxAttempts: pool?.maxAttempts ?? 1, queueTimeoutMs: pool?.queueTimeoutMs ?? 15000, members } }; } available(member) { const current = this.connections().find(value => value.id === member.id); if (!current?.enabled || current.revision !== member.revision || (this.credential(current)?.revision ?? 0) !== member.credentialRevision) return false; const health = this.health.get(this.identity(member)); return (!member.requiresKey || !!this.credential(member)?.key) && !health?.blocked && !(health?.cooldownUntil > Date.now()); } async acquire(routing, runId, excluded, signal, deadline, requirements = []) { if (this.waiting >= 100) throw error('模型请求队列已满,请稍后继续任务。'); this.waiting++; try { while (true) { signal?.throwIfAborted(); const eligible = routing.members.filter(value => capabilityRejections(value, requirements, value.credentialRevision).length === 0); if (!eligible.length) throw Object.assign(error('模型池没有满足本次任务能力的成员;请核对工具调用和流式能力声明。'), { rejections: routing.members.flatMap(value => capabilityRejections(value, requirements, value.credentialRevision)) }); const healthy = eligible.filter(value => this.available(value) && !excluded.has(value.id)); const ready = healthy.filter(value => (this.leases.get(value.id) ?? 0) < value.concurrency); if (ready.length) { const remembered = routing.sticky && this.sticky.get(runId); let selected = ready.find(value => value.id === remembered); if (!selected) { const weighted = ready.flatMap(value => Array(routing.strategy === 'weighted' ? value.weight : 1).fill(value)); const cursor = this.cursors.get(routing.id) ?? 0; selected = weighted[cursor % weighted.length]; this.cursors.set(routing.id, cursor + 1); } this.leases.set(selected.id, (this.leases.get(selected.id) ?? 0) + 1); this.sticky.set(runId, selected.id); if (this.sticky.size > 1000) this.sticky.delete(this.sticky.keys().next().value); let released = false; return { member: selected, release: () => { if (!released) { released = true; this.leases.set(selected.id, Math.max(0, (this.leases.get(selected.id) ?? 0) - 1)); } } }; } if (!healthy.length) throw error('模型池暂时没有可用成员;已完成操作保留,请核对凭据、停用状态和冷却时间。'); if (Date.now() >= deadline) throw error('等待模型并发槽位超时,任务已保留。'); await delay(50, undefined, signal ? { signal } : {}); } } finally { this.waiting--; } } async complete(config, body, signal, onProgress, plan, execute,beforeDispatch) { const operation = this.provider.requestOperation?.(signal); try { return await this.completeOwned(config, body, operation?.signal ?? signal, onProgress, plan, execute,beforeDispatch); } finally { operation?.finish(); } } async completeOwned(config, body, signal, onProgress, plan, execute,beforeDispatch) { const routing = config.modelRouting, excluded = new Set(), deadline = Date.now() + Math.min(routing.queueTimeoutMs, 60000); const requestId = plan?.id ?? randomUUID(), runId = plan?.runId ?? requestId; const requirements = config.capabilityProbe ? [] : ['text', ...(requestVariants(body).some(value => value.tools?.length) ? ['tools'] : []), ...(requestImages(body).length ? ['vision'] : [])]; const rejections = config.capabilityProbe ? [] : routing.members.flatMap(value => capabilityRejections(value, [...requirements, ...(value.streaming ? ['streaming'] : [])], value.credentialRevision)); const eligibleRouting = { ...routing, members: routing.members.filter(value => !rejections.some(item => item.connectionId === value.id)) }; if (!eligibleRouting.members.length) { const attempt = { id: randomUUID(), requestId, taskId: plan?.taskId ?? null, runId, sessionId: plan?.sessionId ?? null, projectId: plan?.projectId ?? null, poolId: routing.type === 'pool' ? routing.id : null, status: 'rejected', reason: '所有成员均不满足本次任务所需能力。', rejections, startedAt: stamp(), finishedAt: stamp(), usage: null, usageStatus: 'not_dispatched' }; this.attempts.push(attempt); this.saveAttempts(); throw error('模型池没有满足本次任务能力的成员;请核对能力声明或选择其它连接。'); } let lastError; for (let count = 1; count <= routing.maxAttempts; count++) { signal?.throwIfAborted(); beforeDispatch?.({previousFailure:lastError?.diagnostic}); let lease; try { lease = await this.acquire(eligibleRouting, runId, excluded, signal, deadline, requirements); } catch (cause) { throw lastError ?? cause; } const member = lease.member, credential = this.credential(member); const actualBody = actualRequest(body, member); const bytes = JSON.stringify(actualBody); const attempt = { id: randomUUID(), requestId, taskId: plan?.taskId ?? null, runId, sessionId: plan?.sessionId ?? null, projectId: plan?.projectId ?? null, connectionId: member.id, connectionRevision: member.revision, credentialRevision: member.credentialRevision, poolId: routing.type === 'pool' ? routing.id : null, poolRevision: routing.revision, model: member.model, protocol: modelProtocol(member.protocol), streaming: member.streaming === true, endpoint: member.baseUrl, requestUrl: modelRequestUrl(member), ...(member.protocol === 'anthropic-messages' ? messagesAuthSettings(member.protocol, member.messagesAuth) : {}), ordinal: count, status: 'running', startedAt: stamp(), requestBytes: Buffer.byteLength(bytes, 'utf8'), requirements, rejections, requestSha256: createHash('sha256').update(bytes).digest('hex'), usage: null, usageStatus: 'unknown' }; try { if (plan?.sessionId) { const session = this.store.session(plan.sessionId), run = this.store.owned('run', plan.runId, session.id); const artifact = this.store.artifactStore.write(session, run, JSON.parse(this.provider.redact(bytes)), { kind: 'model-attempt-request' }); attempt.requestArtifactId = artifact.id; this.store.core.artifact.set(artifact.id, artifact); this.store.save(session); } this.attempts.push(attempt); this.saveAttempts(); const progress = event => { if (event.modelIdentity) attempt.modelIdentity = event.modelIdentity; if (event.wireAttemptId) attempt.wireAttemptId = event.wireAttemptId; if (event.upstreamResponse) attempt.upstreamResponse = event.upstreamResponse; if (event.consumption) { if (event.consumption.modelIdentity) attempt.modelIdentity = event.consumption.modelIdentity; attempt.usage = event.consumption.usage; attempt.usageStatus = event.consumption.usageStatus; onProgress?.({ consumption: { ...event.consumption, wireAttemptId: event.consumption.attemptId, attemptId: attempt.id, connectionId: member.id, connectionName: member.name }, connectionId: member.id, model: member.model, modelAttemptId: attempt.id }); this.saveAttempts(); return; } if (event.streamCapture !== undefined || event.responseCapture !== undefined) { if (event.stream) attempt.stream = event.stream; if (plan?.sessionId) { const session = this.store.session(plan.sessionId), run = this.store.owned('run', plan.runId, session.id); const raw = event.streamCapture ?? event.responseCapture, decoded = raw.toString('utf8'), redacted = sanitizeModelText(decoded, value => this.provider.redact(value)); const captured = redacted === decoded ? raw : Buffer.from(redacted, 'utf8'); const artifact = this.store.artifactStore.write(session, run, captured, { kind: event.streamCapture !== undefined ? 'model-stream-response' : 'model-http-response', mediaType: event.streamCapture !== undefined ? 'text/event-stream' : 'text/plain' }); this.store.core.artifact.set(artifact.id, artifact); attempt.responseArtifactId = artifact.id; if (attempt.modelIdentity) attempt.modelIdentity.responseSource = { artifactId: artifact.id, sha256: artifact.sha256, sessionId: session.id, scope: event.streamCapture !== undefined ? 'sse-stream' : 'http-body' }; attempt.responseSha256 = artifact.sha256; attempt.responseRedacted = redacted !== decoded || event.upstream?.capture?.redacted === true; this.store.save(session); if (event.upstream) Object.assign(event.upstream, { artifactId: artifact.id, artifactSha256: artifact.sha256, artifactScope: event.streamCapture !== undefined ? 'sse-stream' : 'http-body', sessionId: session.id }); if (event.upstream?.capture) Object.assign(event.upstream.capture, { retention: 'artifact', storedBytes: artifact.bytes }); } if (event.upstream) attempt.upstream = event.upstream; return; } if (event.streaming) attempt.partial = { sequence: event.streamSequence, bytes: event.streamBytes, textCharacters: event.textCharacters, textPreview: this.provider.redact(event.textPreview ?? '').slice(0, 8000), toolFragments: event.toolFragments }; if (event.stream) attempt.stream = event.stream; onProgress?.({ ...event, attempt: count, maxAttempts: routing.maxAttempts, connectionId: member.id, connectionName: member.name, protocol: modelProtocol(member.protocol), model: member.model, modelAttemptId: attempt.id }); }; const result = await execute({ ...config, ...modelIdentitySettings(member), ...messagesAuthSettings(member.protocol, member.messagesAuth), baseUrl: member.baseUrl, model: member.model, protocol: modelProtocol(member.protocol), streaming: member.streaming === true }, credential?.key ?? '', actualBody, progress); attempt.status = 'completed'; attempt.usage = result.usage ?? attempt.usage; attempt.usageStatus = attempt.usage ? 'reported' : 'unknown'; if (result.compatibility?.length) attempt.compatibility = result.compatibility; this.health.set(this.identity(member), { state: 'responded', lastSuccessAt: stamp(), cooldownUntil: 0, failures: 0 }); return result; } catch (cause) { const diagnostic = cause.diagnostic; if (diagnostic) attempt.diagnostic = diagnostic; attempt.status = signal?.aborted ? 'interrupted' : diagnostic?.outcome === 'unknown' ? 'unknown' : 'failed'; attempt.reason = this.provider.redact(cause.message).slice(0, 600); attempt.httpStatus = diagnostic?.status ?? diagnostic?.httpStatus ?? null; const health = this.health.get(this.identity(member)) ?? {}; if (signal?.aborted) { // A user stop is not evidence that the provider is unhealthy. Keep // the attempt, release its lease, and leave the connection eligible. this.health.set(this.identity(member), { ...health, state: 'interrupted', lastInterruptedAt: stamp() }); } else { const blocked = [401, 403].includes(attempt.httpStatus) || ['quota', 'model_identity'].includes(diagnostic?.kind); const failedAt = Date.now(); this.health.set(this.identity(member), { state: blocked ? 'needs_attention' : 'failed', blocked, lastFailureAt: new Date(failedAt).toISOString(), failures: (health.failures ?? 0) + 1, cooldownUntil: failedAt + (diagnostic?.retryAfterMs ?? Math.min(60000, 1000 * 2 ** Math.min(health.failures ?? 0, 5))), reason: attempt.reason }); } lastError = cause; excluded.add(member.id); this.sticky.delete(runId); signal?.throwIfAborted(); if ((!diagnostic?.retryable && !diagnostic?.failoverAllowed) || count === routing.maxAttempts) throw cause; onProgress?.({ phase: 'retrying', attempt: count, maxAttempts: routing.maxAttempts, connectionId: member.id, modelAttemptId: attempt.id, diagnostic, waitMs: 0, switching: true }); } finally { attempt.finishedAt = stamp(); lease.release(); this.saveAttempts(); } } throw lastError ?? new ModelRequestError({ message: '模型池请求未完成。', kind: 'connection' }); } }