feat(models): preserve complete incremental request ledger and exports

This commit is contained in:
Agent Workbench Dev
2026-11-07 12:38:44 +08:00
parent a93bfd6885
commit da2e631381
16 changed files with 888 additions and 28 deletions
+34 -14
View File
@@ -16,6 +16,7 @@ import { modelIdentitySettings } from '../../shared/contracts/model-identity.mjs
import { NetworkDiagnostics } from './network-diagnostics.mjs';
import { ModelCatalog, credentialScope } from './model-catalog.mjs';
import { capacitySettings, routedCapacity } from '../../shared/contracts/model-capacity.mjs';
import { ModelRequestLedger } from './request-ledger.mjs';
const legacyId = 'legacy-default';
const stamp = () => new Date().toISOString();
@@ -34,11 +35,9 @@ export class ModelHub {
this.store = store; this.provider = provider;
this.state = store.storage.meta('modelRegistry', { schema: 1, revision: 0, selected: { type: 'legacy', id: legacyId }, connections: [], pools: [], imports: {} });
this.health = new Map(); this.leases = new Map(); this.cursors = new Map(); this.sticky = new Map(); this.waiting = 0;
this.attempts = store.storage.meta('modelAttempts', []);
this.ledger = new ModelRequestLedger(store.storage);
// A process exit leaves a request outcome unknown; never replay it here.
let recovered = false;
for (const attempt of this.attempts) if (attempt.status === 'running') { attempt.status = 'unknown'; attempt.finishedAt = stamp(); attempt.reason = '宿主重启,请求结果与用量待核对。'; recovered = true; }
if (recovered) this.saveAttempts();
this.ledger.recoverRunning();
provider.credentials ??= new Map();
if (store.storage.meta('modelCredentials', null)) this.vault().restore(this);
}
@@ -86,7 +85,27 @@ export class ModelHub {
throw cause;
} finally { this.editingRegistry = false; }
}
saveAttempts() { if (!unrestricted(this.store)) this.attempts = this.attempts.slice(-1000); this.store.storage.transaction(() => this.store.storage.setMeta('modelAttempts', this.attempts)); }
get attempts() { return this.compatAttempts ??= this.ledger.all(); }
set attempts(value) { this.compatAttempts = value; }
adoptRequestLedger() {
if (this.ledger) { Object.setPrototypeOf(this.ledger, ModelRequestLedger.prototype); return this.ledger; }
const retained = Object.getOwnPropertyDescriptor(this, 'attempts');
if (retained && (!Array.isArray(retained.value) || JSON.stringify(retained.value) !== JSON.stringify(this.store.storage.meta('modelAttempts', [])))) throw error('原内存请求账本与保存来源不同,请先核对,未重放任何请求。', 409);
const ledger = new ModelRequestLedger(this.store.storage);
if (retained) { delete this.attempts; this.compatAttempts = retained.value; }
this.ledger = ledger; return ledger;
}
saveAttempts(attempt = null) {
if (!attempt) return this.ledger.saveAll(this.attempts);
const sequence = this.ledger.save(attempt);
if (this.compatAttempts) {
const positions = this.compatAttempts.map((value, index) => ({ value, index })).filter(({ value }) => value === attempt || this.ledger.positions.get(value) === sequence);
const sameId = positions.length ? positions : this.compatAttempts.map((value, index) => ({ value, index })).filter(({ value }) => value.id === attempt.id);
if (sameId.length === 1) this.compatAttempts[sameId[0].index] = attempt;
}
return sequence;
}
addAttempt(attempt) { this.ledger.save(attempt); this.compatAttempts?.push(attempt); }
connections() {
const saved = this.state.legacyConnection;
return [{ ...saved, id: legacyId, name: saved?.name || this.store.config.model || '未配置模型', brand: saved?.brand || '自定义', protocol: saved?.protocol || 'chat-completions', baseUrl: this.store.config.baseUrl,
@@ -144,9 +163,10 @@ export class ModelHub {
...(this.state.capabilityResults?.[connection.id] ? { capabilityEvidence: this.state.capabilityResults[connection.id] } : {}),
capabilities: capabilityView({ ...connection, capabilityEvidence: this.state.capabilityResults?.[connection.id] }, credential?.revision ?? 0) };
}
view() { return { revision: this.state.revision, selected: this.state.selected, waitSettings: modelWaitSettings(this.store.config), connections: this.connections().map(value => this.status(value)), pools: this.state.pools,
imports: Object.entries(this.state.imports).map(([id, value]) => ({ id, ...value })), attempts: this.attempts.slice(-100).reverse(), waiting: this.waiting,
storage: { credentials: this.credentialVault ? 'memory-or-windows-user-dpapi' : 'memory', restart: '临时凭据在宿主退出后需要重新填写;已明确本机加密保存的连接按用户与原配置加载。', attemptsRetained: 1000 } }; }
view() { const page = this.ledger.page({ limit: 100 }); return { revision: this.state.revision, selected: this.state.selected, waitSettings: modelWaitSettings(this.store.config), connections: this.connections().map(value => this.status(value)), pools: this.state.pools,
imports: Object.entries(this.state.imports).map(([id, value]) => ({ id, ...value })), attempts: page.items, attemptCount: page.retainedTotal, attemptsHasMore: page.hasMore, waiting: this.waiting,
storage: { credentials: this.credentialVault ? 'memory-or-windows-user-dpapi' : 'memory', restart: '临时凭据在宿主退出后需要重新填写;已明确本机加密保存的连接按用户与原配置加载。', attemptsRetained: null,
attemptStorage: 'incremental-sqlite', attemptPreview: 100, originalMetadataPreserved: true } }; }
selectionView() {
const selected = this.state.selected;
const name = selected.type === 'pool' ? this.state.pools.find(value => value.id === selected.id)?.name
@@ -192,7 +212,7 @@ export class ModelHub {
const previous = this.state.capabilityResults[id];
const results = previous?.connectionRevision === connection.revision && previous.credentialRevision === member.credentialRevision ? { ...previous.results } : {};
results[scope] = 'verified';
const attempt = this.attempts.find(value => value.id === attemptId);
const attempt = this.ledger.find(attemptId);
if (!attempt || attempt.connectionId !== id || attempt.status !== 'completed') throw error('检测原请求账本不完整,未更新能力结论。');
this.state.capabilityResults[id] = { schema: 1, connectionRevision: connection.revision, credentialRevision: member.credentialRevision, results,
checkedAt: stamp(), protocol: modelProtocol(connection.protocol), attemptId: attempt?.id, source: 'explicit-model-probe',
@@ -364,9 +384,9 @@ export class ModelHub {
try { lease = await this.acquire(routing, runId, excluded, signal, deadline, requirements, { probe: config.capabilityProbe === true }); }
catch (cause) {
if (!lastError && cause.routingDecision) {
this.attempts.push({ id: randomUUID(), requestId, taskId: plan?.taskId ?? null, runId, sessionId: plan?.sessionId ?? null, projectId: plan?.projectId ?? null,
this.addAttempt({ id: randomUUID(), requestId, taskId: plan?.taskId ?? null, runId, sessionId: plan?.sessionId ?? null, projectId: plan?.projectId ?? null,
poolId: routing.type === 'pool' ? routing.id : null, status: 'rejected', reason: cause.message, routingDecision: cause.routingDecision,
startedAt: stamp(), finishedAt: stamp(), usage: null, usageStatus: 'not_dispatched' }); this.saveAttempts();
startedAt: stamp(), finishedAt: stamp(), usage: null, usageStatus: 'not_dispatched' });
}
throw lastError ?? cause;
}
@@ -388,7 +408,7 @@ export class ModelHub {
this.store.core.artifact.set(artifact.id, artifact);
this.store.save(session);
}
this.attempts.push(attempt); this.saveAttempts();
this.addAttempt(attempt);
const progress = event => {
if (event.modelIdentity) attempt.modelIdentity = event.modelIdentity;
if (event.wireAttemptId) attempt.wireAttemptId = event.wireAttemptId;
@@ -398,7 +418,7 @@ export class ModelHub {
attempt.usage = event.consumption.usage; attempt.usageStatus = event.consumption.usageStatus;
onProgress?.({ consumption: { ...event.consumption, wireAttemptId: event.consumption.attemptId, attemptId: attempt.id,
connectionId: member.id, connectionName: member.name }, connectionId: member.id, model: member.model, modelAttemptId: attempt.id });
this.saveAttempts();
this.saveAttempts(attempt);
return;
}
if (event.streamCapture !== undefined || event.responseCapture !== undefined) {
@@ -456,7 +476,7 @@ export class ModelHub {
if (config.capacityMode === 'passthrough' || (!diagnostic?.retryable && !diagnostic?.failoverAllowed) || count === routing.maxAttempts) throw cause;
onProgress?.({ phase: 'retrying', attempt: count, maxAttempts: routing.maxAttempts, connectionId: member.id, modelAttemptId: attempt.id,
diagnostic, waitMs: 0, switching: true });
} finally { attempt.finishedAt = stamp(); lease.release(); this.saveAttempts(); }
} finally { attempt.finishedAt = stamp(); lease.release(); this.saveAttempts(attempt); }
}
throw lastError ?? new ModelRequestError({ message: '模型池请求未完成。', kind: 'connection' });
}
+190
View File
@@ -0,0 +1,190 @@
import { randomUUID, createHash } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
const metadataKey = 'modelAttemptLedger';
const sha = value => createHash('sha256').update(value).digest('hex');
const invalid = (message, status = 400) => Object.assign(new Error(message), { status });
const fields = { connectionId: 'connection_id', sessionId: 'session_id', taskId: 'task_id', runId: 'run_id', status: 'status', model: 'model', protocol: 'protocol' };
const ownership = value => JSON.stringify([value.id, value.connectionId ?? null, value.poolId ?? null, value.projectId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null, value.requestId ?? null, value.model ?? null, value.protocol ?? null]);
const integer = (value, minimum = 0) => Number.isSafeInteger(value) && value >= minimum;
const reference = row => ({ sequence: row.seq, payloadSha256: row.payload_sha256, initialSha256: row.initial_sha256,
origin: row.legacy_position === null ? 'incremental-ledger' : 'legacy-metadata', ...(row.legacy_position === null ? {} : { legacyIndex: row.legacy_position }) });
export function ledgerQuery(url, { exportOnly = false } = {}) {
const allowed = [...Object.keys(fields), 'query', ...(exportOnly ? [] : ['cursor', 'limit'])], params = url.searchParams;
for (const key of params.keys()) if (!allowed.includes(key) || params.getAll(key).length !== 1) throw invalid('请求账本查询参数无效。');
return { ...(exportOnly ? {} : { cursor: params.get('cursor'), limit: params.get('limit') ?? 50 }),
filters: Object.fromEntries([...Object.keys(fields), 'query'].filter(key => params.has(key) && params.get(key) !== '').map(key => [key, params.get(key)])) };
}
function where(filters = {}) {
const terms = [], parameters = [];
for (const key of Object.keys(filters).sort()) {
const value = filters[key]; if (typeof value !== 'string' || !value || ![...Object.keys(fields), 'query'].includes(key)) throw invalid('请求账本筛选字段无效。');
if (key === 'query') { terms.push("payload LIKE ? ESCAPE '\\'"); parameters.push('%' + value.replace(/[\\%_]/g, '\\$&') + '%'); }
else { terms.push(fields[key] + '=?'); parameters.push(value); }
}
return { sql: terms.length ? ' AND ' + terms.join(' AND ') : '', parameters };
}
function decode(row) {
if (sha(row.payload) !== row.payload_sha256) throw invalid('本机请求账本原记录 SHA256 不匹配,请核对保存来源。', 409);
return JSON.parse(row.payload);
}
// The legacy metadata stays byte-for-byte in its original row. A one-time
// transaction copies its entries into an indexed table. New requests update
// only their own row; the all-records accessor is compatibility/read-only use.
export class ModelRequestLedger {
constructor(storage) {
this.storage = storage; this.database = storage.database; this.positions = new WeakMap(); this.serialized = new WeakMap();
const originalVersion = this.database.prepare('PRAGMA user_version').get().user_version;
const state = storage.meta(metadataKey);
if (state && state.schemaVersion !== 1) throw invalid('请求账本由未接入的新版本创建,请使用对应程序。', 409);
if (!state) {
const original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value;
const entries = original ? JSON.parse(original) : [];
if (!Array.isArray(entries) || entries.some(value => !value || typeof value !== 'object' || Array.isArray(value) || typeof value.id !== 'string' || !value.id)) throw invalid('原请求账本格式无法完整迁移,原 metadata 已保留。', 409);
this.state = { schemaVersion: 1, id: randomUUID(), legacyEntries: entries.length, legacyMetadataSha256: original ? sha(original) : null,
originalMetadataPreserved: true, migratedAt: new Date().toISOString() };
storage.transaction(() => {
this.database.exec(`CREATE TABLE IF NOT EXISTS model_attempts (
seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL, connection_id TEXT, session_id TEXT, task_id TEXT, run_id TEXT,
status TEXT, model TEXT, protocol TEXT, legacy_position INTEGER UNIQUE,
payload TEXT NOT NULL CHECK(json_valid(payload)), payload_sha256 TEXT NOT NULL,
initial_payload TEXT NOT NULL CHECK(json_valid(initial_payload)), initial_sha256 TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS model_attempts_id ON model_attempts(id);
CREATE INDEX IF NOT EXISTS model_attempts_connection ON model_attempts(connection_id,seq);
CREATE INDEX IF NOT EXISTS model_attempts_session ON model_attempts(session_id,task_id,run_id,seq);
CREATE INDEX IF NOT EXISTS model_attempts_status ON model_attempts(status,seq);`);
for (const [index, entry] of entries.entries()) this.insert(entry, JSON.stringify(entry), index);
storage.setMeta(metadataKey, this.state);
this.database.exec('PRAGMA user_version=2');
});
this.initialization = { originalVersion };
} else {
this.state = state;
if (originalVersion < 2) storage.transaction(() => this.database.exec('PRAGMA user_version=2'));
}
}
insert(value, serialized, legacyIndex = null) {
const result = this.database.prepare(`INSERT INTO model_attempts(id,connection_id,session_id,task_id,run_id,status,model,protocol,legacy_position,payload,payload_sha256,initial_payload,initial_sha256)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(value.id, value.connectionId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null,
value.status ?? null, value.model ?? null, value.protocol ?? null, legacyIndex, serialized, sha(serialized), serialized, sha(serialized));
return Number(result.lastInsertRowid);
}
attach(row) { const value = decode(row); this.positions.set(value, row.seq); this.serialized.set(value, row.payload); return value; }
all() { return this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all().map(row => this.attach(row)); }
find(id) { const rows = this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(id); if (rows.length > 1) throw invalid('请求账本原身份不唯一,请按序号核对。', 409); return rows[0] ? decode(rows[0]) : undefined; }
rollbackInitialization() {
if (!this.initialization) return false;
// A failed synchronous handler adoption has not dispatched any request.
// Revert only this verified migration, never a ledger with new outcomes.
this.storage.transaction(() => {
const state = this.storage.meta(metadataKey), original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value;
const entries = original ? JSON.parse(original) : [], rows = this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all();
if (state?.id !== this.state.id || (original ? sha(original) : null) !== this.state.legacyMetadataSha256 || rows.length !== entries.length
|| rows.some((row, index) => row.legacy_position !== index || row.payload !== row.initial_payload || row.initial_payload !== JSON.stringify(entries[index])
|| sha(row.payload) !== row.payload_sha256 || sha(row.initial_payload) !== row.initial_sha256)) throw invalid('请求账本已有新记录或来源变化,不能回退原迁移,请保留现场核对。', 409);
this.database.exec('DROP TABLE model_attempts; PRAGMA user_version=' + this.initialization.originalVersion);
this.database.prepare('DELETE FROM metadata WHERE key=?').run(metadataKey);
});
this.initialization = null; return true;
}
save(value) { return this.saveAll([value])[0]; }
saveAll(values) {
if (values.every(value => value && this.serialized.get(value) === JSON.stringify(value))) return values.map(value => this.positions.get(value));
const updates = [];
this.storage.transaction(() => {
for (const value of values) {
if (!value || typeof value !== 'object' || typeof value.id !== 'string' || !value.id) throw invalid('请求账本记录需要原尝试身份。');
const serialized = JSON.stringify(value); if (this.serialized.get(value) === serialized) continue;
const sequence = this.positions.get(value), matches = sequence ? [this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(sequence)]
: this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(value.id);
if (matches.length > 1 || sequence && !matches[0]) throw invalid('请求账本记录身份不唯一或原行已变化,请核对来源。', 409);
let savedSequence;
if (matches[0]) {
const original = decode(matches[0]);
if (this.serialized.has(value) && this.serialized.get(value) !== matches[0].payload) throw invalid('请求账本原记录已更新,请核对保存来源,不能覆盖另一份进展。', 409);
if (ownership(original) !== ownership(value)) throw invalid('请求账本记录不能移动到其他连接、型号、协议、会话或任务。', 409);
this.database.prepare('UPDATE model_attempts SET status=?,payload=?,payload_sha256=? WHERE seq=?').run(value.status ?? null, serialized, sha(serialized), matches[0].seq);
savedSequence = matches[0].seq;
} else savedSequence = this.insert(value, serialized);
updates.push({ value, serialized, sequence: savedSequence });
}
});
for (const update of updates) { this.positions.set(update.value, update.sequence); this.serialized.set(update.value, update.serialized); }
return values.map(value => this.positions.get(value));
}
recoverRunning() {
const running = this.database.prepare("SELECT * FROM model_attempts WHERE status='running' ORDER BY seq").all().map(row => this.attach(row));
for (const value of running) { value.status = 'unknown'; value.finishedAt = new Date().toISOString(); value.reason = '宿主重启,请求结果与用量待核对。'; }
if (running.length) this.saveAll(running); return running.length;
}
page({ cursor = null, limit = 50, filters = {} } = {}) {
if (!/^\d+$/.test(String(limit)) || !integer(Number(limit), 1)) throw invalid('请求账本每页数量需要正整数。');
const filtered = where(filters), scope = sha(JSON.stringify({ ledger: this.state.id, filters: Object.fromEntries(Object.entries(filters).sort()) }));
const current = this.database.prepare('SELECT COALESCE(MAX(seq),0) AS seq,COUNT(*) AS total FROM model_attempts').get();
let snapshot = current.seq, before = null;
if (cursor !== null) {
let value; try { if (typeof cursor !== 'string' || !/^[A-Za-z0-9_-]+$/.test(cursor)) throw new Error(); value = JSON.parse(Buffer.from(cursor, 'base64url').toString()); } catch { throw invalid('请求账本游标无效。'); }
if (value?.schemaVersion !== 1 || value.scope !== scope || !integer(value.snapshot) || !integer(value.before, 1) || value.before > value.snapshot || value.snapshot > current.seq) throw invalid('请求账本游标不属于当前筛选或来源。');
snapshot = value.snapshot; before = value.before;
if (!this.database.prepare('SELECT seq FROM model_attempts WHERE seq=? AND seq<=?' + filtered.sql).get(before, snapshot, ...filtered.parameters)) throw invalid('请求账本原游标已变化,请刷新记录。', 409);
}
const count = this.database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total;
const rows = this.database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + (before === null ? '' : ' AND seq<?') + filtered.sql + ' ORDER BY seq DESC LIMIT ?')
.all(snapshot, ...(before === null ? [] : [before]), ...filtered.parameters, Number(limit) < Number.MAX_SAFE_INTEGER ? Number(limit) + 1 : Number(limit));
const shown = rows.slice(0, Number(limit)), hasMore = rows.length > Number(limit);
return { schemaVersion: 1, items: shown.map(row => decode(row)), references: shown.map(reference), total: count, retainedTotal: current.total, snapshotSequence: snapshot,
connectionIds: this.database.prepare('SELECT DISTINCT connection_id FROM model_attempts WHERE connection_id IS NOT NULL ORDER BY connection_id').all().map(row => row.connection_id),
hasMore, nextCursor: hasMore ? Buffer.from(JSON.stringify({ schemaVersion: 1, scope, snapshot, before: shown.at(-1).seq })).toString('base64url') : null,
filters, source: this.state, viewSnapshot: 'sequence-watermark; running record fields can still update' };
}
record(sequence) {
if (!/^\d+$/.test(String(sequence)) || !integer(Number(sequence), 1)) throw invalid('请求账本序号无效。');
const row = this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(Number(sequence)); if (!row) throw invalid('请求账本原记录不存在。', 404);
if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409);
return { item: decode(row), reference: reference(row), initial: { item: JSON.parse(row.initial_payload), sha256: row.initial_sha256 } };
}
exportSnapshot(filters = {}) {
const filtered = where(filters), database = new DatabaseSync(this.storage.filename, { readOnly: true });
try {
database.exec('PRAGMA query_only=ON; PRAGMA busy_timeout=5000; BEGIN;');
const snapshot = database.prepare('SELECT COALESCE(MAX(seq),0) AS seq FROM model_attempts').get().seq;
const count = database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total;
const rows = database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + filtered.sql + ' ORDER BY seq').iterate(snapshot, ...filtered.parameters);
return { header: { schemaVersion: 1, type: 'header', format: 'orven-model-attempts-ndjson', snapshotSequence: snapshot, expectedRecords: count,
filters, source: this.state, recordedAt: new Date().toISOString(), snapshot: 'SQLite read transaction', includesInitialReceipts: true, artifactBodies: 'Original artifacts remain separately downloadable by their saved session/ID/SHA.' }, rows,
close() { try { rows.return?.(); database.exec('ROLLBACK'); } finally { database.close(); } } };
} catch (cause) { database.close(); throw cause; }
}
}
export async function streamLedgerExport(response, snapshot) {
const checksum = createHash('sha256'); let count = 0;
const write = async line => {
if (response.destroyed || response.writableEnded) throw invalid('请求账本导出连接已关闭。', 499);
// ServerResponse can report backpressure after its socket has already
// flushed, without emitting another drain. The chunk callback reports
// actual write completion and bounds queued data to this one record.
await new Promise((resolve, reject) => {
let settled = false;
const clear = () => { response.off('close', closed); response.off('error', failed); };
const finish = error => { if (settled) return; settled = true; clear(); error ? reject(error) : resolve(); };
const closed = () => finish(invalid('请求账本导出被中断,部分内容不能当作完整导出。', 499)), failed = error => finish(error);
response.once('close', closed); response.once('error', failed);
try { response.write(line, finish); } catch (cause) { finish(cause); }
});
};
try {
await write(JSON.stringify(snapshot.header) + '\n');
for (const row of snapshot.rows) {
decode(row); if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409);
const line = '{"type":"attempt","reference":' + JSON.stringify(reference(row)) + ',"item":' + row.payload
+ ',"initial":{"sha256":' + JSON.stringify(row.initial_sha256) + ',"item":' + row.initial_payload + '}}\n';
checksum.update(line); await write(line); count++;
}
if (count !== snapshot.header.expectedRecords) throw invalid('请求账本导出数量与快照不匹配。', 409);
await write(JSON.stringify({ type: 'footer', complete: true, records: count, recordLinesSha256: checksum.digest('hex') }) + '\n'); response.end();
} finally { snapshot.close(); }
}