feat(models): preserve complete incremental request ledger and exports
This commit is contained in:
@@ -16,6 +16,7 @@ import { modelIdentitySettings } from '../../shared/contracts/model-identity.mjs
|
||||
import { NetworkDiagnostics } from './network-diagnostics.mjs';
|
||||
import { ModelCatalog, credentialScope } from './model-catalog.mjs';
|
||||
import { capacitySettings, routedCapacity } from '../../shared/contracts/model-capacity.mjs';
|
||||
import { ModelRequestLedger } from './request-ledger.mjs';
|
||||
|
||||
const legacyId = 'legacy-default';
|
||||
const stamp = () => new Date().toISOString();
|
||||
@@ -34,11 +35,9 @@ export class ModelHub {
|
||||
this.store = store; this.provider = provider;
|
||||
this.state = store.storage.meta('modelRegistry', { schema: 1, revision: 0, selected: { type: 'legacy', id: legacyId }, connections: [], pools: [], imports: {} });
|
||||
this.health = new Map(); this.leases = new Map(); this.cursors = new Map(); this.sticky = new Map(); this.waiting = 0;
|
||||
this.attempts = store.storage.meta('modelAttempts', []);
|
||||
this.ledger = new ModelRequestLedger(store.storage);
|
||||
// A process exit leaves a request outcome unknown; never replay it here.
|
||||
let recovered = false;
|
||||
for (const attempt of this.attempts) if (attempt.status === 'running') { attempt.status = 'unknown'; attempt.finishedAt = stamp(); attempt.reason = '宿主重启,请求结果与用量待核对。'; recovered = true; }
|
||||
if (recovered) this.saveAttempts();
|
||||
this.ledger.recoverRunning();
|
||||
provider.credentials ??= new Map();
|
||||
if (store.storage.meta('modelCredentials', null)) this.vault().restore(this);
|
||||
}
|
||||
@@ -86,7 +85,27 @@ export class ModelHub {
|
||||
throw cause;
|
||||
} finally { this.editingRegistry = false; }
|
||||
}
|
||||
saveAttempts() { if (!unrestricted(this.store)) this.attempts = this.attempts.slice(-1000); this.store.storage.transaction(() => this.store.storage.setMeta('modelAttempts', this.attempts)); }
|
||||
get attempts() { return this.compatAttempts ??= this.ledger.all(); }
|
||||
set attempts(value) { this.compatAttempts = value; }
|
||||
adoptRequestLedger() {
|
||||
if (this.ledger) { Object.setPrototypeOf(this.ledger, ModelRequestLedger.prototype); return this.ledger; }
|
||||
const retained = Object.getOwnPropertyDescriptor(this, 'attempts');
|
||||
if (retained && (!Array.isArray(retained.value) || JSON.stringify(retained.value) !== JSON.stringify(this.store.storage.meta('modelAttempts', [])))) throw error('原内存请求账本与保存来源不同,请先核对,未重放任何请求。', 409);
|
||||
const ledger = new ModelRequestLedger(this.store.storage);
|
||||
if (retained) { delete this.attempts; this.compatAttempts = retained.value; }
|
||||
this.ledger = ledger; return ledger;
|
||||
}
|
||||
saveAttempts(attempt = null) {
|
||||
if (!attempt) return this.ledger.saveAll(this.attempts);
|
||||
const sequence = this.ledger.save(attempt);
|
||||
if (this.compatAttempts) {
|
||||
const positions = this.compatAttempts.map((value, index) => ({ value, index })).filter(({ value }) => value === attempt || this.ledger.positions.get(value) === sequence);
|
||||
const sameId = positions.length ? positions : this.compatAttempts.map((value, index) => ({ value, index })).filter(({ value }) => value.id === attempt.id);
|
||||
if (sameId.length === 1) this.compatAttempts[sameId[0].index] = attempt;
|
||||
}
|
||||
return sequence;
|
||||
}
|
||||
addAttempt(attempt) { this.ledger.save(attempt); this.compatAttempts?.push(attempt); }
|
||||
connections() {
|
||||
const saved = this.state.legacyConnection;
|
||||
return [{ ...saved, id: legacyId, name: saved?.name || this.store.config.model || '未配置模型', brand: saved?.brand || '自定义', protocol: saved?.protocol || 'chat-completions', baseUrl: this.store.config.baseUrl,
|
||||
@@ -144,9 +163,10 @@ export class ModelHub {
|
||||
...(this.state.capabilityResults?.[connection.id] ? { capabilityEvidence: this.state.capabilityResults[connection.id] } : {}),
|
||||
capabilities: capabilityView({ ...connection, capabilityEvidence: this.state.capabilityResults?.[connection.id] }, credential?.revision ?? 0) };
|
||||
}
|
||||
view() { return { revision: this.state.revision, selected: this.state.selected, waitSettings: modelWaitSettings(this.store.config), connections: this.connections().map(value => this.status(value)), pools: this.state.pools,
|
||||
imports: Object.entries(this.state.imports).map(([id, value]) => ({ id, ...value })), attempts: this.attempts.slice(-100).reverse(), waiting: this.waiting,
|
||||
storage: { credentials: this.credentialVault ? 'memory-or-windows-user-dpapi' : 'memory', restart: '临时凭据在宿主退出后需要重新填写;已明确本机加密保存的连接按用户与原配置加载。', attemptsRetained: 1000 } }; }
|
||||
view() { const page = this.ledger.page({ limit: 100 }); return { revision: this.state.revision, selected: this.state.selected, waitSettings: modelWaitSettings(this.store.config), connections: this.connections().map(value => this.status(value)), pools: this.state.pools,
|
||||
imports: Object.entries(this.state.imports).map(([id, value]) => ({ id, ...value })), attempts: page.items, attemptCount: page.retainedTotal, attemptsHasMore: page.hasMore, waiting: this.waiting,
|
||||
storage: { credentials: this.credentialVault ? 'memory-or-windows-user-dpapi' : 'memory', restart: '临时凭据在宿主退出后需要重新填写;已明确本机加密保存的连接按用户与原配置加载。', attemptsRetained: null,
|
||||
attemptStorage: 'incremental-sqlite', attemptPreview: 100, originalMetadataPreserved: true } }; }
|
||||
selectionView() {
|
||||
const selected = this.state.selected;
|
||||
const name = selected.type === 'pool' ? this.state.pools.find(value => value.id === selected.id)?.name
|
||||
@@ -192,7 +212,7 @@ export class ModelHub {
|
||||
const previous = this.state.capabilityResults[id];
|
||||
const results = previous?.connectionRevision === connection.revision && previous.credentialRevision === member.credentialRevision ? { ...previous.results } : {};
|
||||
results[scope] = 'verified';
|
||||
const attempt = this.attempts.find(value => value.id === attemptId);
|
||||
const attempt = this.ledger.find(attemptId);
|
||||
if (!attempt || attempt.connectionId !== id || attempt.status !== 'completed') throw error('检测原请求账本不完整,未更新能力结论。');
|
||||
this.state.capabilityResults[id] = { schema: 1, connectionRevision: connection.revision, credentialRevision: member.credentialRevision, results,
|
||||
checkedAt: stamp(), protocol: modelProtocol(connection.protocol), attemptId: attempt?.id, source: 'explicit-model-probe',
|
||||
@@ -364,9 +384,9 @@ export class ModelHub {
|
||||
try { lease = await this.acquire(routing, runId, excluded, signal, deadline, requirements, { probe: config.capabilityProbe === true }); }
|
||||
catch (cause) {
|
||||
if (!lastError && cause.routingDecision) {
|
||||
this.attempts.push({ id: randomUUID(), requestId, taskId: plan?.taskId ?? null, runId, sessionId: plan?.sessionId ?? null, projectId: plan?.projectId ?? null,
|
||||
this.addAttempt({ id: randomUUID(), requestId, taskId: plan?.taskId ?? null, runId, sessionId: plan?.sessionId ?? null, projectId: plan?.projectId ?? null,
|
||||
poolId: routing.type === 'pool' ? routing.id : null, status: 'rejected', reason: cause.message, routingDecision: cause.routingDecision,
|
||||
startedAt: stamp(), finishedAt: stamp(), usage: null, usageStatus: 'not_dispatched' }); this.saveAttempts();
|
||||
startedAt: stamp(), finishedAt: stamp(), usage: null, usageStatus: 'not_dispatched' });
|
||||
}
|
||||
throw lastError ?? cause;
|
||||
}
|
||||
@@ -388,7 +408,7 @@ export class ModelHub {
|
||||
this.store.core.artifact.set(artifact.id, artifact);
|
||||
this.store.save(session);
|
||||
}
|
||||
this.attempts.push(attempt); this.saveAttempts();
|
||||
this.addAttempt(attempt);
|
||||
const progress = event => {
|
||||
if (event.modelIdentity) attempt.modelIdentity = event.modelIdentity;
|
||||
if (event.wireAttemptId) attempt.wireAttemptId = event.wireAttemptId;
|
||||
@@ -398,7 +418,7 @@ export class ModelHub {
|
||||
attempt.usage = event.consumption.usage; attempt.usageStatus = event.consumption.usageStatus;
|
||||
onProgress?.({ consumption: { ...event.consumption, wireAttemptId: event.consumption.attemptId, attemptId: attempt.id,
|
||||
connectionId: member.id, connectionName: member.name }, connectionId: member.id, model: member.model, modelAttemptId: attempt.id });
|
||||
this.saveAttempts();
|
||||
this.saveAttempts(attempt);
|
||||
return;
|
||||
}
|
||||
if (event.streamCapture !== undefined || event.responseCapture !== undefined) {
|
||||
@@ -456,7 +476,7 @@ export class ModelHub {
|
||||
if (config.capacityMode === 'passthrough' || (!diagnostic?.retryable && !diagnostic?.failoverAllowed) || count === routing.maxAttempts) throw cause;
|
||||
onProgress?.({ phase: 'retrying', attempt: count, maxAttempts: routing.maxAttempts, connectionId: member.id, modelAttemptId: attempt.id,
|
||||
diagnostic, waitMs: 0, switching: true });
|
||||
} finally { attempt.finishedAt = stamp(); lease.release(); this.saveAttempts(); }
|
||||
} finally { attempt.finishedAt = stamp(); lease.release(); this.saveAttempts(attempt); }
|
||||
}
|
||||
throw lastError ?? new ModelRequestError({ message: '模型池请求未完成。', kind: 'connection' });
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
const metadataKey = 'modelAttemptLedger';
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
const invalid = (message, status = 400) => Object.assign(new Error(message), { status });
|
||||
const fields = { connectionId: 'connection_id', sessionId: 'session_id', taskId: 'task_id', runId: 'run_id', status: 'status', model: 'model', protocol: 'protocol' };
|
||||
const ownership = value => JSON.stringify([value.id, value.connectionId ?? null, value.poolId ?? null, value.projectId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null, value.requestId ?? null, value.model ?? null, value.protocol ?? null]);
|
||||
const integer = (value, minimum = 0) => Number.isSafeInteger(value) && value >= minimum;
|
||||
const reference = row => ({ sequence: row.seq, payloadSha256: row.payload_sha256, initialSha256: row.initial_sha256,
|
||||
origin: row.legacy_position === null ? 'incremental-ledger' : 'legacy-metadata', ...(row.legacy_position === null ? {} : { legacyIndex: row.legacy_position }) });
|
||||
|
||||
export function ledgerQuery(url, { exportOnly = false } = {}) {
|
||||
const allowed = [...Object.keys(fields), 'query', ...(exportOnly ? [] : ['cursor', 'limit'])], params = url.searchParams;
|
||||
for (const key of params.keys()) if (!allowed.includes(key) || params.getAll(key).length !== 1) throw invalid('请求账本查询参数无效。');
|
||||
return { ...(exportOnly ? {} : { cursor: params.get('cursor'), limit: params.get('limit') ?? 50 }),
|
||||
filters: Object.fromEntries([...Object.keys(fields), 'query'].filter(key => params.has(key) && params.get(key) !== '').map(key => [key, params.get(key)])) };
|
||||
}
|
||||
|
||||
function where(filters = {}) {
|
||||
const terms = [], parameters = [];
|
||||
for (const key of Object.keys(filters).sort()) {
|
||||
const value = filters[key]; if (typeof value !== 'string' || !value || ![...Object.keys(fields), 'query'].includes(key)) throw invalid('请求账本筛选字段无效。');
|
||||
if (key === 'query') { terms.push("payload LIKE ? ESCAPE '\\'"); parameters.push('%' + value.replace(/[\\%_]/g, '\\$&') + '%'); }
|
||||
else { terms.push(fields[key] + '=?'); parameters.push(value); }
|
||||
}
|
||||
return { sql: terms.length ? ' AND ' + terms.join(' AND ') : '', parameters };
|
||||
}
|
||||
function decode(row) {
|
||||
if (sha(row.payload) !== row.payload_sha256) throw invalid('本机请求账本原记录 SHA256 不匹配,请核对保存来源。', 409);
|
||||
return JSON.parse(row.payload);
|
||||
}
|
||||
|
||||
// The legacy metadata stays byte-for-byte in its original row. A one-time
|
||||
// transaction copies its entries into an indexed table. New requests update
|
||||
// only their own row; the all-records accessor is compatibility/read-only use.
|
||||
export class ModelRequestLedger {
|
||||
constructor(storage) {
|
||||
this.storage = storage; this.database = storage.database; this.positions = new WeakMap(); this.serialized = new WeakMap();
|
||||
const originalVersion = this.database.prepare('PRAGMA user_version').get().user_version;
|
||||
const state = storage.meta(metadataKey);
|
||||
if (state && state.schemaVersion !== 1) throw invalid('请求账本由未接入的新版本创建,请使用对应程序。', 409);
|
||||
if (!state) {
|
||||
const original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value;
|
||||
const entries = original ? JSON.parse(original) : [];
|
||||
if (!Array.isArray(entries) || entries.some(value => !value || typeof value !== 'object' || Array.isArray(value) || typeof value.id !== 'string' || !value.id)) throw invalid('原请求账本格式无法完整迁移,原 metadata 已保留。', 409);
|
||||
this.state = { schemaVersion: 1, id: randomUUID(), legacyEntries: entries.length, legacyMetadataSha256: original ? sha(original) : null,
|
||||
originalMetadataPreserved: true, migratedAt: new Date().toISOString() };
|
||||
storage.transaction(() => {
|
||||
this.database.exec(`CREATE TABLE IF NOT EXISTS model_attempts (
|
||||
seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL, connection_id TEXT, session_id TEXT, task_id TEXT, run_id TEXT,
|
||||
status TEXT, model TEXT, protocol TEXT, legacy_position INTEGER UNIQUE,
|
||||
payload TEXT NOT NULL CHECK(json_valid(payload)), payload_sha256 TEXT NOT NULL,
|
||||
initial_payload TEXT NOT NULL CHECK(json_valid(initial_payload)), initial_sha256 TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS model_attempts_id ON model_attempts(id);
|
||||
CREATE INDEX IF NOT EXISTS model_attempts_connection ON model_attempts(connection_id,seq);
|
||||
CREATE INDEX IF NOT EXISTS model_attempts_session ON model_attempts(session_id,task_id,run_id,seq);
|
||||
CREATE INDEX IF NOT EXISTS model_attempts_status ON model_attempts(status,seq);`);
|
||||
for (const [index, entry] of entries.entries()) this.insert(entry, JSON.stringify(entry), index);
|
||||
storage.setMeta(metadataKey, this.state);
|
||||
this.database.exec('PRAGMA user_version=2');
|
||||
});
|
||||
this.initialization = { originalVersion };
|
||||
} else {
|
||||
this.state = state;
|
||||
if (originalVersion < 2) storage.transaction(() => this.database.exec('PRAGMA user_version=2'));
|
||||
}
|
||||
}
|
||||
insert(value, serialized, legacyIndex = null) {
|
||||
const result = this.database.prepare(`INSERT INTO model_attempts(id,connection_id,session_id,task_id,run_id,status,model,protocol,legacy_position,payload,payload_sha256,initial_payload,initial_sha256)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(value.id, value.connectionId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null,
|
||||
value.status ?? null, value.model ?? null, value.protocol ?? null, legacyIndex, serialized, sha(serialized), serialized, sha(serialized));
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
attach(row) { const value = decode(row); this.positions.set(value, row.seq); this.serialized.set(value, row.payload); return value; }
|
||||
all() { return this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all().map(row => this.attach(row)); }
|
||||
find(id) { const rows = this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(id); if (rows.length > 1) throw invalid('请求账本原身份不唯一,请按序号核对。', 409); return rows[0] ? decode(rows[0]) : undefined; }
|
||||
rollbackInitialization() {
|
||||
if (!this.initialization) return false;
|
||||
// A failed synchronous handler adoption has not dispatched any request.
|
||||
// Revert only this verified migration, never a ledger with new outcomes.
|
||||
this.storage.transaction(() => {
|
||||
const state = this.storage.meta(metadataKey), original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value;
|
||||
const entries = original ? JSON.parse(original) : [], rows = this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all();
|
||||
if (state?.id !== this.state.id || (original ? sha(original) : null) !== this.state.legacyMetadataSha256 || rows.length !== entries.length
|
||||
|| rows.some((row, index) => row.legacy_position !== index || row.payload !== row.initial_payload || row.initial_payload !== JSON.stringify(entries[index])
|
||||
|| sha(row.payload) !== row.payload_sha256 || sha(row.initial_payload) !== row.initial_sha256)) throw invalid('请求账本已有新记录或来源变化,不能回退原迁移,请保留现场核对。', 409);
|
||||
this.database.exec('DROP TABLE model_attempts; PRAGMA user_version=' + this.initialization.originalVersion);
|
||||
this.database.prepare('DELETE FROM metadata WHERE key=?').run(metadataKey);
|
||||
});
|
||||
this.initialization = null; return true;
|
||||
}
|
||||
save(value) { return this.saveAll([value])[0]; }
|
||||
saveAll(values) {
|
||||
if (values.every(value => value && this.serialized.get(value) === JSON.stringify(value))) return values.map(value => this.positions.get(value));
|
||||
const updates = [];
|
||||
this.storage.transaction(() => {
|
||||
for (const value of values) {
|
||||
if (!value || typeof value !== 'object' || typeof value.id !== 'string' || !value.id) throw invalid('请求账本记录需要原尝试身份。');
|
||||
const serialized = JSON.stringify(value); if (this.serialized.get(value) === serialized) continue;
|
||||
const sequence = this.positions.get(value), matches = sequence ? [this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(sequence)]
|
||||
: this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(value.id);
|
||||
if (matches.length > 1 || sequence && !matches[0]) throw invalid('请求账本记录身份不唯一或原行已变化,请核对来源。', 409);
|
||||
let savedSequence;
|
||||
if (matches[0]) {
|
||||
const original = decode(matches[0]);
|
||||
if (this.serialized.has(value) && this.serialized.get(value) !== matches[0].payload) throw invalid('请求账本原记录已更新,请核对保存来源,不能覆盖另一份进展。', 409);
|
||||
if (ownership(original) !== ownership(value)) throw invalid('请求账本记录不能移动到其他连接、型号、协议、会话或任务。', 409);
|
||||
this.database.prepare('UPDATE model_attempts SET status=?,payload=?,payload_sha256=? WHERE seq=?').run(value.status ?? null, serialized, sha(serialized), matches[0].seq);
|
||||
savedSequence = matches[0].seq;
|
||||
} else savedSequence = this.insert(value, serialized);
|
||||
updates.push({ value, serialized, sequence: savedSequence });
|
||||
}
|
||||
});
|
||||
for (const update of updates) { this.positions.set(update.value, update.sequence); this.serialized.set(update.value, update.serialized); }
|
||||
return values.map(value => this.positions.get(value));
|
||||
}
|
||||
recoverRunning() {
|
||||
const running = this.database.prepare("SELECT * FROM model_attempts WHERE status='running' ORDER BY seq").all().map(row => this.attach(row));
|
||||
for (const value of running) { value.status = 'unknown'; value.finishedAt = new Date().toISOString(); value.reason = '宿主重启,请求结果与用量待核对。'; }
|
||||
if (running.length) this.saveAll(running); return running.length;
|
||||
}
|
||||
page({ cursor = null, limit = 50, filters = {} } = {}) {
|
||||
if (!/^\d+$/.test(String(limit)) || !integer(Number(limit), 1)) throw invalid('请求账本每页数量需要正整数。');
|
||||
const filtered = where(filters), scope = sha(JSON.stringify({ ledger: this.state.id, filters: Object.fromEntries(Object.entries(filters).sort()) }));
|
||||
const current = this.database.prepare('SELECT COALESCE(MAX(seq),0) AS seq,COUNT(*) AS total FROM model_attempts').get();
|
||||
let snapshot = current.seq, before = null;
|
||||
if (cursor !== null) {
|
||||
let value; try { if (typeof cursor !== 'string' || !/^[A-Za-z0-9_-]+$/.test(cursor)) throw new Error(); value = JSON.parse(Buffer.from(cursor, 'base64url').toString()); } catch { throw invalid('请求账本游标无效。'); }
|
||||
if (value?.schemaVersion !== 1 || value.scope !== scope || !integer(value.snapshot) || !integer(value.before, 1) || value.before > value.snapshot || value.snapshot > current.seq) throw invalid('请求账本游标不属于当前筛选或来源。');
|
||||
snapshot = value.snapshot; before = value.before;
|
||||
if (!this.database.prepare('SELECT seq FROM model_attempts WHERE seq=? AND seq<=?' + filtered.sql).get(before, snapshot, ...filtered.parameters)) throw invalid('请求账本原游标已变化,请刷新记录。', 409);
|
||||
}
|
||||
const count = this.database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total;
|
||||
const rows = this.database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + (before === null ? '' : ' AND seq<?') + filtered.sql + ' ORDER BY seq DESC LIMIT ?')
|
||||
.all(snapshot, ...(before === null ? [] : [before]), ...filtered.parameters, Number(limit) < Number.MAX_SAFE_INTEGER ? Number(limit) + 1 : Number(limit));
|
||||
const shown = rows.slice(0, Number(limit)), hasMore = rows.length > Number(limit);
|
||||
return { schemaVersion: 1, items: shown.map(row => decode(row)), references: shown.map(reference), total: count, retainedTotal: current.total, snapshotSequence: snapshot,
|
||||
connectionIds: this.database.prepare('SELECT DISTINCT connection_id FROM model_attempts WHERE connection_id IS NOT NULL ORDER BY connection_id').all().map(row => row.connection_id),
|
||||
hasMore, nextCursor: hasMore ? Buffer.from(JSON.stringify({ schemaVersion: 1, scope, snapshot, before: shown.at(-1).seq })).toString('base64url') : null,
|
||||
filters, source: this.state, viewSnapshot: 'sequence-watermark; running record fields can still update' };
|
||||
}
|
||||
record(sequence) {
|
||||
if (!/^\d+$/.test(String(sequence)) || !integer(Number(sequence), 1)) throw invalid('请求账本序号无效。');
|
||||
const row = this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(Number(sequence)); if (!row) throw invalid('请求账本原记录不存在。', 404);
|
||||
if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409);
|
||||
return { item: decode(row), reference: reference(row), initial: { item: JSON.parse(row.initial_payload), sha256: row.initial_sha256 } };
|
||||
}
|
||||
exportSnapshot(filters = {}) {
|
||||
const filtered = where(filters), database = new DatabaseSync(this.storage.filename, { readOnly: true });
|
||||
try {
|
||||
database.exec('PRAGMA query_only=ON; PRAGMA busy_timeout=5000; BEGIN;');
|
||||
const snapshot = database.prepare('SELECT COALESCE(MAX(seq),0) AS seq FROM model_attempts').get().seq;
|
||||
const count = database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total;
|
||||
const rows = database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + filtered.sql + ' ORDER BY seq').iterate(snapshot, ...filtered.parameters);
|
||||
return { header: { schemaVersion: 1, type: 'header', format: 'orven-model-attempts-ndjson', snapshotSequence: snapshot, expectedRecords: count,
|
||||
filters, source: this.state, recordedAt: new Date().toISOString(), snapshot: 'SQLite read transaction', includesInitialReceipts: true, artifactBodies: 'Original artifacts remain separately downloadable by their saved session/ID/SHA.' }, rows,
|
||||
close() { try { rows.return?.(); database.exec('ROLLBACK'); } finally { database.close(); } } };
|
||||
} catch (cause) { database.close(); throw cause; }
|
||||
}
|
||||
}
|
||||
|
||||
export async function streamLedgerExport(response, snapshot) {
|
||||
const checksum = createHash('sha256'); let count = 0;
|
||||
const write = async line => {
|
||||
if (response.destroyed || response.writableEnded) throw invalid('请求账本导出连接已关闭。', 499);
|
||||
// ServerResponse can report backpressure after its socket has already
|
||||
// flushed, without emitting another drain. The chunk callback reports
|
||||
// actual write completion and bounds queued data to this one record.
|
||||
await new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
const clear = () => { response.off('close', closed); response.off('error', failed); };
|
||||
const finish = error => { if (settled) return; settled = true; clear(); error ? reject(error) : resolve(); };
|
||||
const closed = () => finish(invalid('请求账本导出被中断,部分内容不能当作完整导出。', 499)), failed = error => finish(error);
|
||||
response.once('close', closed); response.once('error', failed);
|
||||
try { response.write(line, finish); } catch (cause) { finish(cause); }
|
||||
});
|
||||
};
|
||||
try {
|
||||
await write(JSON.stringify(snapshot.header) + '\n');
|
||||
for (const row of snapshot.rows) {
|
||||
decode(row); if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409);
|
||||
const line = '{"type":"attempt","reference":' + JSON.stringify(reference(row)) + ',"item":' + row.payload
|
||||
+ ',"initial":{"sha256":' + JSON.stringify(row.initial_sha256) + ',"item":' + row.initial_payload + '}}\n';
|
||||
checksum.update(line); await write(line); count++;
|
||||
}
|
||||
if (count !== snapshot.header.expectedRecords) throw invalid('请求账本导出数量与快照不匹配。', 409);
|
||||
await write(JSON.stringify({ type: 'footer', complete: true, records: count, recordLinesSha256: checksum.digest('hex') }) + '\n'); response.end();
|
||||
} finally { snapshot.close(); }
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
|
||||
const DATABASE_SCHEMA = 1;
|
||||
const DATABASE_SCHEMA = 2;
|
||||
|
||||
export class StateDatabase {
|
||||
constructor(directory, { faultInjector } = {}) {
|
||||
@@ -27,7 +27,7 @@ export class StateDatabase {
|
||||
task_id TEXT, run_id TEXT, type TEXT NOT NULL, revision INTEGER NOT NULL, created_at TEXT NOT NULL, payload TEXT NOT NULL CHECK(json_valid(payload)));
|
||||
CREATE INDEX IF NOT EXISTS events_session ON events(session_id,seq);
|
||||
CREATE TABLE IF NOT EXISTS outbox (seq INTEGER PRIMARY KEY REFERENCES events(seq), published_at TEXT);
|
||||
PRAGMA user_version=${DATABASE_SCHEMA};
|
||||
PRAGMA user_version=${Math.max(version, 1)};
|
||||
`);
|
||||
} catch (error) { this.database.close(); throw error; }
|
||||
}
|
||||
@@ -39,6 +39,14 @@ export class StateDatabase {
|
||||
}
|
||||
|
||||
meta(key, fallback = null) {
|
||||
// Legacy in-process readers receive the full ledger projection. The
|
||||
// original metadata row remains unchanged for archived source verification.
|
||||
if (key === 'modelAttempts' && this.database.prepare("SELECT value FROM metadata WHERE key='modelAttemptLedger'").get()) {
|
||||
return this.database.prepare('SELECT payload,payload_sha256 FROM model_attempts ORDER BY seq').all().map(row => {
|
||||
if (createHash('sha256').update(row.payload).digest('hex') !== row.payload_sha256) throw new Error('本机请求账本原记录 SHA256 不匹配,请核对保存来源。');
|
||||
return JSON.parse(row.payload);
|
||||
});
|
||||
}
|
||||
const row = this.database.prepare('SELECT value FROM metadata WHERE key=?').get(key);
|
||||
return row ? JSON.parse(row.value) : structuredClone(fallback);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user