import { randomUUID, createHash } from 'node:crypto'; import { DatabaseSync } from 'node:sqlite'; import { REQUEST_LEDGER_SQL } from '../storage/request-ledger-schema.mjs'; const metadataKey = 'modelAttemptLedger'; const sha = value => createHash('sha256').update(value).digest('hex'); const invalid = (message, status = 400) => Object.assign(new Error(message), { status }); const fields = { connectionId: 'connection_id', sessionId: 'session_id', taskId: 'task_id', runId: 'run_id', status: 'status', model: 'model', protocol: 'protocol' }; const ownership = value => JSON.stringify([value.id, value.connectionId ?? null, value.poolId ?? null, value.projectId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null, value.requestId ?? null, value.model ?? null, value.protocol ?? null]); const integer = (value, minimum = 0) => Number.isSafeInteger(value) && value >= minimum; const reference = row => ({ sequence: row.seq, payloadSha256: row.payload_sha256, initialSha256: row.initial_sha256, origin: row.legacy_position === null ? 'incremental-ledger' : 'legacy-metadata', ...(row.legacy_position === null ? {} : { legacyIndex: row.legacy_position }) }); export function ledgerQuery(url, { exportOnly = false } = {}) { const allowed = [...Object.keys(fields), 'query', ...(exportOnly ? [] : ['cursor', 'limit'])], params = url.searchParams; for (const key of params.keys()) if (!allowed.includes(key) || params.getAll(key).length !== 1) throw invalid('请求账本查询参数无效。'); return { ...(exportOnly ? {} : { cursor: params.get('cursor'), limit: params.get('limit') ?? 50 }), filters: Object.fromEntries([...Object.keys(fields), 'query'].filter(key => params.has(key) && params.get(key) !== '').map(key => [key, params.get(key)])) }; } function where(filters = {}) { const terms = [], parameters = []; for (const key of Object.keys(filters).sort()) { const value = filters[key]; if (typeof value !== 'string' || !value || ![...Object.keys(fields), 'query'].includes(key)) throw invalid('请求账本筛选字段无效。'); if (key === 'query') { terms.push("payload LIKE ? ESCAPE '\\'"); parameters.push('%' + value.replace(/[\\%_]/g, '\\$&') + '%'); } else { terms.push(fields[key] + '=?'); parameters.push(value); } } return { sql: terms.length ? ' AND ' + terms.join(' AND ') : '', parameters }; } function decode(row) { if (sha(row.payload) !== row.payload_sha256) throw invalid('本机请求账本原记录 SHA256 不匹配,请核对保存来源。', 409); return JSON.parse(row.payload); } // The legacy metadata stays byte-for-byte in its original row. A one-time // transaction copies its entries into an indexed table. New requests update // only their own row; the all-records accessor is compatibility/read-only use. export class ModelRequestLedger { constructor(storage) { this.storage = storage; this.database = storage.database; this.positions = new WeakMap(); this.serialized = new WeakMap(); const originalVersion = this.database.prepare('PRAGMA user_version').get().user_version; const state = storage.meta(metadataKey); if (state && state.schemaVersion !== 1) throw invalid('请求账本由未接入的新版本创建,请使用对应程序。', 409); if (!state) { const original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value; const entries = original ? JSON.parse(original) : []; if (!Array.isArray(entries) || entries.some(value => !value || typeof value !== 'object' || Array.isArray(value) || typeof value.id !== 'string' || !value.id)) throw invalid('原请求账本格式无法完整迁移,原 metadata 已保留。', 409); this.state = { schemaVersion: 1, id: randomUUID(), legacyEntries: entries.length, legacyMetadataSha256: original ? sha(original) : null, originalMetadataPreserved: true, migratedAt: new Date().toISOString() }; storage.transaction(() => { this.database.exec(REQUEST_LEDGER_SQL); for (const [index, entry] of entries.entries()) this.insert(entry, JSON.stringify(entry), index); storage.setMeta(metadataKey, this.state); this.database.exec('PRAGMA user_version=2'); }); this.initialization = { originalVersion }; } else { this.state = state; if (originalVersion < 2) storage.transaction(() => this.database.exec('PRAGMA user_version=2')); } } insert(value, serialized, legacyIndex = null) { const result = this.database.prepare(`INSERT INTO model_attempts(id,connection_id,session_id,task_id,run_id,status,model,protocol,legacy_position,payload,payload_sha256,initial_payload,initial_sha256) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(value.id, value.connectionId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null, value.status ?? null, value.model ?? null, value.protocol ?? null, legacyIndex, serialized, sha(serialized), serialized, sha(serialized)); return Number(result.lastInsertRowid); } attach(row) { const value = decode(row); this.positions.set(value, row.seq); this.serialized.set(value, row.payload); return value; } all() { return this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all().map(row => this.attach(row)); } find(id) { const rows = this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(id); if (rows.length > 1) throw invalid('请求账本原身份不唯一,请按序号核对。', 409); return rows[0] ? decode(rows[0]) : undefined; } rollbackInitialization() { if (!this.initialization) return false; // A failed synchronous handler adoption has not dispatched any request. // Revert only this verified migration, never a ledger with new outcomes. this.storage.transaction(() => { const state = this.storage.meta(metadataKey), original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value; const entries = original ? JSON.parse(original) : [], rows = this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all(); if (state?.id !== this.state.id || (original ? sha(original) : null) !== this.state.legacyMetadataSha256 || rows.length !== entries.length || rows.some((row, index) => row.legacy_position !== index || row.payload !== row.initial_payload || row.initial_payload !== JSON.stringify(entries[index]) || sha(row.payload) !== row.payload_sha256 || sha(row.initial_payload) !== row.initial_sha256)) throw invalid('请求账本已有新记录或来源变化,不能回退原迁移,请保留现场核对。', 409); this.database.exec('DROP TABLE model_attempts; PRAGMA user_version=' + this.initialization.originalVersion); this.database.prepare('DELETE FROM metadata WHERE key=?').run(metadataKey); }); this.initialization = null; return true; } save(value) { return this.saveAll([value])[0]; } saveAll(values) { if (values.every(value => value && this.serialized.get(value) === JSON.stringify(value))) return values.map(value => this.positions.get(value)); const updates = []; this.storage.transaction(() => { for (const value of values) { if (!value || typeof value !== 'object' || typeof value.id !== 'string' || !value.id) throw invalid('请求账本记录需要原尝试身份。'); const serialized = JSON.stringify(value); if (this.serialized.get(value) === serialized) continue; const sequence = this.positions.get(value), matches = sequence ? [this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(sequence)] : this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(value.id); if (matches.length > 1 || sequence && !matches[0]) throw invalid('请求账本记录身份不唯一或原行已变化,请核对来源。', 409); let savedSequence; if (matches[0]) { const original = decode(matches[0]); if (this.serialized.has(value) && this.serialized.get(value) !== matches[0].payload) throw invalid('请求账本原记录已更新,请核对保存来源,不能覆盖另一份进展。', 409); if (ownership(original) !== ownership(value)) throw invalid('请求账本记录不能移动到其他连接、型号、协议、会话或任务。', 409); this.database.prepare('UPDATE model_attempts SET status=?,payload=?,payload_sha256=? WHERE seq=?').run(value.status ?? null, serialized, sha(serialized), matches[0].seq); savedSequence = matches[0].seq; } else savedSequence = this.insert(value, serialized); updates.push({ value, serialized, sequence: savedSequence }); } }); for (const update of updates) { this.positions.set(update.value, update.sequence); this.serialized.set(update.value, update.serialized); } return values.map(value => this.positions.get(value)); } recoverRunning() { const running = this.database.prepare("SELECT * FROM model_attempts WHERE status='running' ORDER BY seq").all().map(row => this.attach(row)); for (const value of running) { value.status = 'unknown'; value.finishedAt = new Date().toISOString(); value.reason = '宿主重启,请求结果与用量待核对。'; } if (running.length) this.saveAll(running); return running.length; } page({ cursor = null, limit = 50, filters = {} } = {}) { if (!/^\d+$/.test(String(limit)) || !integer(Number(limit), 1)) throw invalid('请求账本每页数量需要正整数。'); const filtered = where(filters), scope = sha(JSON.stringify({ ledger: this.state.id, filters: Object.fromEntries(Object.entries(filters).sort()) })); const current = this.database.prepare('SELECT COALESCE(MAX(seq),0) AS seq,COUNT(*) AS total FROM model_attempts').get(); let snapshot = current.seq, before = null; if (cursor !== null) { let value; try { if (typeof cursor !== 'string' || !/^[A-Za-z0-9_-]+$/.test(cursor)) throw new Error(); value = JSON.parse(Buffer.from(cursor, 'base64url').toString()); } catch { throw invalid('请求账本游标无效。'); } if (value?.schemaVersion !== 1 || value.scope !== scope || !integer(value.snapshot) || !integer(value.before, 1) || value.before > value.snapshot || value.snapshot > current.seq) throw invalid('请求账本游标不属于当前筛选或来源。'); snapshot = value.snapshot; before = value.before; if (!this.database.prepare('SELECT seq FROM model_attempts WHERE seq=? AND seq<=?' + filtered.sql).get(before, snapshot, ...filtered.parameters)) throw invalid('请求账本原游标已变化,请刷新记录。', 409); } const count = this.database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total; const rows = this.database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + (before === null ? '' : ' AND seq Number(limit); return { schemaVersion: 1, items: shown.map(row => decode(row)), references: shown.map(reference), total: count, retainedTotal: current.total, snapshotSequence: snapshot, connectionIds: this.database.prepare('SELECT DISTINCT connection_id FROM model_attempts WHERE connection_id IS NOT NULL ORDER BY connection_id').all().map(row => row.connection_id), hasMore, nextCursor: hasMore ? Buffer.from(JSON.stringify({ schemaVersion: 1, scope, snapshot, before: shown.at(-1).seq })).toString('base64url') : null, filters, source: this.state, viewSnapshot: 'sequence-watermark; running record fields can still update' }; } record(sequence) { if (!/^\d+$/.test(String(sequence)) || !integer(Number(sequence), 1)) throw invalid('请求账本序号无效。'); const row = this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(Number(sequence)); if (!row) throw invalid('请求账本原记录不存在。', 404); if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409); return { item: decode(row), reference: reference(row), initial: { item: JSON.parse(row.initial_payload), sha256: row.initial_sha256 } }; } exportSnapshot(filters = {}) { const filtered = where(filters), database = new DatabaseSync(this.storage.filename, { readOnly: true }); try { database.exec('PRAGMA query_only=ON; PRAGMA busy_timeout=5000; BEGIN;'); const snapshot = database.prepare('SELECT COALESCE(MAX(seq),0) AS seq FROM model_attempts').get().seq; const count = database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total; const rows = database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + filtered.sql + ' ORDER BY seq').iterate(snapshot, ...filtered.parameters); return { header: { schemaVersion: 1, type: 'header', format: 'orven-model-attempts-ndjson', snapshotSequence: snapshot, expectedRecords: count, filters, source: this.state, recordedAt: new Date().toISOString(), snapshot: 'SQLite read transaction', includesInitialReceipts: true, artifactBodies: 'Original artifacts remain separately downloadable by their saved session/ID/SHA.' }, rows, close() { try { rows.return?.(); database.exec('ROLLBACK'); } finally { database.close(); } } }; } catch (cause) { database.close(); throw cause; } } } export async function streamLedgerExport(response, snapshot) { const checksum = createHash('sha256'); let count = 0; const write = async line => { if (response.destroyed || response.writableEnded) throw invalid('请求账本导出连接已关闭。', 499); // ServerResponse can report backpressure after its socket has already // flushed, without emitting another drain. The chunk callback reports // actual write completion and bounds queued data to this one record. await new Promise((resolve, reject) => { let settled = false; const clear = () => { response.off('close', closed); response.off('error', failed); }; const finish = error => { if (settled) return; settled = true; clear(); error ? reject(error) : resolve(); }; const closed = () => finish(invalid('请求账本导出被中断,部分内容不能当作完整导出。', 499)), failed = error => finish(error); response.once('close', closed); response.once('error', failed); try { response.write(line, finish); } catch (cause) { finish(cause); } }); }; try { await write(JSON.stringify(snapshot.header) + '\n'); for (const row of snapshot.rows) { decode(row); if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409); const line = '{"type":"attempt","reference":' + JSON.stringify(reference(row)) + ',"item":' + row.payload + ',"initial":{"sha256":' + JSON.stringify(row.initial_sha256) + ',"item":' + row.initial_payload + '}}\n'; checksum.update(line); await write(line); count++; } if (count !== snapshot.header.expectedRecords) throw invalid('请求账本导出数量与快照不匹配。', 409); await write(JSON.stringify({ type: 'footer', complete: true, records: count, recordLinesSha256: checksum.digest('hex') }) + '\n'); response.end(); } finally { snapshot.close(); } }