184 lines
15 KiB
JavaScript
184 lines
15 KiB
JavaScript
import { randomUUID, createHash } from 'node:crypto';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { REQUEST_LEDGER_SQL } from '../storage/request-ledger-schema.mjs';
|
|
|
|
const metadataKey = 'modelAttemptLedger';
|
|
const sha = value => createHash('sha256').update(value).digest('hex');
|
|
const invalid = (message, status = 400) => Object.assign(new Error(message), { status });
|
|
const fields = { connectionId: 'connection_id', sessionId: 'session_id', taskId: 'task_id', runId: 'run_id', status: 'status', model: 'model', protocol: 'protocol' };
|
|
const ownership = value => JSON.stringify([value.id, value.connectionId ?? null, value.poolId ?? null, value.projectId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null, value.requestId ?? null, value.model ?? null, value.protocol ?? null]);
|
|
const integer = (value, minimum = 0) => Number.isSafeInteger(value) && value >= minimum;
|
|
const reference = row => ({ sequence: row.seq, payloadSha256: row.payload_sha256, initialSha256: row.initial_sha256,
|
|
origin: row.legacy_position === null ? 'incremental-ledger' : 'legacy-metadata', ...(row.legacy_position === null ? {} : { legacyIndex: row.legacy_position }) });
|
|
|
|
export function ledgerQuery(url, { exportOnly = false } = {}) {
|
|
const allowed = [...Object.keys(fields), 'query', ...(exportOnly ? [] : ['cursor', 'limit'])], params = url.searchParams;
|
|
for (const key of params.keys()) if (!allowed.includes(key) || params.getAll(key).length !== 1) throw invalid('请求账本查询参数无效。');
|
|
return { ...(exportOnly ? {} : { cursor: params.get('cursor'), limit: params.get('limit') ?? 50 }),
|
|
filters: Object.fromEntries([...Object.keys(fields), 'query'].filter(key => params.has(key) && params.get(key) !== '').map(key => [key, params.get(key)])) };
|
|
}
|
|
|
|
function where(filters = {}) {
|
|
const terms = [], parameters = [];
|
|
for (const key of Object.keys(filters).sort()) {
|
|
const value = filters[key]; if (typeof value !== 'string' || !value || ![...Object.keys(fields), 'query'].includes(key)) throw invalid('请求账本筛选字段无效。');
|
|
if (key === 'query') { terms.push("payload LIKE ? ESCAPE '\\'"); parameters.push('%' + value.replace(/[\\%_]/g, '\\$&') + '%'); }
|
|
else { terms.push(fields[key] + '=?'); parameters.push(value); }
|
|
}
|
|
return { sql: terms.length ? ' AND ' + terms.join(' AND ') : '', parameters };
|
|
}
|
|
function decode(row) {
|
|
if (sha(row.payload) !== row.payload_sha256) throw invalid('本机请求账本原记录 SHA256 不匹配,请核对保存来源。', 409);
|
|
return JSON.parse(row.payload);
|
|
}
|
|
|
|
// The legacy metadata stays byte-for-byte in its original row. A one-time
|
|
// transaction copies its entries into an indexed table. New requests update
|
|
// only their own row; the all-records accessor is compatibility/read-only use.
|
|
export class ModelRequestLedger {
|
|
constructor(storage) {
|
|
this.storage = storage; this.database = storage.database; this.positions = new WeakMap(); this.serialized = new WeakMap();
|
|
const originalVersion = this.database.prepare('PRAGMA user_version').get().user_version;
|
|
const state = storage.meta(metadataKey);
|
|
if (state && state.schemaVersion !== 1) throw invalid('请求账本由未接入的新版本创建,请使用对应程序。', 409);
|
|
if (!state) {
|
|
const original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value;
|
|
const entries = original ? JSON.parse(original) : [];
|
|
if (!Array.isArray(entries) || entries.some(value => !value || typeof value !== 'object' || Array.isArray(value) || typeof value.id !== 'string' || !value.id)) throw invalid('原请求账本格式无法完整迁移,原 metadata 已保留。', 409);
|
|
this.state = { schemaVersion: 1, id: randomUUID(), legacyEntries: entries.length, legacyMetadataSha256: original ? sha(original) : null,
|
|
originalMetadataPreserved: true, migratedAt: new Date().toISOString() };
|
|
storage.transaction(() => {
|
|
this.database.exec(REQUEST_LEDGER_SQL);
|
|
for (const [index, entry] of entries.entries()) this.insert(entry, JSON.stringify(entry), index);
|
|
storage.setMeta(metadataKey, this.state);
|
|
this.database.exec('PRAGMA user_version=2');
|
|
});
|
|
this.initialization = { originalVersion };
|
|
} else {
|
|
this.state = state;
|
|
if (originalVersion < 2) storage.transaction(() => this.database.exec('PRAGMA user_version=2'));
|
|
}
|
|
}
|
|
insert(value, serialized, legacyIndex = null) {
|
|
const result = this.database.prepare(`INSERT INTO model_attempts(id,connection_id,session_id,task_id,run_id,status,model,protocol,legacy_position,payload,payload_sha256,initial_payload,initial_sha256)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(value.id, value.connectionId ?? null, value.sessionId ?? null, value.taskId ?? null, value.runId ?? null,
|
|
value.status ?? null, value.model ?? null, value.protocol ?? null, legacyIndex, serialized, sha(serialized), serialized, sha(serialized));
|
|
return Number(result.lastInsertRowid);
|
|
}
|
|
attach(row) { const value = decode(row); this.positions.set(value, row.seq); this.serialized.set(value, row.payload); return value; }
|
|
all() { return this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all().map(row => this.attach(row)); }
|
|
find(id) { const rows = this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(id); if (rows.length > 1) throw invalid('请求账本原身份不唯一,请按序号核对。', 409); return rows[0] ? decode(rows[0]) : undefined; }
|
|
rollbackInitialization() {
|
|
if (!this.initialization) return false;
|
|
// A failed synchronous handler adoption has not dispatched any request.
|
|
// Revert only this verified migration, never a ledger with new outcomes.
|
|
this.storage.transaction(() => {
|
|
const state = this.storage.meta(metadataKey), original = this.database.prepare("SELECT value FROM metadata WHERE key='modelAttempts'").get()?.value;
|
|
const entries = original ? JSON.parse(original) : [], rows = this.database.prepare('SELECT * FROM model_attempts ORDER BY seq').all();
|
|
if (state?.id !== this.state.id || (original ? sha(original) : null) !== this.state.legacyMetadataSha256 || rows.length !== entries.length
|
|
|| rows.some((row, index) => row.legacy_position !== index || row.payload !== row.initial_payload || row.initial_payload !== JSON.stringify(entries[index])
|
|
|| sha(row.payload) !== row.payload_sha256 || sha(row.initial_payload) !== row.initial_sha256)) throw invalid('请求账本已有新记录或来源变化,不能回退原迁移,请保留现场核对。', 409);
|
|
this.database.exec('DROP TABLE model_attempts; PRAGMA user_version=' + this.initialization.originalVersion);
|
|
this.database.prepare('DELETE FROM metadata WHERE key=?').run(metadataKey);
|
|
});
|
|
this.initialization = null; return true;
|
|
}
|
|
save(value) { return this.saveAll([value])[0]; }
|
|
saveAll(values) {
|
|
if (values.every(value => value && this.serialized.get(value) === JSON.stringify(value))) return values.map(value => this.positions.get(value));
|
|
const updates = [];
|
|
this.storage.transaction(() => {
|
|
for (const value of values) {
|
|
if (!value || typeof value !== 'object' || typeof value.id !== 'string' || !value.id) throw invalid('请求账本记录需要原尝试身份。');
|
|
const serialized = JSON.stringify(value); if (this.serialized.get(value) === serialized) continue;
|
|
const sequence = this.positions.get(value), matches = sequence ? [this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(sequence)]
|
|
: this.database.prepare('SELECT * FROM model_attempts WHERE id=?').all(value.id);
|
|
if (matches.length > 1 || sequence && !matches[0]) throw invalid('请求账本记录身份不唯一或原行已变化,请核对来源。', 409);
|
|
let savedSequence;
|
|
if (matches[0]) {
|
|
const original = decode(matches[0]);
|
|
if (this.serialized.has(value) && this.serialized.get(value) !== matches[0].payload) throw invalid('请求账本原记录已更新,请核对保存来源,不能覆盖另一份进展。', 409);
|
|
if (ownership(original) !== ownership(value)) throw invalid('请求账本记录不能移动到其他连接、型号、协议、会话或任务。', 409);
|
|
this.database.prepare('UPDATE model_attempts SET status=?,payload=?,payload_sha256=? WHERE seq=?').run(value.status ?? null, serialized, sha(serialized), matches[0].seq);
|
|
savedSequence = matches[0].seq;
|
|
} else savedSequence = this.insert(value, serialized);
|
|
updates.push({ value, serialized, sequence: savedSequence });
|
|
}
|
|
});
|
|
for (const update of updates) { this.positions.set(update.value, update.sequence); this.serialized.set(update.value, update.serialized); }
|
|
return values.map(value => this.positions.get(value));
|
|
}
|
|
recoverRunning() {
|
|
const running = this.database.prepare("SELECT * FROM model_attempts WHERE status='running' ORDER BY seq").all().map(row => this.attach(row));
|
|
for (const value of running) { value.status = 'unknown'; value.finishedAt = new Date().toISOString(); value.reason = '宿主重启,请求结果与用量待核对。'; }
|
|
if (running.length) this.saveAll(running); return running.length;
|
|
}
|
|
page({ cursor = null, limit = 50, filters = {} } = {}) {
|
|
if (!/^\d+$/.test(String(limit)) || !integer(Number(limit), 1)) throw invalid('请求账本每页数量需要正整数。');
|
|
const filtered = where(filters), scope = sha(JSON.stringify({ ledger: this.state.id, filters: Object.fromEntries(Object.entries(filters).sort()) }));
|
|
const current = this.database.prepare('SELECT COALESCE(MAX(seq),0) AS seq,COUNT(*) AS total FROM model_attempts').get();
|
|
let snapshot = current.seq, before = null;
|
|
if (cursor !== null) {
|
|
let value; try { if (typeof cursor !== 'string' || !/^[A-Za-z0-9_-]+$/.test(cursor)) throw new Error(); value = JSON.parse(Buffer.from(cursor, 'base64url').toString()); } catch { throw invalid('请求账本游标无效。'); }
|
|
if (value?.schemaVersion !== 1 || value.scope !== scope || !integer(value.snapshot) || !integer(value.before, 1) || value.before > value.snapshot || value.snapshot > current.seq) throw invalid('请求账本游标不属于当前筛选或来源。');
|
|
snapshot = value.snapshot; before = value.before;
|
|
if (!this.database.prepare('SELECT seq FROM model_attempts WHERE seq=? AND seq<=?' + filtered.sql).get(before, snapshot, ...filtered.parameters)) throw invalid('请求账本原游标已变化,请刷新记录。', 409);
|
|
}
|
|
const count = this.database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total;
|
|
const rows = this.database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + (before === null ? '' : ' AND seq<?') + filtered.sql + ' ORDER BY seq DESC LIMIT ?')
|
|
.all(snapshot, ...(before === null ? [] : [before]), ...filtered.parameters, Number(limit) < Number.MAX_SAFE_INTEGER ? Number(limit) + 1 : Number(limit));
|
|
const shown = rows.slice(0, Number(limit)), hasMore = rows.length > Number(limit);
|
|
return { schemaVersion: 1, items: shown.map(row => decode(row)), references: shown.map(reference), total: count, retainedTotal: current.total, snapshotSequence: snapshot,
|
|
connectionIds: this.database.prepare('SELECT DISTINCT connection_id FROM model_attempts WHERE connection_id IS NOT NULL ORDER BY connection_id').all().map(row => row.connection_id),
|
|
hasMore, nextCursor: hasMore ? Buffer.from(JSON.stringify({ schemaVersion: 1, scope, snapshot, before: shown.at(-1).seq })).toString('base64url') : null,
|
|
filters, source: this.state, viewSnapshot: 'sequence-watermark; running record fields can still update' };
|
|
}
|
|
record(sequence) {
|
|
if (!/^\d+$/.test(String(sequence)) || !integer(Number(sequence), 1)) throw invalid('请求账本序号无效。');
|
|
const row = this.database.prepare('SELECT * FROM model_attempts WHERE seq=?').get(Number(sequence)); if (!row) throw invalid('请求账本原记录不存在。', 404);
|
|
if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409);
|
|
return { item: decode(row), reference: reference(row), initial: { item: JSON.parse(row.initial_payload), sha256: row.initial_sha256 } };
|
|
}
|
|
exportSnapshot(filters = {}) {
|
|
const filtered = where(filters), database = new DatabaseSync(this.storage.filename, { readOnly: true });
|
|
try {
|
|
database.exec('PRAGMA query_only=ON; PRAGMA busy_timeout=5000; BEGIN;');
|
|
const snapshot = database.prepare('SELECT COALESCE(MAX(seq),0) AS seq FROM model_attempts').get().seq;
|
|
const count = database.prepare('SELECT COUNT(*) AS total FROM model_attempts WHERE seq<=?' + filtered.sql).get(snapshot, ...filtered.parameters).total;
|
|
const rows = database.prepare('SELECT * FROM model_attempts WHERE seq<=?' + filtered.sql + ' ORDER BY seq').iterate(snapshot, ...filtered.parameters);
|
|
return { header: { schemaVersion: 1, type: 'header', format: 'orven-model-attempts-ndjson', snapshotSequence: snapshot, expectedRecords: count,
|
|
filters, source: this.state, recordedAt: new Date().toISOString(), snapshot: 'SQLite read transaction', includesInitialReceipts: true, artifactBodies: 'Original artifacts remain separately downloadable by their saved session/ID/SHA.' }, rows,
|
|
close() { try { rows.return?.(); database.exec('ROLLBACK'); } finally { database.close(); } } };
|
|
} catch (cause) { database.close(); throw cause; }
|
|
}
|
|
}
|
|
|
|
export async function streamLedgerExport(response, snapshot) {
|
|
const checksum = createHash('sha256'); let count = 0;
|
|
const write = async line => {
|
|
if (response.destroyed || response.writableEnded) throw invalid('请求账本导出连接已关闭。', 499);
|
|
// ServerResponse can report backpressure after its socket has already
|
|
// flushed, without emitting another drain. The chunk callback reports
|
|
// actual write completion and bounds queued data to this one record.
|
|
await new Promise((resolve, reject) => {
|
|
let settled = false;
|
|
const clear = () => { response.off('close', closed); response.off('error', failed); };
|
|
const finish = error => { if (settled) return; settled = true; clear(); error ? reject(error) : resolve(); };
|
|
const closed = () => finish(invalid('请求账本导出被中断,部分内容不能当作完整导出。', 499)), failed = error => finish(error);
|
|
response.once('close', closed); response.once('error', failed);
|
|
try { response.write(line, finish); } catch (cause) { finish(cause); }
|
|
});
|
|
};
|
|
try {
|
|
await write(JSON.stringify(snapshot.header) + '\n');
|
|
for (const row of snapshot.rows) {
|
|
decode(row); if (sha(row.initial_payload) !== row.initial_sha256) throw invalid('请求账本初始来源 SHA256 不匹配。', 409);
|
|
const line = '{"type":"attempt","reference":' + JSON.stringify(reference(row)) + ',"item":' + row.payload
|
|
+ ',"initial":{"sha256":' + JSON.stringify(row.initial_sha256) + ',"item":' + row.initial_payload + '}}\n';
|
|
checksum.update(line); await write(line); count++;
|
|
}
|
|
if (count !== snapshot.header.expectedRecords) throw invalid('请求账本导出数量与快照不匹配。', 409);
|
|
await write(JSON.stringify({ type: 'footer', complete: true, records: count, recordLinesSha256: checksum.digest('hex') }) + '\n'); response.end();
|
|
} finally { snapshot.close(); }
|
|
}
|