72 lines
6.4 KiB
JavaScript
72 lines
6.4 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
const root = path.resolve('docs/live-original-purge-h-evidence');
|
|
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
const read = relative => JSON.parse(fs.readFileSync(path.join(root, relative)));
|
|
const manifest = read('manifest.json');
|
|
for (const entry of manifest.entries) {
|
|
const raw = fs.readFileSync(path.join(root, entry.path)); assert.equal(raw.length, entry.bytes); assert.equal(sha(raw), entry.sha256);
|
|
}
|
|
const core = {};
|
|
for (const row of read('receipts/owned-records.json')) (core[row.kind] ??= new Map()).set(row.id, JSON.parse(row.payload));
|
|
const artifacts = new Map(manifest.artifacts.map(a => [a.id, a]));
|
|
function artifact(id) { const a = artifacts.get(id); assert.ok(a, id); return JSON.parse(fs.readFileSync(path.join(root, a.file))); }
|
|
const attempts = read('receipts/all-model-attempts.json');
|
|
function strings(value, output = [], depth = 0) {
|
|
if (depth > 12) return output;
|
|
if (typeof value === 'string') {
|
|
output.push(value);
|
|
for (const text of new Set([value, ...value.split('\n').filter(line => /^\s*[\[{]/.test(line))])) {
|
|
if (/^\s*[\[{]/.test(text)) { try { strings(JSON.parse(text), output, depth + 1); } catch {} }
|
|
}
|
|
} else if (value && typeof value === 'object') for (const v of Object.values(value)) strings(v, output, depth + 1);
|
|
return output;
|
|
}
|
|
function covered(ranges,start,end){if(end<=start)return false;let cursor=start;for(const r of ranges.slice().sort((a,b)=>a.start-b.start)){if(r.end<=cursor)continue;if(r.start>cursor)return false;cursor=Math.max(cursor,r.end);if(cursor>=end)return true;}return false;}
|
|
assert.equal(covered([{start:0,end:4},{start:4,end:8}],2,6),true);
|
|
assert.equal(covered([{start:0,end:2},{start:4,end:8}],1,5),false);
|
|
assert.equal(covered([{start:0,end:8}],3,3),false);
|
|
const reports = [];
|
|
for (const sample of manifest.samples.slice(0, 2)) {
|
|
const exported=read('owned-exports/'+sample.sessionId+'.json');
|
|
const applied=exported.session.changes.filter(c=>c.status==='applied'&&c.appliedAt);
|
|
const currentSource=(r,at)=>{const c=applied.filter(c=>c.path===r.path&&c.appliedAt<at).sort((a,b)=>a.appliedAt.localeCompare(b.appliedAt)).at(-1);return !c||c.sha256===r.sourceSha256;};
|
|
const tools = [...core.tool.values()].filter(t => t.sessionId === sample.sessionId)
|
|
.map(t => ({ ...t, result: t.result ?? (t.resultArtifactId ? artifact(t.resultArtifactId) : null) }))
|
|
.sort((a, b) => a.startedAt.localeCompare(b.startedAt));
|
|
const reads = tools.filter(t => t.tool === 'read_file' && t.result?.content).map(t => ({ path: t.result.path, start: t.result.start_line, end: t.result.end_line,
|
|
complete: !t.result.truncated && t.result.start_line === 1 && t.result.end_line === t.result.total_lines,
|
|
content: t.result.content, sourceSha256:t.result.sha256, toolId: t.id, at: t.finishedAt }));
|
|
const ranges = new Map();
|
|
const pages = tools.filter(t => t.tool === 'read_artifact').map(t => {
|
|
const r = t.result, id = r?.artifact_id, a = artifacts.get(id);
|
|
if (!id) return { toolId: t.id, state: t.state, resultKeys: Object.keys(r ?? {}) };
|
|
const prior = ranges.get(id) ?? [], start = r.offset, end = r.next_offset;
|
|
const alreadyCovered = covered(prior,start,end);
|
|
const overlap = prior.some(p => p.start < end && start < p.end);
|
|
prior.push({ start, end }); ranges.set(id, prior);
|
|
let source;
|
|
try { const value = JSON.parse(r.content); source = { path: value.path, tool: value.tool, keys: Object.keys(value).slice(0, 10) }; } catch {}
|
|
return { toolId: t.id, at: t.finishedAt, artifactId: id, kind: a?.kind, sha256: a?.sha256, offset: start, nextOffset: end,
|
|
total: r.total_characters, truncated: r.truncated, alreadyCovered, overlap, source };
|
|
});
|
|
const requestReports = attempts.filter(a => a.sessionId === sample.sessionId).sort((a, b) => a.startedAt.localeCompare(b.startedAt)).map((a, i) => {
|
|
const request = artifact(a.requestArtifactId), bodyStrings = strings(request.messages ?? request.input ?? request);
|
|
const before = reads.filter(r => r.at < a.startedAt);
|
|
return { request: i + 1, attemptId: a.id, reportedTokens: a.usage?.total_tokens, startedAt: a.startedAt,
|
|
completeSource: before.filter(r => r.complete).map(r => ({ path: r.path, toolId: r.toolId, sourceSha256:r.sourceSha256, currentAtDispatch:currentSource(r,a.startedAt), retained: bodyStrings.some(s => s.includes(r.content)) })),
|
|
toolsBefore: tools.filter(t => t.finishedAt < a.startedAt).length };
|
|
});
|
|
const prepared = tools.findIndex(t => t.tool === 'prepare_change_set' && t.state === 'completed');
|
|
reports.push({ name: sample.name, sessionId: sample.sessionId, tools: tools.map(t => ({ toolId: t.id, name: t.tool, state: t.state, at: t.finishedAt,
|
|
resultKeys: Object.keys(t.result ?? {}), ...(t.tool === 'prepare_change_set' ? { result: t.result } : {}) })),
|
|
reads: reads.map(({ content, ...r }) => ({ ...r, characters: content.length })), pages, requests: requestReports,
|
|
afterPrepared: prepared < 0 ? [] : tools.slice(prepared + 1).map(t => ({ name: t.tool, at: t.finishedAt, state: t.state })) });
|
|
}
|
|
const result = { success: true, manifestSha256: sha(fs.readFileSync(path.join(root, 'manifest.json'))), realModelRequests: 0, auditVersion:2, originalAuditSha256:sha(fs.readFileSync('docs/live-retest-h-efficiency-audit.json')), corrections:['Repeated artifact page coverage uses the union of all prior returned character ranges.','Source reads invalidated by an applied change are explicitly marked stale; their absence is not a lost-current-source finding.'],
|
|
scope: 'Offline observations of sealed new H samples only. Exact source retention checks inspect nested JSON tool content; repeated artifact pages are measured by original byte-identity and returned character interval. No source changes or inferred model cognition.', reports };
|
|
fs.writeFileSync('docs/live-retest-h-efficiency-audit-v2.json', JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
|
|
for(const r of reports)console.log(JSON.stringify({name:r.name,toolCalls:r.tools.length,completeReads:r.reads.filter(x=>x.complete).length,repeatedPages:r.pages.filter(p=>p.alreadyCovered&&p.nextOffset>p.offset).length,requests:r.requests.map(q=>({request:q.request,completeRead:q.completeSource.length,currentSource:q.completeSource.filter(f=>f.currentAtDispatch).length,currentRetained:q.completeSource.filter(f=>f.currentAtDispatch&&f.retained).length})),afterPrepared:r.afterPrepared}));
|