Files
Agent-Workbench/scripts/audit-live-retest-h.mjs
2026-11-03 09:49:07 +08:00

72 lines
6.4 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
const root = path.resolve('docs/live-original-purge-h-evidence');
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
const read = relative => JSON.parse(fs.readFileSync(path.join(root, relative)));
const manifest = read('manifest.json');
for (const entry of manifest.entries) {
const raw = fs.readFileSync(path.join(root, entry.path)); assert.equal(raw.length, entry.bytes); assert.equal(sha(raw), entry.sha256);
}
const core = {};
for (const row of read('receipts/owned-records.json')) (core[row.kind] ??= new Map()).set(row.id, JSON.parse(row.payload));
const artifacts = new Map(manifest.artifacts.map(a => [a.id, a]));
function artifact(id) { const a = artifacts.get(id); assert.ok(a, id); return JSON.parse(fs.readFileSync(path.join(root, a.file))); }
const attempts = read('receipts/all-model-attempts.json');
function strings(value, output = [], depth = 0) {
if (depth > 12) return output;
if (typeof value === 'string') {
output.push(value);
for (const text of new Set([value, ...value.split('\n').filter(line => /^\s*[\[{]/.test(line))])) {
if (/^\s*[\[{]/.test(text)) { try { strings(JSON.parse(text), output, depth + 1); } catch {} }
}
} else if (value && typeof value === 'object') for (const v of Object.values(value)) strings(v, output, depth + 1);
return output;
}
function covered(ranges,start,end){if(end<=start)return false;let cursor=start;for(const r of ranges.slice().sort((a,b)=>a.start-b.start)){if(r.end<=cursor)continue;if(r.start>cursor)return false;cursor=Math.max(cursor,r.end);if(cursor>=end)return true;}return false;}
assert.equal(covered([{start:0,end:4},{start:4,end:8}],2,6),true);
assert.equal(covered([{start:0,end:2},{start:4,end:8}],1,5),false);
assert.equal(covered([{start:0,end:8}],3,3),false);
const reports = [];
for (const sample of manifest.samples.slice(0, 2)) {
const exported=read('owned-exports/'+sample.sessionId+'.json');
const applied=exported.session.changes.filter(c=>c.status==='applied'&&c.appliedAt);
const currentSource=(r,at)=>{const c=applied.filter(c=>c.path===r.path&&c.appliedAt<at).sort((a,b)=>a.appliedAt.localeCompare(b.appliedAt)).at(-1);return !c||c.sha256===r.sourceSha256;};
const tools = [...core.tool.values()].filter(t => t.sessionId === sample.sessionId)
.map(t => ({ ...t, result: t.result ?? (t.resultArtifactId ? artifact(t.resultArtifactId) : null) }))
.sort((a, b) => a.startedAt.localeCompare(b.startedAt));
const reads = tools.filter(t => t.tool === 'read_file' && t.result?.content).map(t => ({ path: t.result.path, start: t.result.start_line, end: t.result.end_line,
complete: !t.result.truncated && t.result.start_line === 1 && t.result.end_line === t.result.total_lines,
content: t.result.content, sourceSha256:t.result.sha256, toolId: t.id, at: t.finishedAt }));
const ranges = new Map();
const pages = tools.filter(t => t.tool === 'read_artifact').map(t => {
const r = t.result, id = r?.artifact_id, a = artifacts.get(id);
if (!id) return { toolId: t.id, state: t.state, resultKeys: Object.keys(r ?? {}) };
const prior = ranges.get(id) ?? [], start = r.offset, end = r.next_offset;
const alreadyCovered = covered(prior,start,end);
const overlap = prior.some(p => p.start < end && start < p.end);
prior.push({ start, end }); ranges.set(id, prior);
let source;
try { const value = JSON.parse(r.content); source = { path: value.path, tool: value.tool, keys: Object.keys(value).slice(0, 10) }; } catch {}
return { toolId: t.id, at: t.finishedAt, artifactId: id, kind: a?.kind, sha256: a?.sha256, offset: start, nextOffset: end,
total: r.total_characters, truncated: r.truncated, alreadyCovered, overlap, source };
});
const requestReports = attempts.filter(a => a.sessionId === sample.sessionId).sort((a, b) => a.startedAt.localeCompare(b.startedAt)).map((a, i) => {
const request = artifact(a.requestArtifactId), bodyStrings = strings(request.messages ?? request.input ?? request);
const before = reads.filter(r => r.at < a.startedAt);
return { request: i + 1, attemptId: a.id, reportedTokens: a.usage?.total_tokens, startedAt: a.startedAt,
completeSource: before.filter(r => r.complete).map(r => ({ path: r.path, toolId: r.toolId, sourceSha256:r.sourceSha256, currentAtDispatch:currentSource(r,a.startedAt), retained: bodyStrings.some(s => s.includes(r.content)) })),
toolsBefore: tools.filter(t => t.finishedAt < a.startedAt).length };
});
const prepared = tools.findIndex(t => t.tool === 'prepare_change_set' && t.state === 'completed');
reports.push({ name: sample.name, sessionId: sample.sessionId, tools: tools.map(t => ({ toolId: t.id, name: t.tool, state: t.state, at: t.finishedAt,
resultKeys: Object.keys(t.result ?? {}), ...(t.tool === 'prepare_change_set' ? { result: t.result } : {}) })),
reads: reads.map(({ content, ...r }) => ({ ...r, characters: content.length })), pages, requests: requestReports,
afterPrepared: prepared < 0 ? [] : tools.slice(prepared + 1).map(t => ({ name: t.tool, at: t.finishedAt, state: t.state })) });
}
const result = { success: true, manifestSha256: sha(fs.readFileSync(path.join(root, 'manifest.json'))), realModelRequests: 0, auditVersion:2, originalAuditSha256:sha(fs.readFileSync('docs/live-retest-h-efficiency-audit.json')), corrections:['Repeated artifact page coverage uses the union of all prior returned character ranges.','Source reads invalidated by an applied change are explicitly marked stale; their absence is not a lost-current-source finding.'],
scope: 'Offline observations of sealed new H samples only. Exact source retention checks inspect nested JSON tool content; repeated artifact pages are measured by original byte-identity and returned character interval. No source changes or inferred model cognition.', reports };
fs.writeFileSync('docs/live-retest-h-efficiency-audit-v2.json', JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
for(const r of reports)console.log(JSON.stringify({name:r.name,toolCalls:r.tools.length,completeReads:r.reads.filter(x=>x.complete).length,repeatedPages:r.pages.filter(p=>p.alreadyCovered&&p.nextOffset>p.offset).length,requests:r.requests.map(q=>({request:q.request,completeRead:q.completeSource.length,currentSource:q.completeSource.filter(f=>f.currentAtDispatch).length,currentRetained:q.completeSource.filter(f=>f.currentAtDispatch&&f.retained).length})),afterPrepared:r.afterPrepared}));