79 lines
6.8 KiB
JavaScript
79 lines
6.8 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { purgeEvidence, root, read, save, sha } from '../live-ad/common.mjs';
|
|
|
|
purgeEvidence();
|
|
const auditPath = process.argv[2]; assert.ok(auditPath, 'Explicit completed audit path is required.');
|
|
const audit = read(auditPath); assert.equal(audit.success, true);
|
|
const frozen = read('current.json'), result = read('result.json'), samples = [], allAttempts = [], tools = new Set();
|
|
for (const sample of result.samples) {
|
|
if (!sample.workerSessionId) { samples.push({ ...sample, requests: 0, usage: { requests: 0, inputTokens: 0, outputTokens: 0, totalTokens: 0,
|
|
unknownRequests: 0, estimatedUnknownTokens: 0 }, dispatchStage: 'before-worker-and-real-model', originalPurge: [] }); continue; }
|
|
const prefix = sample.sample, state = read(prefix + '/worker/session.json'), attempts = read(prefix + '/worker/attempts.json');
|
|
const jobs = read(prefix + '/worker/engine-jobs.json'), proof = read(prefix + '/worker/engine-proof.json');
|
|
assert.equal(attempts.length, state.execution.task.usage.requests);
|
|
for (const attempt of attempts) {
|
|
allAttempts.push(attempt);
|
|
for (const type of ['request', 'response']) {
|
|
if (!attempt[type + 'ArtifactId']) continue;
|
|
const names = fs.readdirSync(path.join(root, prefix, 'worker/captures')).filter(name => name.startsWith(attempt.id + '-' + type));
|
|
assert.equal(names.length, 1);
|
|
const bytes = fs.readFileSync(path.join(root, prefix, 'worker/captures', names[0])); assert.equal(sha(bytes), attempt[type + 'Sha256']);
|
|
if (type === 'request') {
|
|
const body = JSON.parse(bytes);
|
|
assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
|
|
const request = proof.find(row => row.requestId === attempt.requestId); assert.ok(request);
|
|
assert.ok(request.variants.some(row => row.bodySha256 === sha(bytes)));
|
|
const toolSha256 = sha(JSON.stringify(body.tools)); tools.add(toolSha256);
|
|
assert.ok(request.variants.some(row => row.toolSha256 === toolSha256));
|
|
}
|
|
}
|
|
}
|
|
const reads = state.events.filter(row => row.tool === 'read_file' && row.phase === 'completed').map(row => row.input);
|
|
const turns = state.execution.run.progressShadow?.turns ?? [];
|
|
samples.push({ ...sample, captures: undefined, engineRequestProof: undefined, reads,
|
|
requestTimings: attempts.map(row => ({ id: row.id, startedAt: row.startedAt, finishedAt: row.finishedAt,
|
|
elapsedMs: row.finishedAt ? Date.parse(row.finishedAt) - Date.parse(row.startedAt) : null,
|
|
firstTokenLatencyMs: null, firstTokenLatencyNote: 'The attempt ledger does not record this metric; no estimate substituted.' })),
|
|
toolCalls: state.events.filter(row => row.kind === 'tool' && row.phase === 'completed').map(row => ({ tool: row.tool,
|
|
startedAt: row.startedAt ?? null, finishedAt: row.finishedAt ?? null, durationMs: row.durationMs ?? null })),
|
|
shadowRepeatedReads: turns.filter(row => row.phase === 'settled').reduce((sum, row) => sum + (row.delta?.repeated_reads ?? 0), 0),
|
|
originalPurge: jobs.map(job => ({ id: job.id, kind: job.installation.strategy?.kind ?? null,
|
|
sourceSha256: job.installation.strategy?.source.sourceSha256 ?? null,
|
|
entryLoaded: job.native?.originalPurge?.readiness.originalEntryLoaded ?? false,
|
|
applyCalled: job.native?.originalPurge?.readiness.originalApplyCalled ?? false,
|
|
loadedModules: job.native?.originalPurge?.loadedModules?.length ?? 0,
|
|
nativeLoads: job.native?.originalPurge?.nativeLoads?.length ?? 0,
|
|
rootExited: job.native?.cleanup?.rootExited ?? false, treeConfirmed: job.native?.cleanup?.treeConfirmed ?? false })) });
|
|
}
|
|
const fourSuccessful = samples.length === 4 && samples.every(row => row.success), sameTools = tools.size === 1;
|
|
const metric = (enabled, key) => samples.filter(row => row.enabled === enabled).reduce((sum, row) => sum + (key === 'tokens' ? row.usage?.totalTokens ?? 0 : row[key] ?? 0), 0);
|
|
const comparison = fourSuccessful && sameTools ? { eligible: true, off: { tokens: metric(false, 'tokens'), requests: metric(false, 'requests'), engineElapsedMs: metric(false, 'engineElapsedMs') },
|
|
on: { tokens: metric(true, 'tokens'), requests: metric(true, 'requests'), engineElapsedMs: metric(true, 'engineElapsedMs') },
|
|
sampleSize: 2, interpretation: 'Descriptive totals for two order-balanced pairs only; no universal or causal benefit and no billing claim.' }
|
|
: { eligible: false, reason: 'Four equal-success samples with stable tools are required; early failure is not token savings.' };
|
|
const summary = { schema: 1, version: '1.25.8', driverCommit: frozen.driverCommit, samples,
|
|
requests: allAttempts.length, reportedRequests: allAttempts.filter(row => row.usageStatus === 'reported').length,
|
|
reportedTokens: allAttempts.reduce((sum, row) => sum + (row.usage?.total_tokens ?? 0), 0),
|
|
unknownRequests: allAttempts.filter(row => row.usageStatus === 'unknown').length,
|
|
unknownLocalEstimate: samples.reduce((sum, row) => sum + (row.usage?.estimatedUnknownTokens ?? 0), 0),
|
|
actualToolSha256: [...tools], sameTools, comparison, originalStateAudit: auditPath, originalStateAuditResult: audit,
|
|
keysReadOrResaved: false, historicalSamplesReplayed: false, priorFailedExperimentsRemainUnchanged: true };
|
|
save('summary.json', summary);
|
|
const files = [];
|
|
function walk(directory) {
|
|
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
const file = path.join(directory, entry.name); assert.equal(entry.isSymbolicLink(), false);
|
|
if (entry.isDirectory()) walk(file);
|
|
else { assert.equal(entry.isFile(), true); const value = fs.readFileSync(file); files.push({ path: path.relative(root, file).replaceAll('\\', '/'), bytes: value.length, sha256: sha(value) }); }
|
|
}
|
|
}
|
|
walk(root); files.sort((a, b) => a.path.localeCompare(b.path)); const bytes = files.reduce((sum, row) => sum + row.bytes, 0);
|
|
save('FINAL-MANIFEST.json', { schema: 1, files, bytes, scope: 'All ' + (frozen.batch ?? 'af').toUpperCase() + ' raw requests/responses, original engine inputs/reports, source/Git, checks and audits. Historical evidence is not repackaged.' });
|
|
for (const row of files) { const value = fs.readFileSync(path.join(root, row.path)); assert.equal(value.length, row.bytes); assert.equal(sha(value), row.sha256); }
|
|
if (frozen.fixture !== true) fs.writeFileSync('docs/live-purge-' + (frozen.batch ?? 'af') + '-evidence.json', JSON.stringify({ summary, evidence: { root, files: files.length, bytes,
|
|
manifestSha256: sha(fs.readFileSync(path.join(root, 'FINAL-MANIFEST.json'))) } }, null, 2) + '\n', { flag: 'wx' });
|
|
console.log(JSON.stringify({ success: true, realRequests: summary.requests, reportedTokens: summary.reportedTokens, unknown: summary.unknownRequests,
|
|
allFourSuccessful: fourSuccessful, sameTools, comparison, files: files.length, bytes }));
|