import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { purgeEvidence, root, read, save, sha } from '../live-ad/common.mjs'; purgeEvidence(); const auditPath = process.argv[2]; assert.ok(auditPath, 'Explicit completed audit path is required.'); const audit = read(auditPath); assert.equal(audit.success, true); const frozen = read('current.json'), result = read('result.json'), samples = [], allAttempts = [], tools = new Set(); for (const sample of result.samples) { if (!sample.workerSessionId) { samples.push({ ...sample, requests: 0, usage: { requests: 0, inputTokens: 0, outputTokens: 0, totalTokens: 0, unknownRequests: 0, estimatedUnknownTokens: 0 }, dispatchStage: 'before-worker-and-real-model', originalPurge: [] }); continue; } const prefix = sample.sample, state = read(prefix + '/worker/session.json'), attempts = read(prefix + '/worker/attempts.json'); const jobs = read(prefix + '/worker/engine-jobs.json'), proof = read(prefix + '/worker/engine-proof.json'); assert.equal(attempts.length, state.execution.task.usage.requests); for (const attempt of attempts) { allAttempts.push(attempt); for (const type of ['request', 'response']) { if (!attempt[type + 'ArtifactId']) continue; const names = fs.readdirSync(path.join(root, prefix, 'worker/captures')).filter(name => name.startsWith(attempt.id + '-' + type)); assert.equal(names.length, 1); const bytes = fs.readFileSync(path.join(root, prefix, 'worker/captures', names[0])); assert.equal(sha(bytes), attempt[type + 'Sha256']); if (type === 'request') { const body = JSON.parse(bytes); assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key))); const request = proof.find(row => row.requestId === attempt.requestId); assert.ok(request); assert.ok(request.variants.some(row => row.bodySha256 === sha(bytes))); const toolSha256 = sha(JSON.stringify(body.tools)); tools.add(toolSha256); assert.ok(request.variants.some(row => row.toolSha256 === toolSha256)); } } } const reads = state.events.filter(row => row.tool === 'read_file' && row.phase === 'completed').map(row => row.input); const turns = state.execution.run.progressShadow?.turns ?? []; samples.push({ ...sample, captures: undefined, engineRequestProof: undefined, reads, requestTimings: attempts.map(row => ({ id: row.id, startedAt: row.startedAt, finishedAt: row.finishedAt, elapsedMs: row.finishedAt ? Date.parse(row.finishedAt) - Date.parse(row.startedAt) : null, firstTokenLatencyMs: null, firstTokenLatencyNote: 'The attempt ledger does not record this metric; no estimate substituted.' })), toolCalls: state.events.filter(row => row.kind === 'tool' && row.phase === 'completed').map(row => ({ tool: row.tool, startedAt: row.startedAt ?? null, finishedAt: row.finishedAt ?? null, durationMs: row.durationMs ?? null })), shadowRepeatedReads: turns.filter(row => row.phase === 'settled').reduce((sum, row) => sum + (row.delta?.repeated_reads ?? 0), 0), originalPurge: jobs.map(job => ({ id: job.id, kind: job.installation.strategy?.kind ?? null, sourceSha256: job.installation.strategy?.source.sourceSha256 ?? null, entryLoaded: job.native?.originalPurge?.readiness.originalEntryLoaded ?? false, applyCalled: job.native?.originalPurge?.readiness.originalApplyCalled ?? false, loadedModules: job.native?.originalPurge?.loadedModules?.length ?? 0, nativeLoads: job.native?.originalPurge?.nativeLoads?.length ?? 0, rootExited: job.native?.cleanup?.rootExited ?? false, treeConfirmed: job.native?.cleanup?.treeConfirmed ?? false })) }); } const fourSuccessful = samples.length === 4 && samples.every(row => row.success), sameTools = tools.size === 1; const metric = (enabled, key) => samples.filter(row => row.enabled === enabled).reduce((sum, row) => sum + (key === 'tokens' ? row.usage?.totalTokens ?? 0 : row[key] ?? 0), 0); const comparison = fourSuccessful && sameTools ? { eligible: true, off: { tokens: metric(false, 'tokens'), requests: metric(false, 'requests'), engineElapsedMs: metric(false, 'engineElapsedMs') }, on: { tokens: metric(true, 'tokens'), requests: metric(true, 'requests'), engineElapsedMs: metric(true, 'engineElapsedMs') }, sampleSize: 2, interpretation: 'Descriptive totals for two order-balanced pairs only; no universal or causal benefit and no billing claim.' } : { eligible: false, reason: 'Four equal-success samples with stable tools are required; early failure is not token savings.' }; const summary = { schema: 1, version: '1.25.8', driverCommit: frozen.driverCommit, samples, requests: allAttempts.length, reportedRequests: allAttempts.filter(row => row.usageStatus === 'reported').length, reportedTokens: allAttempts.reduce((sum, row) => sum + (row.usage?.total_tokens ?? 0), 0), unknownRequests: allAttempts.filter(row => row.usageStatus === 'unknown').length, unknownLocalEstimate: samples.reduce((sum, row) => sum + (row.usage?.estimatedUnknownTokens ?? 0), 0), actualToolSha256: [...tools], sameTools, comparison, originalStateAudit: auditPath, originalStateAuditResult: audit, keysReadOrResaved: false, historicalSamplesReplayed: false, priorFailedExperimentsRemainUnchanged: true }; save('summary.json', summary); const files = []; function walk(directory) { for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { const file = path.join(directory, entry.name); assert.equal(entry.isSymbolicLink(), false); if (entry.isDirectory()) walk(file); else { assert.equal(entry.isFile(), true); const value = fs.readFileSync(file); files.push({ path: path.relative(root, file).replaceAll('\\', '/'), bytes: value.length, sha256: sha(value) }); } } } walk(root); files.sort((a, b) => a.path.localeCompare(b.path)); const bytes = files.reduce((sum, row) => sum + row.bytes, 0); save('FINAL-MANIFEST.json', { schema: 1, files, bytes, scope: 'All ' + (frozen.batch ?? 'af').toUpperCase() + ' raw requests/responses, original engine inputs/reports, source/Git, checks and audits. Historical evidence is not repackaged.' }); for (const row of files) { const value = fs.readFileSync(path.join(root, row.path)); assert.equal(value.length, row.bytes); assert.equal(sha(value), row.sha256); } if (frozen.fixture !== true) fs.writeFileSync('docs/live-purge-' + (frozen.batch ?? 'af') + '-evidence.json', JSON.stringify({ summary, evidence: { root, files: files.length, bytes, manifestSha256: sha(fs.readFileSync(path.join(root, 'FINAL-MANIFEST.json'))) } }, null, 2) + '\n', { flag: 'wx' }); console.log(JSON.stringify({ success: true, realRequests: summary.requests, reportedTokens: summary.reportedTokens, unknown: summary.unknownRequests, allFourSuccessful: fourSuccessful, sameTools, comparison, files: files.length, bytes }));