97 lines
9.3 KiB
JavaScript
97 lines
9.3 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { root, read, save, sha } from './common.mjs';
|
|
|
|
const stage = process.argv[2], auditPrefix = process.argv[3];
|
|
assert.ok(['node', 'react', 'python', 'final'].includes(stage));
|
|
assert.match(auditPrefix ?? '', new RegExp('^audits/' + stage + '-[a-f0-9-]{36}$'));
|
|
const audit = read(auditPrefix + '/result.json'); assert.equal(audit.success, true);
|
|
const source = relative => {
|
|
const bytes = fs.readFileSync(path.join(root, relative));
|
|
return { path: relative, bytes: bytes.length, sha256: sha(bytes) };
|
|
};
|
|
const compatibility = read('compatibility/result.json');
|
|
const compatibilitySummary = {
|
|
requests: compatibility.requests, unknownRequests: compatibility.results.flatMap(value => value.preflights).filter(value => value.usageStatus !== 'reported').length,
|
|
reportedTotalTokens: compatibility.results.flatMap(value => value.preflights).reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0),
|
|
source: source('compatibility/result.json'),
|
|
connections: compatibility.results.map(value => ({ connectionId: value.connectionId, name: value.name, model: value.model, protocol: value.protocol,
|
|
endpointKind: value.endpointKind, network: value.network, catalog: value.catalog, engineeringAllowed: value.engineeringAllowed,
|
|
sources: ['network.json', 'catalog.json', 'result.json'].filter(file => fs.existsSync(path.join(root, 'compatibility', value.connectionId, file)))
|
|
.map(file => source('compatibility/' + value.connectionId + '/' + file)),
|
|
preflights: value.preflights.map(row => ({ scope: row.scope, passed: row.passed, sessionId: row.sessionId, taskId: row.taskId, runId: row.runId,
|
|
identity: row.identity, status: row.status, usageStatus: row.usageStatus, usage: row.usage, actualToolsExecuted: row.actualToolsExecuted,
|
|
streaming: row.streaming, wirePassed: row.wirePassed, outputFields: row.outputFields, expectedOutput: row.expectedOutput,
|
|
sources: row.captures.map(capture => ({ ...capture, ...source(capture.path) })) })) })),
|
|
};
|
|
const engineering = [];
|
|
for (const kind of ['node', 'react', 'python']) {
|
|
if (!fs.existsSync(path.join(root, kind, 'result.json'))) continue;
|
|
const result = read(kind + '/result.json');
|
|
if (!result.executed) { engineering.push({ kind, result, source: source(kind + '/result.json') }); continue; }
|
|
const session = read(kind + '/session.json'), attempts = read(kind + '/attempts.json');
|
|
const metrics = {};
|
|
for (const key of ['input', 'output', 'total', 'cacheRead', 'cacheWrite', 'uncached', 'reasoning']) {
|
|
const values = attempts.map(value => value.usage?.tokenAccounting?.metrics?.[key]);
|
|
const known = values.filter(value => Number.isSafeInteger(value?.tokens) && ['reported', 'derived'].includes(value.status));
|
|
metrics[key] = { knownSumTokens: known.length ? known.reduce((sum, value) => sum + value.tokens, 0) : null,
|
|
knownRequests: known.length, unknownRequests: values.length - known.length,
|
|
statuses: [...new Set(values.map(value => value?.status ?? 'unknown'))] };
|
|
}
|
|
const reads = [], seenReads = new Set();
|
|
for (const event of session.events.filter(value => value.kind === 'tool' && value.phase === 'completed' && value.tool === 'read_file')) {
|
|
let output; try { output = JSON.parse(event.output); } catch { output = null; }
|
|
const row = { eventId: event.id, path: output?.path ?? event.input?.path, startLine: output?.start_line ?? null, endLine: output?.end_line ?? null,
|
|
fileSha256: output?.sha256 ?? null, outputSha256: sha(event.output ?? ''), parsed: output !== null };
|
|
const key = JSON.stringify([row.path, row.startLine, row.endLine, row.fileSha256 ?? row.outputSha256]);
|
|
row.sameByteRangePreviouslyRead = seenReads.has(key); seenReads.add(key); reads.push(row);
|
|
}
|
|
const completedTools = session.events.filter(value => value.kind === 'tool' && value.phase === 'completed');
|
|
const wire = attempts.map(value => {
|
|
const capture = result.captures.find(item => item.attemptId === value.id && item.kind === 'request');
|
|
assert.ok(capture); const body = read(capture.path);
|
|
return { attemptId: value.id, model: body.model, streaming: body.stream === true,
|
|
outputFields: Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])),
|
|
nativeReasoning: Object.fromEntries(['thinking', 'reasoning', 'reasoning_effort'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])),
|
|
toolsCount: body.tools?.length ?? 0, toolsSchemaSha256: sha(JSON.stringify(body.tools ?? [])), source: source(capture.path) };
|
|
});
|
|
const reports = result.reportArtifacts.filter(value => value.kind === 'delivery-report').map(value => {
|
|
const file = result.captures.find(capture => capture.artifactId === value.id).path, report = read(file);
|
|
return { artifactId: value.id, status: report.status, taskStatus: report.task?.status, lifecycle: report.lifecycle,
|
|
deliveries: report.deliveries?.map(item => ({ id: item.id, status: item.status, commit: item.commit })), source: source(file) };
|
|
});
|
|
engineering.push({ kind, success: result.success, executed: result.executed, projectId: result.projectId, sessionId: result.sessionId,
|
|
taskId: result.taskId, runId: result.runId, connectionId: result.connectionId, requests: result.requests, usage: result.usage,
|
|
startedAt: session.execution.run.startedAt, finishedAt: session.execution.run.finishedAt,
|
|
elapsedMs: Date.parse(session.execution.run.finishedAt) - Date.parse(session.execution.run.startedAt),
|
|
stopReason: result.productStopReason, failure: result.failure, identitiesPassed: result.identitiesPassed,
|
|
commit: result.commit, initialHead: result.initialHead, committedPaths: result.committedPaths, committedCurrent: result.committedCurrent,
|
|
otherUntracked: result.otherUntracked, final: result.final, browser: result.browser,
|
|
ownerBytesAndIndexPreserved: result.ownerBytesAndIndexPreserved, supervisorContinuations: result.supervisorContinuations,
|
|
localTokenCapsChanged: result.localTokenCapsChanged, internalDeliveries: result.internalDeliveries, reports,
|
|
accounting: metrics, reads: { calls: reads.length, distinctByteRanges: seenReads.size, repeatedSameByteRanges: reads.filter(value => value.sameByteRangePreviouslyRead).length, observations: reads },
|
|
tools: Object.fromEntries([...new Set(completedTools.map(value => value.tool))].map(tool => [tool, completedTools.filter(value => value.tool === tool).length])),
|
|
wire, toolsSchemaVariants: [...new Set(wire.map(value => value.toolsSchemaSha256))],
|
|
sources: ['claim.json', 'goal.txt', 'source-before.json', 'result.json', 'session.json', 'export.json', 'attempts.json', 'restoration.json', 'git-status.txt', 'git-log.txt']
|
|
.map(file => source(kind + '/' + file)), captures: result.captures.map(value => ({ ...value, ...source(value.path) })) });
|
|
}
|
|
const auditHistory = fs.readdirSync(path.join(root, 'audits')).filter(name => fs.existsSync(path.join(root, 'audits', name, 'result.json')))
|
|
.map(name => { const file = 'audits/' + name + '/result.json', value = read(file); return { stage: value.stage, success: value.success, error: value.error ?? null, source: source(file) }; });
|
|
const summary = { schemaVersion: 1, stage, generatedAt: new Date().toISOString(), evidenceRoot: path.relative(process.cwd(), root).replaceAll('\\', '/'),
|
|
frozen: source('frozen.json'), compatibility: compatibilitySummary, engineering,
|
|
requests: compatibilitySummary.requests + engineering.reduce((sum, value) => sum + (value.requests ?? 0), 0),
|
|
reportedTotalTokens: compatibilitySummary.reportedTotalTokens + engineering.reduce((sum, value) => sum + (value.usage?.totalTokens ?? 0), 0),
|
|
unknownRequests: compatibilitySummary.unknownRequests + engineering.reduce((sum, value) => sum + (value.usage?.unknownRequests ?? 0), 0),
|
|
audit: { ...audit, source: source(auditPrefix + '/result.json'), after: source(auditPrefix + '/after.json') }, auditHistory,
|
|
drivers: fs.readdirSync(path.resolve('scripts/m4-live')).filter(name => name.endsWith('.mjs')).sort().map(name => { const file = 'scripts/m4-live/' + name, bytes = fs.readFileSync(file); return { path: file, bytes: bytes.length, sha256: sha(bytes) }; }),
|
|
keysReadOrResaved: false, historicalSamplesReplayed: false, productSourceChanged: false, releaseOrRestart: false, purgeExecuted: false,
|
|
unverified: ['Real Claude identity on the configured proxy', 'Long-term provider reliability', 'Real Mnemo production binding and semantic benefit',
|
|
'Physical iPhone', 'General repeated-read improvement', 'R/W v2 live exclusion boundary', 'Causal cache benefit', 'Purge benefit'] };
|
|
save(stage + '-summary.json', summary);
|
|
const name = stage === 'node' ? 'compatibility-node' : stage;
|
|
const file = path.resolve('docs/m4-' + name + '-2026-11-11.json');
|
|
fs.writeFileSync(file, JSON.stringify(summary, null, 2) + '\n', { flag: 'wx' });
|
|
console.log(JSON.stringify({ file, stage, requests: summary.requests, reportedTotalTokens: summary.reportedTotalTokens, unknownRequests: summary.unknownRequests,
|
|
engineering: engineering.map(value => ({ kind: value.kind, success: value.success, requests: value.requests, elapsedMs: value.elapsedMs, cacheRead: value.accounting?.cacheRead, reads: value.reads?.calls, repeated: value.reads?.repeatedSameByteRanges })), audit: audit.success }));
|