test(m4): verify live protocols and autonomous Node delivery
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { publicSnapshot, digest } from '../retained-host-state.mjs';
|
||||
import { root, origin, read, save, client, sha, compareConfig, compareConnectionDefinitions } from './common.mjs';
|
||||
import { inspectorClosed } from './inspector.mjs';
|
||||
|
||||
const stage = process.argv[2]; assert.ok(['node', 'react', 'python', 'final'].includes(stage));
|
||||
const prefix = 'audits/' + stage + '-' + randomUUID(), frozen = read('frozen.json'), compatibility = read('compatibility/result.json');
|
||||
const engineering = ['node', 'react', 'python'].filter(kind => fs.existsSync(path.join(root, kind, 'result.json'))).map(kind => ({ kind, ...read(kind + '/result.json') }));
|
||||
const sessionIds = new Set(compatibility.results.flatMap(value => value.preflights.map(row => row.sessionId)).concat(engineering.map(value => value.sessionId).filter(Boolean)));
|
||||
const projectIds = new Set(compatibility.results.map(value => value.projectId).concat(engineering.map(value => value.projectId).filter(Boolean)));
|
||||
const external = fs.existsSync(path.join(root, 'external-observations/declared.json')) ? read('external-observations/declared.json').sources.map(source => {
|
||||
assert.match(source.path, /^external-observations\/[a-f0-9-]{36}\/index\.json$/);
|
||||
assert.equal(sha(fs.readFileSync(path.join(root, source.path))), source.sha256);
|
||||
const value = read(source.path); assert.equal(value.m4Engineering, false); assert.equal(value.requestsReplayed, false);
|
||||
assert.equal(sessionIds.has(value.sessionId), false); assert.equal(frozen.baseline.database.sessionIds.includes(value.sessionId), false);
|
||||
assert.ok(frozen.originalProjects.some(project => project.id === value.projectId)); return value;
|
||||
}) : [];
|
||||
const allSessionIds = new Set([...sessionIds, ...external.map(value => value.sessionId)]);
|
||||
assert.equal(allSessionIds.size, sessionIds.size + external.length, 'No duplicate declared external sessions.');
|
||||
const sessionProjects = new Map(compatibility.results.flatMap(value => value.preflights.map(row => [row.sessionId, value.projectId]))
|
||||
.concat(engineering.filter(value => value.sessionId).map(value => [value.sessionId, value.projectId]), external.map(value => [value.sessionId, value.projectId])));
|
||||
const diagnosticIds = new Set(compatibility.results.map(value => read('compatibility/' + value.connectionId + '/network-start.json').job.id));
|
||||
const expectedRequests = compatibility.requests + engineering.reduce((sum, value) => sum + (value.requests ?? 0), 0);
|
||||
const externalRequests = external.reduce((sum, value) => sum + value.requests, 0), databaseRequests = expectedRequests + externalRequests;
|
||||
const { api, boot, close } = await client(); let result;
|
||||
try {
|
||||
await inspectorClosed(); const host = await api('/host/status'); assert.equal(host.activity.active, false);
|
||||
assert.equal(host.identity.pid, frozen.host.identity.pid); assert.equal(host.identity.instanceId, frozen.host.identity.instanceId); assert.equal(boot.version, '1.25.13');
|
||||
const after = await publicSnapshot(origin, host.identity.dataRoot); save(prefix + '/after.json', after); const before = frozen.baseline;
|
||||
assert.deepEqual(new Set(boot.projects.filter(value => !frozen.originalProjects.some(old => old.id === value.id)).map(value => value.id)), projectIds);
|
||||
assert.deepEqual(boot.projects.filter(value => !projectIds.has(value.id)), frozen.originalProjects);
|
||||
assert.deepEqual(after.database.sessionIds.filter(id => !before.database.sessionIds.includes(id)).sort(), [...allSessionIds].sort());
|
||||
const configComparison = compareConfig(frozen.config, boot.config), configDelta = boot.config.configRevision - frozen.config.configRevision;
|
||||
assert.equal(configDelta, engineering.filter(value => value.executed).length * 2, 'Exactly one select and restore per new engineering sample.');
|
||||
const definitions = compareConnectionDefinitions(frozen.center, after.modelCenter, frozen.config, boot.config);
|
||||
assert.deepEqual(after.modelCenter.selected, frozen.center.selected); assert.deepEqual(after.modelCenter.pools, frozen.center.pools);
|
||||
const exportMap = new Map(after.exports.map(value => [value.id, value])), artifactMap = new Map(after.persistent.artifacts.map(value => [value.path, value]));
|
||||
for (const original of before.exports) assert.deepEqual(exportMap.get(original.id), original, 'Original session export: ' + original.id);
|
||||
for (const original of before.persistent.artifacts) assert.deepEqual(artifactMap.get(original.path), original, 'Original artifact byte: ' + original.path);
|
||||
for (const value of external) {
|
||||
assert.deepEqual(exportMap.get(value.sessionId), value.exports, 'Explicit external session export: ' + value.sessionId);
|
||||
for (const artifact of value.artifacts) assert.deepEqual(artifactMap.get(artifact.path), { path: artifact.path, bytes: artifact.bytes, sha256: artifact.sha256 }, 'Explicit external artifact byte: ' + artifact.path);
|
||||
}
|
||||
const db = new DatabaseSync(path.join(host.identity.dataRoot, 'workbench.sqlite'), { readOnly: true }); const addedRows = {}, metadataChanges = [], sequenceChanges = []; let checkedOriginalRows = 0;
|
||||
try {
|
||||
assert.deepEqual(Object.keys(after.database.tables), Object.keys(before.database.tables));
|
||||
for (const [table, originalTable] of Object.entries(before.database.tables)) {
|
||||
const currentTable = after.database.tables[table]; assert.equal(currentTable.schemaSha256, originalTable.schemaSha256); addedRows[table] = currentTable.count - originalTable.count;
|
||||
const originals = new Set(originalTable.rows.map(value => (value.kind ?? '') + ':' + value.id)), currentRows = new Map(currentTable.rows.map(value => [(value.kind ?? '') + ':' + value.id, value]));
|
||||
for (const value of external) if (value.tables[table]) {
|
||||
assert.equal(value.tables[table].schemaSha256, originalTable.schemaSha256);
|
||||
for (const row of value.tables[table].rows) {
|
||||
assert.equal(originals.has((row.kind ?? '') + ':' + row.id), false, 'External observation cannot replace an original row.');
|
||||
assert.deepEqual(currentRows.get((row.kind ?? '') + ':' + row.id), row, 'Every explicitly recorded external row: ' + table + ':' + row.id);
|
||||
}
|
||||
}
|
||||
for (const original of originalTable.rows) {
|
||||
if (table === 'projects') {
|
||||
const current = db.prepare('SELECT * FROM projects WHERE id=?').get(original.id); assert.ok(current);
|
||||
assert.equal(digest({ ...current, position: current.position - projectIds.size }), original.sha256, 'Original project fields and declared front insertion positions.');
|
||||
} else if (table === 'sqlite_sequence') {
|
||||
const sequences = db.prepare('SELECT * FROM sqlite_sequence').all();
|
||||
const current = sequences.find(value => digest({ ...value, seq: original.id }) === original.sha256); assert.ok(current, 'Original sequence authority.');
|
||||
const increment = current.name === 'events' ? after.eventCursor - before.eventCursor : current.name === 'model_attempts' ? databaseRequests : NaN;
|
||||
assert.ok(Number.isSafeInteger(increment)); assert.equal(current.seq, original.id + increment);
|
||||
sequenceChanges.push({ name: current.name, original: original.id, current: current.seq, increment });
|
||||
} else if (table === 'metadata' && ['config', 'modelRegistry', 'modelNetworkDiagnostics'].includes(original.id)) {
|
||||
const current = db.prepare('SELECT * FROM metadata WHERE key=?').get(original.id), value = JSON.parse(current.value); let normalized;
|
||||
if (original.id === 'config') { assert.equal(value.configRevision, boot.config.configRevision); normalized = { ...value, configRevision: frozen.config.configRevision }; }
|
||||
else if (original.id === 'modelRegistry') {
|
||||
assert.equal(value.revision, frozen.center.revision + configDelta); assert.deepEqual(value.selected, frozen.center.selected);
|
||||
normalized = { ...value, revision: frozen.center.revision };
|
||||
} else {
|
||||
assert.ok(Array.isArray(value)); const additions = value.filter(row => diagnosticIds.has(row.id)); assert.equal(additions.length, diagnosticIds.size);
|
||||
normalized = value.filter(row => !diagnosticIds.has(row.id));
|
||||
}
|
||||
assert.equal(digest({ ...current, value: JSON.stringify(normalized) }), original.sha256, 'Original public metadata bytes after the declared increment: ' + original.id);
|
||||
metadataChanges.push({ key: original.id, originalRowSha256: original.sha256, currentRowSha256: digest(current), originalRestoredByDeclaredIncrement: true });
|
||||
} else {
|
||||
const current = currentRows.get((original.kind ?? '') + ':' + original.id); assert.ok(current, table + ': missing original ' + original.id);
|
||||
assert.equal(current.sha256, original.sha256, table + ': changed original ' + original.id);
|
||||
}
|
||||
checkedOriginalRows++;
|
||||
}
|
||||
if (['events', 'records', 'sessions'].includes(table)) {
|
||||
const columns = table === 'records' ? 'id,kind,session_id,project_id' : table === 'events' ? 'id,session_id,project_id' : 'id,project_id';
|
||||
for (const row of db.prepare('SELECT ' + columns + ' FROM ' + table).all()) if (!originals.has((row.kind ?? '') + ':' + row.id)) {
|
||||
const sessionId = table === 'sessions' ? row.id : row.session_id;
|
||||
assert.ok(allSessionIds.has(sessionId), 'Every new ' + table + ' needs a declared session.'); assert.equal(row.project_id, sessionProjects.get(sessionId));
|
||||
const observation = external.find(value => value.sessionId === sessionId);
|
||||
if (observation) assert.ok(observation.tables[table].rows.some(value => value.id === row.id && (value.kind ?? '') === (row.kind ?? '')), 'External rows must match their explicit immutable list.');
|
||||
}
|
||||
} else if (table === 'outbox') {
|
||||
for (const row of db.prepare('SELECT outbox.seq,events.session_id FROM outbox JOIN events USING(seq) WHERE outbox.seq>?').all(before.eventCursor)) {
|
||||
assert.ok(allSessionIds.has(row.session_id));
|
||||
const observation = external.find(value => value.sessionId === row.session_id);
|
||||
if (observation) assert.ok(observation.tables.outbox.rows.some(value => value.id === row.seq), 'Explicit external outbox sequence.');
|
||||
}
|
||||
} else if (table === 'model_attempts') {
|
||||
assert.equal(addedRows[table], databaseRequests);
|
||||
// The ledger stores session_id; its project authority is the existing
|
||||
// sessions relation, not a nonexistent model_attempts.project_id field.
|
||||
for (const row of db.prepare('SELECT model_attempts.id,model_attempts.session_id,sessions.project_id FROM model_attempts LEFT JOIN sessions ON sessions.id=model_attempts.session_id').all()) if (!originals.has(':' + row.id)) {
|
||||
assert.ok(allSessionIds.has(row.session_id)); assert.equal(row.project_id, sessionProjects.get(row.session_id));
|
||||
const observation = external.find(value => value.sessionId === row.session_id);
|
||||
if (observation) assert.ok(observation.tables.model_attempts.rows.some(value => value.id === row.id), 'Explicit external request ID.');
|
||||
}
|
||||
} else if (['metadata', 'sqlite_sequence'].includes(table)) assert.equal(addedRows[table], 0);
|
||||
}
|
||||
const references = new Map(db.prepare("SELECT payload FROM records WHERE kind='artifact'").all().map(row => { const value = JSON.parse(row.payload); return [value.relative, value]; }));
|
||||
const originals = new Set(before.persistent.artifacts.map(value => value.path));
|
||||
for (const artifact of after.persistent.artifacts) if (!originals.has(artifact.path)) {
|
||||
const reference = references.get(artifact.path); assert.ok(reference && allSessionIds.has(reference.sessionId) && reference.projectId === sessionProjects.get(reference.sessionId), 'New original artifact must be owned or explicitly recorded externally.');
|
||||
assert.equal(reference.sha256, artifact.sha256); assert.equal(reference.bytes, artifact.bytes);
|
||||
}
|
||||
} finally { db.close(); }
|
||||
for (const original of before.persistent.files) {
|
||||
const current = after.persistent.files.find(value => value.path === original.path); assert.ok(current);
|
||||
if (original.path === 'config.json' && current.sha256 !== original.sha256) {
|
||||
const value = JSON.parse(fs.readFileSync(path.join(host.identity.dataRoot, original.path), 'utf8'));
|
||||
assert.equal(sha(JSON.stringify({ ...value, configRevision: frozen.config.configRevision }, null, 2)), original.sha256, 'Original config file bytes after the exact config revision increment.');
|
||||
} else assert.equal(current.sha256, original.sha256, 'Original persistent file: ' + original.path);
|
||||
}
|
||||
result = { success: true, stage, version: boot.version, pid: host.identity.pid, instanceId: host.identity.instanceId, originalProjects: frozen.originalProjects.length,
|
||||
originalSessions: before.database.sessionIds.length, originalModelRequests: before.database.tables.model_attempts.count, originalArtifacts: before.persistent.artifacts.length,
|
||||
addedProjects: projectIds.size, addedSessions: sessionIds.size, addedRows, newArtifacts: after.persistent.artifacts.length - before.persistent.artifacts.length, checkedOriginalRows,
|
||||
originalSessionExports: 'exact', originalArtifactBytes: 'exact', originalRows: 'exact with explicitly hashed project-position/public-metadata/sequence increments',
|
||||
configComparison, definitions, metadataChanges, sequenceChanges, batchModelRequests: expectedRequests, externalModelRequests: externalRequests,
|
||||
externalObservations: external.map(value => ({ sessionId: value.sessionId, projectId: value.projectId, requests: value.requests, tables: Object.fromEntries(Object.entries(value.tables).map(([table, source]) => [table, source.count])), artifacts: value.artifacts.length, scope: value.scope })),
|
||||
originalCredentialRow: 'exact SHA; never decrypted or copied',
|
||||
engineering: engineering.map(value => ({ kind: value.kind, success: value.success, executed: value.executed })), inspectorClosed: true, keysReadOrResaved: false, historicalSamplesReplayed: false, purgeExecuted: false };
|
||||
} catch (error) { result = { success: false, stage, error: error.stack, keysReadOrResaved: false, noMutationOrRetry: true }; process.exitCode = 1; }
|
||||
finally { save(prefix + '/result.json', result); await close(); }
|
||||
console.log(JSON.stringify({ prefix, ...result }));
|
||||
@@ -0,0 +1,50 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { digest } from '../retained-host-state.mjs';
|
||||
import { root, read, save, client, sha, captureSession } from './common.mjs';
|
||||
|
||||
// Explicitly record the independently discovered greeting; do not automatically
|
||||
// accept every new session, repeat its request, or add it to engineering results.
|
||||
const sessionId = '8f896f90-50f2-4d07-8da0-3b98832c6df5';
|
||||
const attemptId = '821c6dc9-7e52-4e41-a1d9-36b4de3ecbdb';
|
||||
const prefix = 'external-observations/' + sessionId, frozen = read('frozen.json');
|
||||
assert.equal(fs.existsSync(path.join(root, prefix)), false, 'Do not overwrite an external observation.');
|
||||
assert.equal(frozen.baseline.database.sessionIds.includes(sessionId), false);
|
||||
const { api, close } = await client();
|
||||
try {
|
||||
const host = await api('/host/status');
|
||||
assert.equal(host.identity.pid, frozen.host.identity.pid); assert.equal(host.activity.active, false);
|
||||
const captured = await captureSession(api, sessionId, prefix), state = captured.state;
|
||||
assert.ok(frozen.originalProjects.some(value => value.id === state.projectId));
|
||||
assert.equal(state.status, 'completed'); assert.equal(state.execution.run.resumeOf, null);
|
||||
assert.deepEqual(state.messages.filter(value => value.role === 'user').map(value => value.content), ['你好']);
|
||||
assert.equal(state.changes.length, 0); assert.equal(state.execution.deliveries.length, 0);
|
||||
assert.equal(state.events.filter(value => value.kind === 'tool').length, 0);
|
||||
assert.deepEqual(captured.rows.map(value => value.id), [attemptId]);
|
||||
const db = new DatabaseSync(path.join(host.identity.dataRoot, 'workbench.sqlite'), { readOnly: true }), tables = {};
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
for (const table of ['sessions', 'events', 'records', 'model_attempts', 'outbox']) {
|
||||
const rows = table === 'outbox'
|
||||
? db.prepare('SELECT outbox.* FROM outbox JOIN events USING(seq) WHERE events.session_id=? ORDER BY outbox.seq').all(sessionId)
|
||||
: db.prepare('SELECT * FROM ' + table + ' WHERE ' + (table === 'sessions' ? 'id' : 'session_id') + '=? ORDER BY rowid').all(sessionId);
|
||||
const sql = db.prepare("SELECT sql FROM sqlite_schema WHERE type='table' AND name=?").get(table).sql;
|
||||
assert.equal(sha(sql), frozen.baseline.database.tables[table].schemaSha256);
|
||||
tables[table] = { count: rows.length, schemaSha256: sha(sql), rows: rows.map(row => ({ id: row.id ?? row.seq, ...(row.kind ? { kind: row.kind } : {}), sha256: digest(row) })) };
|
||||
}
|
||||
} finally { db.exec('ROLLBACK'); db.close(); }
|
||||
const index = { schemaVersion: 1, observedAt: new Date().toISOString(), sessionId, projectId: state.projectId,
|
||||
taskId: state.execution.task.id, runId: state.execution.run.id, title: state.title, scope: 'Independent new greeting in an existing project; attribution to a person is not asserted.',
|
||||
m4Engineering: false, requests: captured.total, usage: state.execution.task.usage,
|
||||
attempts: captured.rows.map(value => ({ id: value.id, status: value.status, usageStatus: value.usageStatus, modelIdentity: value.modelIdentity, requestSha256: value.requestSha256, responseSha256: value.responseSha256 })),
|
||||
exports: { id: sessionId, project: digest(captured.exported.project), session: digest(captured.exported.session), execution: digest(captured.exported.execution) },
|
||||
artifacts: state.execution.artifacts.map(value => ({ id: value.id, path: value.relative, bytes: value.bytes, sha256: value.sha256 })),
|
||||
tables, captures: captured.captures, resumedHistoricalTask: false, toolCalls: 0, changes: 0, deliveries: 0,
|
||||
keysReadOrResaved: false, requestsReplayed: false, productOrProjectMutation: false };
|
||||
const relative = prefix + '/index.json'; save(relative, index);
|
||||
save('external-observations/declared.json', { schemaVersion: 1, sources: [{ path: relative, sha256: sha(fs.readFileSync(path.join(root, relative))) }] });
|
||||
console.log(JSON.stringify({ sessionId, projectId: state.projectId, requests: index.requests, totalTokens: index.usage.totalTokens,
|
||||
tableCounts: Object.fromEntries(Object.entries(tables).map(([key, value]) => [key, value.count])), artifacts: index.artifacts.length, modelIdentity: index.attempts[0].modelIdentity.status, scope: index.scope }));
|
||||
} finally { await close(); }
|
||||
@@ -0,0 +1,96 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { root, read, save, sha } from './common.mjs';
|
||||
|
||||
const stage = process.argv[2], auditPrefix = process.argv[3];
|
||||
assert.ok(['node', 'react', 'python', 'final'].includes(stage));
|
||||
assert.match(auditPrefix ?? '', new RegExp('^audits/' + stage + '-[a-f0-9-]{36}$'));
|
||||
const audit = read(auditPrefix + '/result.json'); assert.equal(audit.success, true);
|
||||
const source = relative => {
|
||||
const bytes = fs.readFileSync(path.join(root, relative));
|
||||
return { path: relative, bytes: bytes.length, sha256: sha(bytes) };
|
||||
};
|
||||
const compatibility = read('compatibility/result.json');
|
||||
const compatibilitySummary = {
|
||||
requests: compatibility.requests, unknownRequests: compatibility.results.flatMap(value => value.preflights).filter(value => value.usageStatus !== 'reported').length,
|
||||
reportedTotalTokens: compatibility.results.flatMap(value => value.preflights).reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0),
|
||||
source: source('compatibility/result.json'),
|
||||
connections: compatibility.results.map(value => ({ connectionId: value.connectionId, name: value.name, model: value.model, protocol: value.protocol,
|
||||
endpointKind: value.endpointKind, network: value.network, catalog: value.catalog, engineeringAllowed: value.engineeringAllowed,
|
||||
sources: ['network.json', 'catalog.json', 'result.json'].filter(file => fs.existsSync(path.join(root, 'compatibility', value.connectionId, file)))
|
||||
.map(file => source('compatibility/' + value.connectionId + '/' + file)),
|
||||
preflights: value.preflights.map(row => ({ scope: row.scope, passed: row.passed, sessionId: row.sessionId, taskId: row.taskId, runId: row.runId,
|
||||
identity: row.identity, status: row.status, usageStatus: row.usageStatus, usage: row.usage, actualToolsExecuted: row.actualToolsExecuted,
|
||||
streaming: row.streaming, wirePassed: row.wirePassed, outputFields: row.outputFields, expectedOutput: row.expectedOutput,
|
||||
sources: row.captures.map(capture => ({ ...capture, ...source(capture.path) })) })) })),
|
||||
};
|
||||
const engineering = [];
|
||||
for (const kind of ['node', 'react', 'python']) {
|
||||
if (!fs.existsSync(path.join(root, kind, 'result.json'))) continue;
|
||||
const result = read(kind + '/result.json');
|
||||
if (!result.executed) { engineering.push({ kind, result, source: source(kind + '/result.json') }); continue; }
|
||||
const session = read(kind + '/session.json'), attempts = read(kind + '/attempts.json');
|
||||
const metrics = {};
|
||||
for (const key of ['input', 'output', 'total', 'cacheRead', 'cacheWrite', 'uncached', 'reasoning']) {
|
||||
const values = attempts.map(value => value.usage?.tokenAccounting?.metrics?.[key]);
|
||||
const known = values.filter(value => Number.isSafeInteger(value?.tokens) && ['reported', 'derived'].includes(value.status));
|
||||
metrics[key] = { knownSumTokens: known.length ? known.reduce((sum, value) => sum + value.tokens, 0) : null,
|
||||
knownRequests: known.length, unknownRequests: values.length - known.length,
|
||||
statuses: [...new Set(values.map(value => value?.status ?? 'unknown'))] };
|
||||
}
|
||||
const reads = [], seenReads = new Set();
|
||||
for (const event of session.events.filter(value => value.kind === 'tool' && value.phase === 'completed' && value.tool === 'read_file')) {
|
||||
let output; try { output = JSON.parse(event.output); } catch { output = null; }
|
||||
const row = { eventId: event.id, path: output?.path ?? event.input?.path, startLine: output?.start_line ?? null, endLine: output?.end_line ?? null,
|
||||
fileSha256: output?.sha256 ?? null, outputSha256: sha(event.output ?? ''), parsed: output !== null };
|
||||
const key = JSON.stringify([row.path, row.startLine, row.endLine, row.fileSha256 ?? row.outputSha256]);
|
||||
row.sameByteRangePreviouslyRead = seenReads.has(key); seenReads.add(key); reads.push(row);
|
||||
}
|
||||
const completedTools = session.events.filter(value => value.kind === 'tool' && value.phase === 'completed');
|
||||
const wire = attempts.map(value => {
|
||||
const capture = result.captures.find(item => item.attemptId === value.id && item.kind === 'request');
|
||||
assert.ok(capture); const body = read(capture.path);
|
||||
return { attemptId: value.id, model: body.model, streaming: body.stream === true,
|
||||
outputFields: Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])),
|
||||
nativeReasoning: Object.fromEntries(['thinking', 'reasoning', 'reasoning_effort'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])),
|
||||
toolsCount: body.tools?.length ?? 0, toolsSchemaSha256: sha(JSON.stringify(body.tools ?? [])), source: source(capture.path) };
|
||||
});
|
||||
const reports = result.reportArtifacts.filter(value => value.kind === 'delivery-report').map(value => {
|
||||
const file = result.captures.find(capture => capture.artifactId === value.id).path, report = read(file);
|
||||
return { artifactId: value.id, status: report.status, taskStatus: report.task?.status, lifecycle: report.lifecycle,
|
||||
deliveries: report.deliveries?.map(item => ({ id: item.id, status: item.status, commit: item.commit })), source: source(file) };
|
||||
});
|
||||
engineering.push({ kind, success: result.success, executed: result.executed, projectId: result.projectId, sessionId: result.sessionId,
|
||||
taskId: result.taskId, runId: result.runId, connectionId: result.connectionId, requests: result.requests, usage: result.usage,
|
||||
startedAt: session.execution.run.startedAt, finishedAt: session.execution.run.finishedAt,
|
||||
elapsedMs: Date.parse(session.execution.run.finishedAt) - Date.parse(session.execution.run.startedAt),
|
||||
stopReason: result.productStopReason, failure: result.failure, identitiesPassed: result.identitiesPassed,
|
||||
commit: result.commit, initialHead: result.initialHead, committedPaths: result.committedPaths, committedCurrent: result.committedCurrent,
|
||||
otherUntracked: result.otherUntracked, final: result.final, browser: result.browser,
|
||||
ownerBytesAndIndexPreserved: result.ownerBytesAndIndexPreserved, supervisorContinuations: result.supervisorContinuations,
|
||||
localTokenCapsChanged: result.localTokenCapsChanged, internalDeliveries: result.internalDeliveries, reports,
|
||||
accounting: metrics, reads: { calls: reads.length, distinctByteRanges: seenReads.size, repeatedSameByteRanges: reads.filter(value => value.sameByteRangePreviouslyRead).length, observations: reads },
|
||||
tools: Object.fromEntries([...new Set(completedTools.map(value => value.tool))].map(tool => [tool, completedTools.filter(value => value.tool === tool).length])),
|
||||
wire, toolsSchemaVariants: [...new Set(wire.map(value => value.toolsSchemaSha256))],
|
||||
sources: ['claim.json', 'goal.txt', 'source-before.json', 'result.json', 'session.json', 'export.json', 'attempts.json', 'restoration.json', 'git-status.txt', 'git-log.txt']
|
||||
.map(file => source(kind + '/' + file)), captures: result.captures.map(value => ({ ...value, ...source(value.path) })) });
|
||||
}
|
||||
const auditHistory = fs.readdirSync(path.join(root, 'audits')).filter(name => fs.existsSync(path.join(root, 'audits', name, 'result.json')))
|
||||
.map(name => { const file = 'audits/' + name + '/result.json', value = read(file); return { stage: value.stage, success: value.success, error: value.error ?? null, source: source(file) }; });
|
||||
const summary = { schemaVersion: 1, stage, generatedAt: new Date().toISOString(), evidenceRoot: path.relative(process.cwd(), root).replaceAll('\\', '/'),
|
||||
frozen: source('frozen.json'), compatibility: compatibilitySummary, engineering,
|
||||
requests: compatibilitySummary.requests + engineering.reduce((sum, value) => sum + (value.requests ?? 0), 0),
|
||||
reportedTotalTokens: compatibilitySummary.reportedTotalTokens + engineering.reduce((sum, value) => sum + (value.usage?.totalTokens ?? 0), 0),
|
||||
unknownRequests: compatibilitySummary.unknownRequests + engineering.reduce((sum, value) => sum + (value.usage?.unknownRequests ?? 0), 0),
|
||||
audit: { ...audit, source: source(auditPrefix + '/result.json'), after: source(auditPrefix + '/after.json') }, auditHistory,
|
||||
drivers: fs.readdirSync(path.resolve('scripts/m4-live')).filter(name => name.endsWith('.mjs')).sort().map(name => { const file = 'scripts/m4-live/' + name, bytes = fs.readFileSync(file); return { path: file, bytes: bytes.length, sha256: sha(bytes) }; }),
|
||||
keysReadOrResaved: false, historicalSamplesReplayed: false, productSourceChanged: false, releaseOrRestart: false, purgeExecuted: false,
|
||||
unverified: ['Real Claude identity on the configured proxy', 'Long-term provider reliability', 'Real Mnemo production binding and semantic benefit',
|
||||
'Physical iPhone', 'General repeated-read improvement', 'R/W v2 live exclusion boundary', 'Causal cache benefit', 'Purge benefit'] };
|
||||
save(stage + '-summary.json', summary);
|
||||
const name = stage === 'node' ? 'compatibility-node' : stage;
|
||||
const file = path.resolve('docs/m4-' + name + '-2026-11-11.json');
|
||||
fs.writeFileSync(file, JSON.stringify(summary, null, 2) + '\n', { flag: 'wx' });
|
||||
console.log(JSON.stringify({ file, stage, requests: summary.requests, reportedTotalTokens: summary.reportedTotalTokens, unknownRequests: summary.unknownRequests,
|
||||
engineering: engineering.map(value => ({ kind: value.kind, success: value.success, requests: value.requests, elapsedMs: value.elapsedMs, cacheRead: value.accounting?.cacheRead, reads: value.reads?.calls, repeated: value.reads?.repeatedSameByteRanges })), audit: audit.success }));
|
||||
Reference in New Issue
Block a user