import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { DatabaseSync } from 'node:sqlite'; import { client, sessionAttempts, artifactBytes, connectionDefinition, expectedOutputFields, sha } from './m4-live/common.mjs'; import { invokeProbe, inspectorClosed } from './m4-live/inspector.mjs'; import { publicSnapshot, digest } from './retained-host-state.mjs'; export const root = path.resolve('.qa/m4-claude-correction-2026-11-12'); const origin = 'http://127.0.0.1:4318'; const connectionId = '9ae348a5-d0a6-42ed-8664-0da5f9effe34'; const load = relative => JSON.parse(fs.readFileSync(path.join(root, relative), 'utf8')); function save(relative, value) { const file = path.resolve(root, relative); assert.ok(file.startsWith(root + path.sep)); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' }); } const git = args => execFileSync('git', args, { encoding: 'utf8', windowsHide: true }).trim(); function source(relative) { const bytes = fs.readFileSync(path.join(root, relative)); return { path: relative, bytes: bytes.length, sha256: sha(bytes) }; } function requireOpen(config) { for (const [key, expected] of Object.entries({ executionMode: 'unrestricted', capacityMode: 'passthrough', approvalMode: 'full', maxSteps: 0, maxRunTokens: 0, maxTaskTokens: 0 })) assert.equal(config[key], expected, key); } export function sseFrames(raw) { return raw.split(/\r?\n\r?\n/).flatMap(packet => { const data = packet.split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trimStart()).join('\n'); if (!data || data === '[DONE]') return []; try { return [{ value: JSON.parse(data) }]; } catch { return [{ invalidJsonSha256: sha(data) }]; } }); } export function assessPreflight({ input, invoked, attempt, requests, body, raw, toolEvents = 0 }) { const outputFields = body ? Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])) : null; const frames = sseFrames(raw), starts = frames.filter(frame => frame.value?.type === 'message_start'); const modelFields = starts.map(frame => frame.value.message?.model ?? null); const stopReasons = frames.filter(frame => frame.value?.type === 'message_delta').map(frame => frame.value.delta?.stop_reason).filter(value => value != null); const rawComplete = starts.length === 1 && frames.some(frame => frame.value?.type === 'message_stop') && frames.every(frame => !frame.invalidJsonSha256); const usageComplete = attempt?.usageStatus === 'reported' && frames.some(frame => frame.value?.message?.usage) && frames.some(frame => frame.value?.usage); const streaming = { requested: body?.stream === true, sseObserved: /^(?:event:|data:)/m.test(raw), messageStartCount: starts.length, messageStopCount: frames.filter(frame => frame.value?.type === 'message_stop').length, invalidJsonFrames: frames.filter(frame => frame.invalidJsonSha256).length, rawComplete, usageComplete, complete: rawComplete && attempt?.modelIdentity?.complete === true }; const wirePassed = !!body && body.model === input.connection.model && JSON.stringify(outputFields) === JSON.stringify(input.expectedOutput.fields) && body.stream === true && streaming.sseObserved; const passed = invoked?.result.contentPassed === true && invoked.result.actualToolsExecuted === 0 && toolEvents === 0 && requests === 1 && wirePassed && streaming.complete && usageComplete && attempt?.status === 'completed' && attempt?.modelIdentity?.status === 'matched' && modelFields.length === 1 && modelFields[0] === input.connection.model; return { passed, wirePassed, streaming, modelFields, stopReasons, outputFields, rawUsageFrames: frames.filter(frame => frame.value?.usage || frame.value?.message?.usage).map(frame => ({ type: frame.value.type, usage: frame.value.usage ?? frame.value.message.usage })) }; } async function capture(api, sessionId, prefix) { const state = await api('/sessions/' + sessionId), exported = await api('/sessions/' + sessionId + '/export'), ledger = await sessionAttempts(api, sessionId); save(prefix + '/session.json', state); save(prefix + '/export.json', exported); save(prefix + '/attempts.json', ledger.rows); save(prefix + '/ledger-references.json', ledger); assert.ok(ledger.total <= 1, 'Each original case has at most one physical request.'); const captures = [], ids = new Set(); for (const row of ledger.rows) for (const kind of ['request', 'response']) { const id = row[kind + 'ArtifactId']; if (!id) continue; const artifact = state.execution.artifacts.find(value => value.id === id) ?? { id, sha256: row[kind + 'Sha256'] }; const bytes = await artifactBytes(api, sessionId, artifact); assert.equal(sha(bytes), row[kind + 'Sha256']); const relative = prefix + '/captures/' + row.id + '-' + kind + (kind === 'request' ? '.json' : row.streaming ? '.sse' : '.txt'); save(relative, bytes); captures.push({ attemptId: row.id, artifactId: id, kind, ...source(relative) }); ids.add(id); } for (const artifact of state.execution.artifacts) { if (ids.has(artifact.id) || !/^(text\/|application\/(json|x-ndjson))/.test(artifact.mediaType ?? '')) continue; const bytes = await artifactBytes(api, sessionId, artifact), relative = prefix + '/artifacts/' + artifact.id + (artifact.mediaType === 'application/json' ? '.json' : '.txt'); save(relative, bytes); captures.push({ artifactId: artifact.id, kind: artifact.kind, ...source(relative) }); } save(prefix + '/artifact-sources.json', captures); return { state, exported, ledger, captures }; } async function captureCase(api, scope, prefix, input, invoked) { const captured = await capture(api, input.sessionId, prefix), attempt = captured.ledger.rows[0]; const request = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'request'); const response = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'response'); const body = request ? load(request.path) : null, raw = response ? fs.readFileSync(path.join(root, response.path), 'utf8') : ''; const toolEvents = captured.state.events.filter(value => value.kind === 'tool').length; const run = captured.state.execution.run; const entry = { scope, sessionId: input.sessionId, taskId: invoked?.result.taskId ?? run?.taskId ?? null, runId: invoked?.result.runId ?? run?.id ?? null, attemptId: attempt?.id ?? null, requestKey: attempt?.requestKey ?? null, identity: attempt?.modelIdentity ?? null, status: attempt?.status ?? 'not_dispatched', usageStatus: attempt?.usageStatus ?? 'not_dispatched', usage: attempt?.usage ?? null, cost: attempt?.cost ?? null, costScope: 'Only an explicit ledger cost; absent upstream cost remains null, never inferred from tokens.', requests: captured.ledger.total, actualToolsExecuted: invoked?.result.actualToolsExecuted ?? run?.toolCalls ?? null, toolEvents, diagnostic: invoked?.result.diagnostic ?? attempt?.diagnostic ?? captured.state.session?.lastFailure ?? null, expectedOutput: input.expectedOutput, ...assessPreflight({ input, invoked, attempt, requests: captured.ledger.total, body, raw, toolEvents }), sources: captured.captures, invocationSource: fs.existsSync(path.join(root, scope, 'invocation.json')) ? source(scope + '/invocation.json') : null }; save(prefix + '/result.json', entry); return entry; } async function freeze() { assert.equal(fs.existsSync(path.join(root, 'frozen.json')), false); assert.equal(git(['status', '--porcelain']), '', 'Commit validated driver before freezing real requests.'); const { api, boot, close } = await client(); try { await inspectorClosed(); requireOpen(boot.config); assert.equal(boot.version, '1.25.13'); const host = await api('/host/status'), center = await api('/model-center'); assert.equal(host.activity.active, false); const connection = center.connections.find(value => value.id === connectionId); assert.ok(connection?.hasKey && connection.enabled); assert.equal(connection.model, 'claude-opus-5-5'); assert.equal(connection.protocol, 'anthropic-messages'); const baseline = await publicSnapshot(origin, host.identity.dataRoot); assert.equal(baseline.active.length, 0); const old = JSON.parse(fs.readFileSync('.qa/m4-live-baseline-2026-11-11/audits/final-bfadc758-619b-4288-b516-95e461da314d/after.json', 'utf8')); const oldConnection = old.modelCenter.connections.find(value => value.id === connectionId); const definitionChanges = Object.keys(connectionDefinition(connection)).filter(key => digest({ value: connectionDefinition(connection)[key] }) !== digest({ value: connectionDefinition(oldConnection)[key] })); const observed = load('observation.json'); save('frozen.json', { at: new Date().toISOString(), host, config: boot.config, center, connection, baseline, originalProjects: boot.projects, expectedOutput: expectedOutputFields(connection), driverCommit: git(['rev-parse', 'HEAD']), driverSha256: sha(fs.readFileSync('scripts/m4-claude-correction.mjs')), postSealPublicChanges: { configRevision: [old.modelCenter.connections.find(value => value.id === 'legacy-default').revision, boot.config.configRevision], registryRevision: [old.modelCenter.revision, center.revision], selectedConnectionRevision: [oldConnection.revision, connection.revision], credentialRevision: [oldConnection.credentialRevision, connection.credentialRevision], definitionChangedFields: definitionChanges, addedConnections: center.connections.filter(value => !old.modelCenter.connections.some(original => original.id === value.id)).map(value => ({ id: value.id, name: value.name, model: value.model, revision: value.revision })), observationRegistryRevision: observed.center.revision, actorOfAdditionalChanges: 'unestablished; preserved as observed; no connection restored' }, scopes: ['text', 'tools'], maximumPhysicalRequests: 2, observationMsPerCase: 180000, toolsExecuted: 0, keysReadOrResaved: false, oldM4Replayed: false }); save('driver-source.mjs', fs.readFileSync('scripts/m4-claude-correction.mjs')); console.log(JSON.stringify({ frozen: true, pid: host.identity.pid, revision: connection.revision, credentialRevision: connection.credentialRevision, registryRevision: center.revision, selected: center.selected, originalSessions: baseline.database.sessionIds.length, modelRequests: 0 })); } finally { await close(); } } async function run() { const frozen = load('frozen.json'); assert.equal(fs.existsSync(path.join(root, 'claim.json')), false, 'Never redispatch a claimed correction batch; reconcile its saved identities only.'); assert.equal(sha(fs.readFileSync('scripts/m4-claude-correction.mjs')), frozen.driverSha256); save('claim.json', { id: randomUUID(), at: new Date().toISOString(), scopes: ['text', 'tools'], maximumPhysicalRequests: 2, noRetry: true }); const { api, boot, close } = await client(); try { requireOpen(boot.config); assert.deepEqual(boot.config, frozen.config); await inspectorClosed(); const host = await api('/host/status'); assert.equal(host.identity.pid, frozen.host.identity.pid); assert.equal(host.activity.active, false); assert.deepEqual(connectionDefinition((await api('/model-center')).connections.find(value => value.id === connectionId)), connectionDefinition(frozen.connection)); const catalogId = randomUUID(); save('catalog-request.json', { id: catalogId, connectionId, revision: frozen.connection.revision }); try { save('catalog.json', await api('/model-center/catalog', { id: connectionId, revision: frozen.connection.revision, requestId: catalogId }, 90000)); } catch (error) { save('catalog-error.json', { message: error.message, response: error.response ?? null, noRetry: true }); } finally { save('catalog-cancel.json', await api('/model-center/catalog/cancel', { id: catalogId })); } const projectRoot = path.join(root, 'project'); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), 'Separate corrected Claude text/tool observations. No engineering or tool execution.\n', { flag: 'wx' }); const project = await api('/projects', { root: projectRoot }); save('project.json', project); const preflights = []; for (const scope of ['text', 'tools']) { assert.deepEqual(connectionDefinition((await api('/model-center')).connections.find(value => value.id === connectionId)), connectionDefinition(frozen.connection)); const session = await api('/sessions', { projectId: project.id }); const input = { pid: frozen.host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, projectId: project.id, sessionId: session.id, connection: frozen.connection, scope, nonce: randomUUID(), expectedOutput: frozen.expectedOutput, observationMs: frozen.observationMsPerCase }; save(scope + '/input.json', input); let invoked; try { invoked = await invokeProbe(input); save(scope + '/invocation.json', invoked); } catch (error) { save(scope + '/invocation-error.json', { message: error.stack, noRetry: true, reconcileOnly: true }); } const entry = await captureCase(api, scope, scope, input, invoked); preflights.push(entry); await inspectorClosed(); console.log(JSON.stringify({ scope, passed: entry.passed, attemptId: entry.attemptId, identity: entry.identity?.status, modelFields: entry.modelFields, usageStatus: entry.usageStatus, tokens: entry.usage?.total_tokens ?? null, toolsExecuted: entry.actualToolsExecuted })); } const catalog = fs.existsSync(path.join(root, 'catalog.json')) ? load('catalog.json') : null; const requests = preflights.reduce((sum, value) => sum + value.requests, 0); assert.ok(requests <= 2); save('result.json', { at: new Date().toISOString(), projectId: project.id, preflights, requests, reportedTotalTokens: preflights.reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0), unknownRequests: preflights.filter(value => value.requests && value.usageStatus !== 'reported').length, catalog: { complete: catalog?.complete === true, models: catalog?.entries?.length ?? null, selectedListed: catalog?.entries?.some(value => value.id === frozen.connection.model) ?? false, source: catalog ? source('catalog.json') : source('catalog-error.json'), evidenceScope: 'Public product catalogue with supplied source metadata; no invented raw HTTP capture.' }, selectedConnectionDefinitions: 'Frozen current connection only; no saved preference or credential mutation.', modelWeightsAuthenticated: false, savedConfigurationChanged: false, actualToolsExecuted: 0, engineeringOrPurgeReplayed: false, keysReadOrResaved: false }); } catch (error) { save('run-error.json', { message: error.stack, noRetry: true, reconcileOnly: true }); throw error; } finally { await inspectorClosed(); await close(); } } async function reconcile() { const frozen = load('frozen.json'), { api, close } = await client(); const prefix = 'reconcile-' + randomUUID(); try { const preflights = []; for (const scope of ['text', 'tools']) { if (!fs.existsSync(path.join(root, scope, 'input.json'))) continue; const input = load(scope + '/input.json'), invoked = fs.existsSync(path.join(root, scope, 'invocation.json')) ? load(scope + '/invocation.json') : null; preflights.push(await captureCase(api, scope, prefix + '/' + scope, input, invoked)); } assert.ok(preflights.reduce((sum, row) => sum + row.requests, 0) <= frozen.maximumPhysicalRequests); save(prefix + '/result.json', { readOnly: true, newModelRequests: 0, preflights, originalIdentitiesOnly: true }); console.log(JSON.stringify({ prefix, cases: preflights.length, newModelRequests: 0 })); } finally { await close(); } } async function audit() { const frozen = load('frozen.json'), result = load('result.json'), prefix = 'audit-' + randomUUID(), { api, boot, close } = await client(); let receipt; try { const host = await api('/host/status'); await inspectorClosed(); assert.equal(host.activity.active, false); assert.equal(host.identity.pid, frozen.host.identity.pid); assert.equal(host.identity.instanceId, frozen.host.identity.instanceId); requireOpen(boot.config); assert.deepEqual(boot.config, frozen.config); const after = await publicSnapshot(origin, host.identity.dataRoot); save(prefix + '/after.json', after); const before = frozen.baseline; assert.deepEqual(after.modelCenter.selected, frozen.center.selected); assert.deepEqual(after.modelCenter.pools, frozen.center.pools); assert.equal(after.modelCenter.revision, frozen.center.revision); assert.deepEqual(after.modelCenter.connections.map(connectionDefinition), frozen.center.connections.map(connectionDefinition)); const sessionIds = new Set(result.preflights.map(value => value.sessionId)); assert.equal(sessionIds.size, 2); assert.deepEqual(after.database.sessionIds.filter(value => !before.database.sessionIds.includes(value)).sort(), [...sessionIds].sort()); assert.deepEqual(boot.projects.filter(value => value.id !== result.projectId), frozen.originalProjects); assert.equal(boot.projects.length, frozen.originalProjects.length + 1); const exports = new Map(after.exports.map(value => [value.id, value])), artifacts = new Map(after.persistent.artifacts.map(value => [value.path, value])); for (const value of before.exports) assert.deepEqual(exports.get(value.id), value, 'Original export: ' + value.id); for (const value of before.persistent.artifacts) assert.deepEqual(artifacts.get(value.path), value, 'Original artifact: ' + value.path); for (const value of before.persistent.files) assert.deepEqual(after.persistent.files.find(file => file.path === value.path), value, 'Original persistent file: ' + value.path); const db = new DatabaseSync(path.join(host.identity.dataRoot, 'workbench.sqlite'), { readOnly: true }); const addedRows = {}, sequenceChanges = []; let checkedOriginalRows = 0; try { assert.deepEqual(Object.keys(after.database.tables), Object.keys(before.database.tables)); for (const [table, original] of Object.entries(before.database.tables)) { const current = after.database.tables[table]; assert.equal(current.schemaSha256, original.schemaSha256); addedRows[table] = current.count - original.count; const ids = new Set(original.rows.map(row => (row.kind ?? '') + ':' + row.id)), currentRows = new Map(current.rows.map(row => [(row.kind ?? '') + ':' + row.id, row])); for (const row of original.rows) { if (table === 'projects') { const value = db.prepare('SELECT * FROM projects WHERE id=?').get(row.id); assert.equal(digest({ ...value, position: value.position - 1 }), row.sha256); } else if (table === 'sqlite_sequence') { const value = db.prepare('SELECT * FROM sqlite_sequence').all().find(value => digest({ ...value, seq: row.id }) === row.sha256); assert.ok(value); const increment = value.name === 'events' ? after.eventCursor - before.eventCursor : value.name === 'model_attempts' ? result.requests : NaN; assert.ok(Number.isSafeInteger(increment)); assert.equal(value.seq, row.id + increment); sequenceChanges.push({ name: value.name, before: row.id, after: value.seq, increment }); } else assert.equal(currentRows.get((row.kind ?? '') + ':' + row.id)?.sha256, row.sha256, table + ':' + row.id); checkedOriginalRows++; } if (['events', 'records', 'sessions'].includes(table)) { const columns = table === 'records' ? 'id,kind,session_id,project_id' : table === 'events' ? 'id,session_id,project_id' : 'id,project_id'; for (const row of db.prepare('SELECT ' + columns + ' FROM ' + table).all()) if (!ids.has((row.kind ?? '') + ':' + row.id)) { assert.ok(sessionIds.has(table === 'sessions' ? row.id : row.session_id)); assert.equal(row.project_id, result.projectId); } } else if (table === 'outbox') { for (const row of db.prepare('SELECT outbox.seq,events.session_id FROM outbox JOIN events USING(seq) WHERE outbox.seq>?').all(before.eventCursor)) assert.ok(sessionIds.has(row.session_id)); } else if (table === 'model_attempts') { assert.equal(addedRows[table], result.requests); for (const row of db.prepare('SELECT id,session_id FROM model_attempts').all()) if (!ids.has(':' + row.id)) assert.ok(sessionIds.has(row.session_id)); } else if (table === 'projects') assert.equal(addedRows[table], 1); else assert.equal(addedRows[table], 0, 'No other new persistent rows: ' + table); } const references = new Map(db.prepare("SELECT payload FROM records WHERE kind='artifact'").all().map(row => { const value = JSON.parse(row.payload); return [value.relative, value]; })); for (const value of after.persistent.artifacts) if (!before.persistent.artifacts.some(original => original.path === value.path)) { const reference = references.get(value.path); assert.ok(reference && sessionIds.has(reference.sessionId)); assert.equal(reference.projectId, result.projectId); assert.equal(reference.sha256, value.sha256); assert.equal(reference.bytes, value.bytes); } } finally { db.close(); } receipt = { success: true, at: new Date().toISOString(), pid: host.identity.pid, instanceId: host.identity.instanceId, version: boot.version, originalSessions: before.database.sessionIds.length, originalModelRequests: before.database.tables.model_attempts.count, originalArtifacts: before.persistent.artifacts.length, checkedOriginalRows, addedRows, sequenceChanges, newArtifacts: after.persistent.artifacts.length - before.persistent.artifacts.length, currentSevenConnectionDefinitionsPoolsSelection: 'exact', originalExportsAndArtifactBytes: 'exact', originalMetadataAndCredentialRow: 'exact SHA; no credential decrypted or copied', originalProjects: 'exact after declared single front insertion', requests: result.requests, inspectorClosed: true, keysReadOrResaved: false, noRestorationOfOldHumanSettings: true }; } catch (error) { receipt = { success: false, at: new Date().toISOString(), error: error.stack, noMutationOrRetry: true }; process.exitCode = 1; } finally { save(prefix + '/result.json', receipt); await close(); } console.log(JSON.stringify({ prefix, ...receipt })); } async function seal() { assert.equal(fs.existsSync(path.join(root, 'MANIFEST.json')), false); const result = load('result.json'), frozen = load('frozen.json'); assert.ok(result.requests <= 2); const auditDirs = fs.readdirSync(root).filter(value => /^audit-/.test(value)); assert.ok(auditDirs.some(value => load(value + '/result.json').success)); const entries = []; const walk = relative => { for (const value of fs.readdirSync(path.join(root, relative), { withFileTypes: true })) { const filename = path.posix.join(relative.replaceAll('\\', '/'), value.name); assert.equal(value.isSymbolicLink(), false); if (value.isDirectory()) walk(filename); else if (value.isFile()) entries.push(source(filename)); else assert.fail('Unsupported evidence entry.'); } }; walk(''); entries.sort((a, b) => a.path.localeCompare(b.path, 'en')); save('MANIFEST.json', { schemaVersion: 1, at: new Date().toISOString(), batch: 'm4-claude-correction-2026-11-12', entries }); const receipt = { ...source('MANIFEST.json'), files: entries.length, bytes: entries.reduce((sum, value) => sum + value.bytes, 0), requests: result.requests, reportedTotalTokens: result.reportedTotalTokens, unknownRequests: result.unknownRequests, maximumPhysicalRequests: frozen.maximumPhysicalRequests, keysReadOrResaved: false, oldM4RootModified: false, engineeringOrPurgeReplayed: false }; save('seal.json', receipt); console.log(JSON.stringify(receipt)); } const action = process.argv[2]; if (action === 'freeze') await freeze(); else if (action === 'run') await run(); else if (action === 'reconcile') await reconcile(); else if (action === 'audit') await audit(); else if (action === 'seal') await seal(); else if (action !== undefined) throw Error('Supported actions: freeze, run, reconcile, audit, seal.');