Files
Agent-Workbench/scripts/m4-claude-correction.mjs
2026-11-12 00:53:14 +08:00

231 lines
24 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { DatabaseSync } from 'node:sqlite';
import { client, sessionAttempts, artifactBytes, connectionDefinition, expectedOutputFields, sha } from './m4-live/common.mjs';
import { invokeProbe, inspectorClosed } from './m4-live/inspector.mjs';
import { publicSnapshot, digest } from './retained-host-state.mjs';
export const root = path.resolve('.qa/m4-claude-correction-2026-11-12');
const origin = 'http://127.0.0.1:4318';
const connectionId = '9ae348a5-d0a6-42ed-8664-0da5f9effe34';
const load = relative => JSON.parse(fs.readFileSync(path.join(root, relative), 'utf8'));
function save(relative, value) {
const file = path.resolve(root, relative); assert.ok(file.startsWith(root + path.sep));
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
}
const git = args => execFileSync('git', args, { encoding: 'utf8', windowsHide: true }).trim();
function source(relative) {
const bytes = fs.readFileSync(path.join(root, relative)); return { path: relative, bytes: bytes.length, sha256: sha(bytes) };
}
function requireOpen(config) {
for (const [key, expected] of Object.entries({ executionMode: 'unrestricted', capacityMode: 'passthrough', approvalMode: 'full', maxSteps: 0, maxRunTokens: 0, maxTaskTokens: 0 })) assert.equal(config[key], expected, key);
}
export function sseFrames(raw) {
return raw.split(/\r?\n\r?\n/).flatMap(packet => {
const data = packet.split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trimStart()).join('\n');
if (!data || data === '[DONE]') return [];
try { return [{ value: JSON.parse(data) }]; } catch { return [{ invalidJsonSha256: sha(data) }]; }
});
}
export function assessPreflight({ input, invoked, attempt, requests, body, raw, toolEvents = 0 }) {
const outputFields = body ? Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])) : null;
const frames = sseFrames(raw), starts = frames.filter(frame => frame.value?.type === 'message_start');
const modelFields = starts.map(frame => frame.value.message?.model ?? null);
const stopReasons = frames.filter(frame => frame.value?.type === 'message_delta').map(frame => frame.value.delta?.stop_reason).filter(value => value != null);
const rawComplete = starts.length === 1 && frames.some(frame => frame.value?.type === 'message_stop') && frames.every(frame => !frame.invalidJsonSha256);
const usageComplete = attempt?.usageStatus === 'reported' && frames.some(frame => frame.value?.message?.usage) && frames.some(frame => frame.value?.usage);
const streaming = { requested: body?.stream === true, sseObserved: /^(?:event:|data:)/m.test(raw), messageStartCount: starts.length,
messageStopCount: frames.filter(frame => frame.value?.type === 'message_stop').length, invalidJsonFrames: frames.filter(frame => frame.invalidJsonSha256).length,
rawComplete, usageComplete, complete: rawComplete && attempt?.modelIdentity?.complete === true };
const wirePassed = !!body && body.model === input.connection.model && JSON.stringify(outputFields) === JSON.stringify(input.expectedOutput.fields) && body.stream === true && streaming.sseObserved;
const passed = invoked?.result.contentPassed === true && invoked.result.actualToolsExecuted === 0 && toolEvents === 0 && requests === 1 && wirePassed && streaming.complete && usageComplete && attempt?.status === 'completed'
&& attempt?.modelIdentity?.status === 'matched' && modelFields.length === 1 && modelFields[0] === input.connection.model;
return { passed, wirePassed, streaming, modelFields, stopReasons, outputFields,
rawUsageFrames: frames.filter(frame => frame.value?.usage || frame.value?.message?.usage).map(frame => ({ type: frame.value.type, usage: frame.value.usage ?? frame.value.message.usage })) };
}
async function capture(api, sessionId, prefix) {
const state = await api('/sessions/' + sessionId), exported = await api('/sessions/' + sessionId + '/export'), ledger = await sessionAttempts(api, sessionId);
save(prefix + '/session.json', state); save(prefix + '/export.json', exported); save(prefix + '/attempts.json', ledger.rows); save(prefix + '/ledger-references.json', ledger);
assert.ok(ledger.total <= 1, 'Each original case has at most one physical request.');
const captures = [], ids = new Set();
for (const row of ledger.rows) for (const kind of ['request', 'response']) {
const id = row[kind + 'ArtifactId']; if (!id) continue;
const artifact = state.execution.artifacts.find(value => value.id === id) ?? { id, sha256: row[kind + 'Sha256'] };
const bytes = await artifactBytes(api, sessionId, artifact); assert.equal(sha(bytes), row[kind + 'Sha256']);
const relative = prefix + '/captures/' + row.id + '-' + kind + (kind === 'request' ? '.json' : row.streaming ? '.sse' : '.txt');
save(relative, bytes); captures.push({ attemptId: row.id, artifactId: id, kind, ...source(relative) }); ids.add(id);
}
for (const artifact of state.execution.artifacts) {
if (ids.has(artifact.id) || !/^(text\/|application\/(json|x-ndjson))/.test(artifact.mediaType ?? '')) continue;
const bytes = await artifactBytes(api, sessionId, artifact), relative = prefix + '/artifacts/' + artifact.id + (artifact.mediaType === 'application/json' ? '.json' : '.txt');
save(relative, bytes); captures.push({ artifactId: artifact.id, kind: artifact.kind, ...source(relative) });
}
save(prefix + '/artifact-sources.json', captures); return { state, exported, ledger, captures };
}
async function captureCase(api, scope, prefix, input, invoked) {
const captured = await capture(api, input.sessionId, prefix), attempt = captured.ledger.rows[0];
const request = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'request');
const response = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'response');
const body = request ? load(request.path) : null, raw = response ? fs.readFileSync(path.join(root, response.path), 'utf8') : '';
const toolEvents = captured.state.events.filter(value => value.kind === 'tool').length;
const run = captured.state.execution.run;
const entry = { scope, sessionId: input.sessionId, taskId: invoked?.result.taskId ?? run?.taskId ?? null, runId: invoked?.result.runId ?? run?.id ?? null,
attemptId: attempt?.id ?? null, requestKey: attempt?.requestKey ?? null, identity: attempt?.modelIdentity ?? null, status: attempt?.status ?? 'not_dispatched',
usageStatus: attempt?.usageStatus ?? 'not_dispatched', usage: attempt?.usage ?? null, cost: attempt?.cost ?? null,
costScope: 'Only an explicit ledger cost; absent upstream cost remains null, never inferred from tokens.',
requests: captured.ledger.total, actualToolsExecuted: invoked?.result.actualToolsExecuted ?? run?.toolCalls ?? null, toolEvents,
diagnostic: invoked?.result.diagnostic ?? attempt?.diagnostic ?? captured.state.session?.lastFailure ?? null,
expectedOutput: input.expectedOutput, ...assessPreflight({ input, invoked, attempt, requests: captured.ledger.total, body, raw, toolEvents }),
sources: captured.captures, invocationSource: fs.existsSync(path.join(root, scope, 'invocation.json')) ? source(scope + '/invocation.json') : null };
save(prefix + '/result.json', entry); return entry;
}
async function freeze() {
assert.equal(fs.existsSync(path.join(root, 'frozen.json')), false); assert.equal(git(['status', '--porcelain']), '', 'Commit validated driver before freezing real requests.');
const { api, boot, close } = await client(); try {
await inspectorClosed(); requireOpen(boot.config); assert.equal(boot.version, '1.25.13');
const host = await api('/host/status'), center = await api('/model-center'); assert.equal(host.activity.active, false);
const connection = center.connections.find(value => value.id === connectionId); assert.ok(connection?.hasKey && connection.enabled);
assert.equal(connection.model, 'claude-opus-5-5'); assert.equal(connection.protocol, 'anthropic-messages');
const baseline = await publicSnapshot(origin, host.identity.dataRoot); assert.equal(baseline.active.length, 0);
const old = JSON.parse(fs.readFileSync('.qa/m4-live-baseline-2026-11-11/audits/final-bfadc758-619b-4288-b516-95e461da314d/after.json', 'utf8'));
const oldConnection = old.modelCenter.connections.find(value => value.id === connectionId);
const definitionChanges = Object.keys(connectionDefinition(connection)).filter(key => digest({ value: connectionDefinition(connection)[key] }) !== digest({ value: connectionDefinition(oldConnection)[key] }));
const observed = load('observation.json');
save('frozen.json', { at: new Date().toISOString(), host, config: boot.config, center, connection, baseline, originalProjects: boot.projects,
expectedOutput: expectedOutputFields(connection), driverCommit: git(['rev-parse', 'HEAD']), driverSha256: sha(fs.readFileSync('scripts/m4-claude-correction.mjs')),
postSealPublicChanges: { configRevision: [old.modelCenter.connections.find(value => value.id === 'legacy-default').revision, boot.config.configRevision],
registryRevision: [old.modelCenter.revision, center.revision], selectedConnectionRevision: [oldConnection.revision, connection.revision],
credentialRevision: [oldConnection.credentialRevision, connection.credentialRevision], definitionChangedFields: definitionChanges,
addedConnections: center.connections.filter(value => !old.modelCenter.connections.some(original => original.id === value.id)).map(value => ({ id: value.id, name: value.name, model: value.model, revision: value.revision })),
observationRegistryRevision: observed.center.revision, actorOfAdditionalChanges: 'unestablished; preserved as observed; no connection restored' },
scopes: ['text', 'tools'], maximumPhysicalRequests: 2, observationMsPerCase: 180000, toolsExecuted: 0, keysReadOrResaved: false, oldM4Replayed: false });
save('driver-source.mjs', fs.readFileSync('scripts/m4-claude-correction.mjs'));
console.log(JSON.stringify({ frozen: true, pid: host.identity.pid, revision: connection.revision, credentialRevision: connection.credentialRevision, registryRevision: center.revision, selected: center.selected, originalSessions: baseline.database.sessionIds.length, modelRequests: 0 }));
} finally { await close(); }
}
async function run() {
const frozen = load('frozen.json'); assert.equal(fs.existsSync(path.join(root, 'claim.json')), false, 'Never redispatch a claimed correction batch; reconcile its saved identities only.');
assert.equal(sha(fs.readFileSync('scripts/m4-claude-correction.mjs')), frozen.driverSha256);
save('claim.json', { id: randomUUID(), at: new Date().toISOString(), scopes: ['text', 'tools'], maximumPhysicalRequests: 2, noRetry: true });
const { api, boot, close } = await client(); try {
requireOpen(boot.config); assert.deepEqual(boot.config, frozen.config); await inspectorClosed();
const host = await api('/host/status'); assert.equal(host.identity.pid, frozen.host.identity.pid); assert.equal(host.activity.active, false);
assert.deepEqual(connectionDefinition((await api('/model-center')).connections.find(value => value.id === connectionId)), connectionDefinition(frozen.connection));
const catalogId = randomUUID(); save('catalog-request.json', { id: catalogId, connectionId, revision: frozen.connection.revision });
try { save('catalog.json', await api('/model-center/catalog', { id: connectionId, revision: frozen.connection.revision, requestId: catalogId }, 90000)); }
catch (error) { save('catalog-error.json', { message: error.message, response: error.response ?? null, noRetry: true }); }
finally { save('catalog-cancel.json', await api('/model-center/catalog/cancel', { id: catalogId })); }
const projectRoot = path.join(root, 'project'); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), 'Separate corrected Claude text/tool observations. No engineering or tool execution.\n', { flag: 'wx' });
const project = await api('/projects', { root: projectRoot }); save('project.json', project);
const preflights = [];
for (const scope of ['text', 'tools']) {
assert.deepEqual(connectionDefinition((await api('/model-center')).connections.find(value => value.id === connectionId)), connectionDefinition(frozen.connection));
const session = await api('/sessions', { projectId: project.id });
const input = { pid: frozen.host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, projectId: project.id, sessionId: session.id,
connection: frozen.connection, scope, nonce: randomUUID(), expectedOutput: frozen.expectedOutput, observationMs: frozen.observationMsPerCase };
save(scope + '/input.json', input); let invoked;
try { invoked = await invokeProbe(input); save(scope + '/invocation.json', invoked); }
catch (error) { save(scope + '/invocation-error.json', { message: error.stack, noRetry: true, reconcileOnly: true }); }
const entry = await captureCase(api, scope, scope, input, invoked); preflights.push(entry); await inspectorClosed();
console.log(JSON.stringify({ scope, passed: entry.passed, attemptId: entry.attemptId, identity: entry.identity?.status, modelFields: entry.modelFields, usageStatus: entry.usageStatus, tokens: entry.usage?.total_tokens ?? null, toolsExecuted: entry.actualToolsExecuted }));
}
const catalog = fs.existsSync(path.join(root, 'catalog.json')) ? load('catalog.json') : null;
const requests = preflights.reduce((sum, value) => sum + value.requests, 0); assert.ok(requests <= 2);
save('result.json', { at: new Date().toISOString(), projectId: project.id, preflights, requests,
reportedTotalTokens: preflights.reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0), unknownRequests: preflights.filter(value => value.requests && value.usageStatus !== 'reported').length,
catalog: { complete: catalog?.complete === true, models: catalog?.entries?.length ?? null, selectedListed: catalog?.entries?.some(value => value.id === frozen.connection.model) ?? false,
source: catalog ? source('catalog.json') : source('catalog-error.json'), evidenceScope: 'Public product catalogue with supplied source metadata; no invented raw HTTP capture.' },
selectedConnectionDefinitions: 'Frozen current connection only; no saved preference or credential mutation.', modelWeightsAuthenticated: false,
savedConfigurationChanged: false, actualToolsExecuted: 0, engineeringOrPurgeReplayed: false, keysReadOrResaved: false });
} catch (error) { save('run-error.json', { message: error.stack, noRetry: true, reconcileOnly: true }); throw error; }
finally { await inspectorClosed(); await close(); }
}
async function reconcile() {
const frozen = load('frozen.json'), { api, close } = await client(); const prefix = 'reconcile-' + randomUUID();
try {
const preflights = [];
for (const scope of ['text', 'tools']) {
if (!fs.existsSync(path.join(root, scope, 'input.json'))) continue;
const input = load(scope + '/input.json'), invoked = fs.existsSync(path.join(root, scope, 'invocation.json')) ? load(scope + '/invocation.json') : null;
preflights.push(await captureCase(api, scope, prefix + '/' + scope, input, invoked));
}
assert.ok(preflights.reduce((sum, row) => sum + row.requests, 0) <= frozen.maximumPhysicalRequests);
save(prefix + '/result.json', { readOnly: true, newModelRequests: 0, preflights, originalIdentitiesOnly: true }); console.log(JSON.stringify({ prefix, cases: preflights.length, newModelRequests: 0 }));
} finally { await close(); }
}
async function audit() {
const frozen = load('frozen.json'), result = load('result.json'), prefix = 'audit-' + randomUUID(), { api, boot, close } = await client(); let receipt;
try {
const host = await api('/host/status'); await inspectorClosed(); assert.equal(host.activity.active, false); assert.equal(host.identity.pid, frozen.host.identity.pid); assert.equal(host.identity.instanceId, frozen.host.identity.instanceId);
requireOpen(boot.config); assert.deepEqual(boot.config, frozen.config);
const after = await publicSnapshot(origin, host.identity.dataRoot); save(prefix + '/after.json', after); const before = frozen.baseline;
assert.deepEqual(after.modelCenter.selected, frozen.center.selected); assert.deepEqual(after.modelCenter.pools, frozen.center.pools); assert.equal(after.modelCenter.revision, frozen.center.revision);
assert.deepEqual(after.modelCenter.connections.map(connectionDefinition), frozen.center.connections.map(connectionDefinition));
const sessionIds = new Set(result.preflights.map(value => value.sessionId)); assert.equal(sessionIds.size, 2);
assert.deepEqual(after.database.sessionIds.filter(value => !before.database.sessionIds.includes(value)).sort(), [...sessionIds].sort());
assert.deepEqual(boot.projects.filter(value => value.id !== result.projectId), frozen.originalProjects); assert.equal(boot.projects.length, frozen.originalProjects.length + 1);
const exports = new Map(after.exports.map(value => [value.id, value])), artifacts = new Map(after.persistent.artifacts.map(value => [value.path, value]));
for (const value of before.exports) assert.deepEqual(exports.get(value.id), value, 'Original export: ' + value.id);
for (const value of before.persistent.artifacts) assert.deepEqual(artifacts.get(value.path), value, 'Original artifact: ' + value.path);
for (const value of before.persistent.files) assert.deepEqual(after.persistent.files.find(file => file.path === value.path), value, 'Original persistent file: ' + value.path);
const db = new DatabaseSync(path.join(host.identity.dataRoot, 'workbench.sqlite'), { readOnly: true }); const addedRows = {}, sequenceChanges = []; let checkedOriginalRows = 0;
try {
assert.deepEqual(Object.keys(after.database.tables), Object.keys(before.database.tables));
for (const [table, original] of Object.entries(before.database.tables)) {
const current = after.database.tables[table]; assert.equal(current.schemaSha256, original.schemaSha256); addedRows[table] = current.count - original.count;
const ids = new Set(original.rows.map(row => (row.kind ?? '') + ':' + row.id)), currentRows = new Map(current.rows.map(row => [(row.kind ?? '') + ':' + row.id, row]));
for (const row of original.rows) {
if (table === 'projects') { const value = db.prepare('SELECT * FROM projects WHERE id=?').get(row.id); assert.equal(digest({ ...value, position: value.position - 1 }), row.sha256); }
else if (table === 'sqlite_sequence') {
const value = db.prepare('SELECT * FROM sqlite_sequence').all().find(value => digest({ ...value, seq: row.id }) === row.sha256); assert.ok(value);
const increment = value.name === 'events' ? after.eventCursor - before.eventCursor : value.name === 'model_attempts' ? result.requests : NaN;
assert.ok(Number.isSafeInteger(increment)); assert.equal(value.seq, row.id + increment); sequenceChanges.push({ name: value.name, before: row.id, after: value.seq, increment });
} else assert.equal(currentRows.get((row.kind ?? '') + ':' + row.id)?.sha256, row.sha256, table + ':' + row.id);
checkedOriginalRows++;
}
if (['events', 'records', 'sessions'].includes(table)) {
const columns = table === 'records' ? 'id,kind,session_id,project_id' : table === 'events' ? 'id,session_id,project_id' : 'id,project_id';
for (const row of db.prepare('SELECT ' + columns + ' FROM ' + table).all()) if (!ids.has((row.kind ?? '') + ':' + row.id)) { assert.ok(sessionIds.has(table === 'sessions' ? row.id : row.session_id)); assert.equal(row.project_id, result.projectId); }
} else if (table === 'outbox') {
for (const row of db.prepare('SELECT outbox.seq,events.session_id FROM outbox JOIN events USING(seq) WHERE outbox.seq>?').all(before.eventCursor)) assert.ok(sessionIds.has(row.session_id));
} else if (table === 'model_attempts') {
assert.equal(addedRows[table], result.requests);
for (const row of db.prepare('SELECT id,session_id FROM model_attempts').all()) if (!ids.has(':' + row.id)) assert.ok(sessionIds.has(row.session_id));
} else if (table === 'projects') assert.equal(addedRows[table], 1);
else assert.equal(addedRows[table], 0, 'No other new persistent rows: ' + table);
}
const references = new Map(db.prepare("SELECT payload FROM records WHERE kind='artifact'").all().map(row => { const value = JSON.parse(row.payload); return [value.relative, value]; }));
for (const value of after.persistent.artifacts) if (!before.persistent.artifacts.some(original => original.path === value.path)) {
const reference = references.get(value.path); assert.ok(reference && sessionIds.has(reference.sessionId)); assert.equal(reference.projectId, result.projectId); assert.equal(reference.sha256, value.sha256); assert.equal(reference.bytes, value.bytes);
}
} finally { db.close(); }
receipt = { success: true, at: new Date().toISOString(), pid: host.identity.pid, instanceId: host.identity.instanceId, version: boot.version, originalSessions: before.database.sessionIds.length,
originalModelRequests: before.database.tables.model_attempts.count, originalArtifacts: before.persistent.artifacts.length, checkedOriginalRows, addedRows, sequenceChanges,
newArtifacts: after.persistent.artifacts.length - before.persistent.artifacts.length, currentSevenConnectionDefinitionsPoolsSelection: 'exact', originalExportsAndArtifactBytes: 'exact',
originalMetadataAndCredentialRow: 'exact SHA; no credential decrypted or copied', originalProjects: 'exact after declared single front insertion', requests: result.requests, inspectorClosed: true, keysReadOrResaved: false, noRestorationOfOldHumanSettings: true };
} catch (error) { receipt = { success: false, at: new Date().toISOString(), error: error.stack, noMutationOrRetry: true }; process.exitCode = 1; }
finally { save(prefix + '/result.json', receipt); await close(); }
console.log(JSON.stringify({ prefix, ...receipt }));
}
async function seal() {
assert.equal(fs.existsSync(path.join(root, 'MANIFEST.json')), false);
const result = load('result.json'), frozen = load('frozen.json'); assert.ok(result.requests <= 2);
const auditDirs = fs.readdirSync(root).filter(value => /^audit-/.test(value)); assert.ok(auditDirs.some(value => load(value + '/result.json').success));
const entries = []; const walk = relative => { for (const value of fs.readdirSync(path.join(root, relative), { withFileTypes: true })) {
const filename = path.posix.join(relative.replaceAll('\\', '/'), value.name); assert.equal(value.isSymbolicLink(), false);
if (value.isDirectory()) walk(filename); else if (value.isFile()) entries.push(source(filename)); else assert.fail('Unsupported evidence entry.');
} }; walk(''); entries.sort((a, b) => a.path.localeCompare(b.path, 'en'));
save('MANIFEST.json', { schemaVersion: 1, at: new Date().toISOString(), batch: 'm4-claude-correction-2026-11-12', entries });
const receipt = { ...source('MANIFEST.json'), files: entries.length, bytes: entries.reduce((sum, value) => sum + value.bytes, 0), requests: result.requests,
reportedTotalTokens: result.reportedTotalTokens, unknownRequests: result.unknownRequests, maximumPhysicalRequests: frozen.maximumPhysicalRequests,
keysReadOrResaved: false, oldM4RootModified: false, engineeringOrPurgeReplayed: false };
save('seal.json', receipt); console.log(JSON.stringify(receipt));
}
const action = process.argv[2];
if (action === 'freeze') await freeze(); else if (action === 'run') await run(); else if (action === 'reconcile') await reconcile(); else if (action === 'audit') await audit(); else if (action === 'seal') await seal();
else if (action !== undefined) throw Error('Supported actions: freeze, run, reconcile, audit, seal.');