import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; const [directory, ...options] = process.argv.slice(2), root = path.resolve(directory ?? 'docs/turn-progress-b-evidence'); const digest = value => createHash('sha256').update(typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value)).digest('hex'); const read = relative => { const file = path.resolve(root, relative); assert.ok(file.startsWith(root + path.sep)); return fs.readFileSync(file); }; const json = relative => JSON.parse(read(relative)); execFileSync(process.execPath, ['scripts/verify-adoption-evidence.mjs', root, ...options], { stdio: 'pipe' }); const manifest = json('manifest.json'); assert.equal(manifest.stage, 'turn-progress-b'); let snapshots = 0, receipts = 0, toolResults = 0, originalArtifacts = 0; for (const directory of manifest.observedRuns) { const result = json(directory + '/result.json'), index = new Map(json(directory + '/source-artifacts.json').map(row => [row.id, row])); assert.equal(result.success, true); assert.equal(result.passed.length, 9); assert.equal(result.realModelRequests, 0); const artifact = id => { const row = index.get(id); assert.ok(row, 'Missing original artifact: ' + id); const bytes = read(directory + '/' + row.path); assert.equal(digest(bytes), row.sha256); assert.equal(bytes.length, row.bytes); return { row, bytes }; }; for (const id of index.keys()) { artifact(id); originalArtifacts++; } const snapshot = name => json(directory + '/' + name + '.json'); for (const name of result.snapshots) { const value = json(directory + '/' + name), { session, execution } = value, run = execution.run, shadow = run.progressShadow; snapshots++; assert.deepEqual(shadow.scope, { projectId: session.projectId, sessionId: session.id, taskId: run.taskId, runId: run.id }); assert.equal(shadow.mode, 'shadow'); assert.equal(shadow.turns.length, run.progressShadowArtifactIds.length); const ranges = new Map(); for (const [i, turn] of shadow.turns.entries()) { const original = artifact(run.progressShadowArtifactIds[i]); assert.equal(original.row.kind, 'turn-progress-shadow'); assert.equal(digest(JSON.parse(original.bytes)), digest(turn)); assert.equal(turn.runId, run.id); assert.equal(turn.taskId, run.taskId); receipts++; let growth = 0, repeated = 0; for (const ref of turn.tools) { if (ref.tool === 'model_request') { const event = session.events.find(row => row.id === ref.source_event_id && row.runId === run.id); assert.ok(event?.diagnostic); assert.equal(event.diagnostic.id, ref.id); assert.equal(digest(event.diagnostic), ref.diagnostic_sha256); if (ref.result_artifact_id) assert.equal(artifact(ref.result_artifact_id).row.sha256, ref.result_sha256); continue; } const event = session.events.find(row => row.invocationId === ref.id && row.runId === run.id && row.taskId === run.taskId); assert.ok(event); assert.equal(event.tool, ref.tool); assert.equal(event.artifactId, ref.result_artifact_id); const call = session.providerMessages.flatMap(row => row.tool_calls ?? []).find(row => row.id === event.toolCallId); assert.ok(call); assert.equal(call.function.name, ref.tool); assert.equal(digest(String(call.function.arguments)), ref.args_sha256); const source = artifact(ref.result_artifact_id); assert.equal(source.row.kind, 'tool-result'); assert.equal(source.row.sha256, ref.result_sha256); toolResults++; const metadata = execution.artifacts.find(row => row.id === ref.result_artifact_id); assert.ok(metadata); assert.equal(metadata.runId, run.id); assert.equal(metadata.taskId, run.taskId); assert.equal(metadata.projectId, session.projectId); const result = JSON.parse(source.bytes); if (ref.tool !== 'read_file' || typeof result.content !== 'string') continue; const start = result.offset ?? 0, end = result.next_offset ?? start + result.content.length; assert.equal(end - start, result.content.length); const key = JSON.stringify([result.path, result.sha256, result.range_start_line ?? result.start_line ?? null, result.range_end_line ?? result.end_line ?? null]); const covered = ranges.get(key) ?? new Set(); let added = 0; for (let position = start; position < end; position++) if (!covered.has(position)) { covered.add(position); added++; } ranges.set(key, covered); growth += added; if (!added) repeated++; } assert.equal(turn.delta.read_characters, growth); assert.equal(turn.delta.repeated_reads, repeated); if (turn.phase === 'waiting') assert.equal(turn.classification, 'waiting'); } } const pages = snapshot('pages-original'), settled = pages.execution.run.progressShadow.turns.filter(row => row.phase === 'settled'); assert.equal(pages.execution.run.stopReason, 'step_limit'); assert.deepEqual(settled.map(row => [row.stepIndex, row.classification, row.delta.read_characters]), [[1, 'evidence_growth', 10], [2, 'evidence_growth', 10], [3, 'repeated_read_candidate', 0]]); const waiting = snapshot('approval-waiting'), denied = snapshot('approval-denied'), failure = snapshot('network-original'); assert.equal(waiting.session.status, 'awaiting_approval'); assert.equal(waiting.execution.run.progressShadow.turns.at(-1).waiting[0].kind, 'approval'); assert.equal(denied.session.taskId, pages.session.taskId); assert.notEqual(denied.execution.run.id, pages.execution.run.id); assert.equal(denied.session.approvals.at(-1).status, 'denied'); assert.equal(denied.execution.run.progressShadow.turns.find(row => row.phase === 'settled').delta.repeated_reads, 0); assert.equal(failure.session.status, 'failed'); assert.equal(failure.session.lastFailure.httpStatus, 502); assert.equal(failure.execution.run.stopReason, null); assert.equal(failure.execution.run.progressShadow.turns.at(-1).classification, 'new_failure_evidence'); const report = json(directory + '/report.json'); assert.equal(report.turnObservations.length, 2); assert.ok(report.turnObservations.every(row => row.evidence_verified)); } console.log(JSON.stringify({ ok: true, runs: manifest.observedRuns.length, snapshots, receipts, toolResults, originalArtifacts, realModelRequests: 0, replayedOperations: 0 }));