69 lines
6.3 KiB
JavaScript
69 lines
6.3 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
|
|
const [directory, ...options] = process.argv.slice(2), root = path.resolve(directory ?? 'docs/turn-progress-b-evidence');
|
|
const digest = value => createHash('sha256').update(typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value)).digest('hex');
|
|
const read = relative => { const file = path.resolve(root, relative); assert.ok(file.startsWith(root + path.sep)); return fs.readFileSync(file); };
|
|
const json = relative => JSON.parse(read(relative));
|
|
execFileSync(process.execPath, ['scripts/verify-adoption-evidence.mjs', root, ...options], { stdio: 'pipe' });
|
|
const manifest = json('manifest.json'); assert.equal(manifest.stage, 'turn-progress-b');
|
|
let snapshots = 0, receipts = 0, toolResults = 0, originalArtifacts = 0;
|
|
for (const directory of manifest.observedRuns) {
|
|
const result = json(directory + '/result.json'), index = new Map(json(directory + '/source-artifacts.json').map(row => [row.id, row]));
|
|
assert.equal(result.success, true); assert.equal(result.passed.length, 9); assert.equal(result.realModelRequests, 0);
|
|
const artifact = id => { const row = index.get(id); assert.ok(row, 'Missing original artifact: ' + id); const bytes = read(directory + '/' + row.path);
|
|
assert.equal(digest(bytes), row.sha256); assert.equal(bytes.length, row.bytes); return { row, bytes }; };
|
|
for (const id of index.keys()) { artifact(id); originalArtifacts++; }
|
|
const snapshot = name => json(directory + '/' + name + '.json');
|
|
for (const name of result.snapshots) {
|
|
const value = json(directory + '/' + name), { session, execution } = value, run = execution.run, shadow = run.progressShadow; snapshots++;
|
|
assert.deepEqual(shadow.scope, { projectId: session.projectId, sessionId: session.id, taskId: run.taskId, runId: run.id });
|
|
assert.equal(shadow.mode, 'shadow'); assert.equal(shadow.turns.length, run.progressShadowArtifactIds.length);
|
|
const ranges = new Map();
|
|
for (const [i, turn] of shadow.turns.entries()) {
|
|
const original = artifact(run.progressShadowArtifactIds[i]); assert.equal(original.row.kind, 'turn-progress-shadow');
|
|
assert.equal(digest(JSON.parse(original.bytes)), digest(turn)); assert.equal(turn.runId, run.id); assert.equal(turn.taskId, run.taskId); receipts++;
|
|
let growth = 0, repeated = 0;
|
|
for (const ref of turn.tools) {
|
|
if (ref.tool === 'model_request') {
|
|
const event = session.events.find(row => row.id === ref.source_event_id && row.runId === run.id);
|
|
assert.ok(event?.diagnostic); assert.equal(event.diagnostic.id, ref.id); assert.equal(digest(event.diagnostic), ref.diagnostic_sha256);
|
|
if (ref.result_artifact_id) assert.equal(artifact(ref.result_artifact_id).row.sha256, ref.result_sha256);
|
|
continue;
|
|
}
|
|
const event = session.events.find(row => row.invocationId === ref.id && row.runId === run.id && row.taskId === run.taskId);
|
|
assert.ok(event); assert.equal(event.tool, ref.tool); assert.equal(event.artifactId, ref.result_artifact_id);
|
|
const call = session.providerMessages.flatMap(row => row.tool_calls ?? []).find(row => row.id === event.toolCallId);
|
|
assert.ok(call); assert.equal(call.function.name, ref.tool); assert.equal(digest(String(call.function.arguments)), ref.args_sha256);
|
|
const source = artifact(ref.result_artifact_id); assert.equal(source.row.kind, 'tool-result'); assert.equal(source.row.sha256, ref.result_sha256); toolResults++;
|
|
const metadata = execution.artifacts.find(row => row.id === ref.result_artifact_id);
|
|
assert.ok(metadata); assert.equal(metadata.runId, run.id); assert.equal(metadata.taskId, run.taskId); assert.equal(metadata.projectId, session.projectId);
|
|
const result = JSON.parse(source.bytes);
|
|
if (ref.tool !== 'read_file' || typeof result.content !== 'string') continue;
|
|
const start = result.offset ?? 0, end = result.next_offset ?? start + result.content.length;
|
|
assert.equal(end - start, result.content.length);
|
|
const key = JSON.stringify([result.path, result.sha256, result.range_start_line ?? result.start_line ?? null, result.range_end_line ?? result.end_line ?? null]);
|
|
const covered = ranges.get(key) ?? new Set(); let added = 0;
|
|
for (let position = start; position < end; position++) if (!covered.has(position)) { covered.add(position); added++; }
|
|
ranges.set(key, covered); growth += added; if (!added) repeated++;
|
|
}
|
|
assert.equal(turn.delta.read_characters, growth); assert.equal(turn.delta.repeated_reads, repeated);
|
|
if (turn.phase === 'waiting') assert.equal(turn.classification, 'waiting');
|
|
}
|
|
}
|
|
const pages = snapshot('pages-original'), settled = pages.execution.run.progressShadow.turns.filter(row => row.phase === 'settled');
|
|
assert.equal(pages.execution.run.stopReason, 'step_limit'); assert.deepEqual(settled.map(row => [row.stepIndex, row.classification, row.delta.read_characters]),
|
|
[[1, 'evidence_growth', 10], [2, 'evidence_growth', 10], [3, 'repeated_read_candidate', 0]]);
|
|
const waiting = snapshot('approval-waiting'), denied = snapshot('approval-denied'), failure = snapshot('network-original');
|
|
assert.equal(waiting.session.status, 'awaiting_approval'); assert.equal(waiting.execution.run.progressShadow.turns.at(-1).waiting[0].kind, 'approval');
|
|
assert.equal(denied.session.taskId, pages.session.taskId); assert.notEqual(denied.execution.run.id, pages.execution.run.id);
|
|
assert.equal(denied.session.approvals.at(-1).status, 'denied'); assert.equal(denied.execution.run.progressShadow.turns.find(row => row.phase === 'settled').delta.repeated_reads, 0);
|
|
assert.equal(failure.session.status, 'failed'); assert.equal(failure.session.lastFailure.httpStatus, 502); assert.equal(failure.execution.run.stopReason, null);
|
|
assert.equal(failure.execution.run.progressShadow.turns.at(-1).classification, 'new_failure_evidence');
|
|
const report = json(directory + '/report.json'); assert.equal(report.turnObservations.length, 2); assert.ok(report.turnObservations.every(row => row.evidence_verified));
|
|
}
|
|
console.log(JSON.stringify({ ok: true, runs: manifest.observedRuns.length, snapshots, receipts, toolResults, originalArtifacts, realModelRequests: 0, replayedOperations: 0 }));
|