43 lines
3.2 KiB
JavaScript
43 lines
3.2 KiB
JavaScript
// Verify already-saved preflight snapshots; no host/model request or config write.
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { root, save, sha, experiment } from './common.mjs';
|
|
const read = file => JSON.parse(fs.readFileSync(path.join(root, file)));
|
|
const before = read('preflight/before.json'), after = read('preflight/after.json');
|
|
const result = read('preflight/result.json'), owned = read('preflight/owned.json');
|
|
const attempts = read('preflight/attempts.json'), session = read('preflight/session.json');
|
|
for (const field of ['pid', 'version', 'runtimeRoot', 'hasKey', 'configSha256']) assert.equal(after[field], before[field], field);
|
|
assert.deepEqual(before.active, []); assert.deepEqual(after.active, []);
|
|
assert.equal(after.projects, before.projects + 1); assert.equal(after.exports.length, before.exports.length + 1);
|
|
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item);
|
|
for (const table of ['sessions', 'records', 'events', 'outbox']) {
|
|
assert.equal(after.database.tables[table].schemaSha256, before.database.tables[table].schemaSha256);
|
|
const rows = new Map(after.database.tables[table].rows.map(value => [value.id, value]));
|
|
for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, table + '/' + row.id);
|
|
}
|
|
const metadata = new Map(after.database.tables.metadata.rows.map(value => [value.id, value]));
|
|
for (const row of before.database.tables.metadata.rows.filter(value => value.id !== 'modelAttempts')) {
|
|
assert.deepEqual(metadata.get(row.id), row, 'No preflight config or diagnostic write: ' + row.id);
|
|
}
|
|
for (const row of before.persistent.artifacts) assert.deepEqual(after.persistent.artifacts.find(value => value.path === row.path), row);
|
|
assert.deepEqual(after.modelCenter.selected, before.modelCenter.selected);
|
|
assert.deepEqual(after.modelCenter.pools, before.modelCenter.pools);
|
|
assert.equal(attempts.length, 1); assert.equal(result.requests, 1);
|
|
assert.equal(attempts[0].sessionId, owned.session.id); assert.equal(attempts[0].poolId, null); assert.equal(attempts[0].ordinal, 1);
|
|
assert.equal(attempts[0].model, experiment.model); assert.equal(attempts[0].protocol, experiment.protocol);
|
|
assert.equal(result.actualTools, 0); assert.equal(session.execution.run.toolCalls, 0);
|
|
assert.equal(read('preflight/inspector-final.json').closed, true);
|
|
for (const capture of result.captures) {
|
|
const bytes = fs.readFileSync(path.join(root, capture.path));
|
|
assert.equal(bytes.length, capture.bytes); assert.equal(sha(bytes), capture.sha256);
|
|
}
|
|
const verification = { passed: true, route: experiment.name, engineeringAllowed: result.passed,
|
|
originalExportsExact: before.exports.length, originalArtifactFilesExact: before.persistent.artifacts.length,
|
|
originalRowsExact: ['sessions', 'records', 'events', 'outbox'], publicMetadataExactExceptOneNewAttempt: true,
|
|
configSelectionPoolAndCredentialsUnchanged: true, originalPidPreserved: after.pid, inspectorClosed: true,
|
|
preflightRequests: 1, preflightReportedTokens: result.usage.total_tokens, toolsExecuted: 0,
|
|
verificationModelRequests: 0, stopReason: result.stopReason };
|
|
save('preflight/verification.json', verification);
|
|
console.log(JSON.stringify(verification));
|