Files
Agent-Workbench/scripts/verify-model-catalog-evidence.mjs

162 lines
15 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
// Read-only historical verification: no server startup or model/catalog requests.
// The installed release, complete source test run and subsequent QA-only fixes
// retain distinct source identities and immutable original receipts.
const root = path.resolve('docs/model-catalog-1.25.3-evidence'), index = process.argv.includes('--index');
assert.ok(process.argv.slice(2).every(value => value === '--index'));
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const blob = bytes => createHash('sha1').update(Buffer.from('blob ' + bytes.length + '\0')).update(bytes).digest('hex');
const bytes = relative => fs.readFileSync(path.join(root, relative));
const read = relative => JSON.parse(bytes(relative).toString('utf8').replace(/^\uFEFF/, ''));
const manifest = read('manifest.json'), names = new Set(), originalFiles = new Map(); let total = 0;
for (const row of manifest.entries) {
assert.ok(row.path && !path.isAbsolute(row.path) && !row.path.includes('\\') && !row.path.split('/').includes('..'));
assert.equal(names.has(row.path), false); names.add(row.path);
const value = bytes(row.path); assert.equal(value.length, row.bytes); assert.equal(hash(value), row.sha256, row.path); total += value.length;
originalFiles.set(path.resolve(row.source), row.path);
}
assert.equal(manifest.count, names.size); assert.equal(manifest.bytes, total);
const localOriginal = file => { const name = originalFiles.get(path.resolve(file)); assert.ok(name, file); return name; };
const freeze = read('release-verified/freeze.json'), candidate = read('release-verified/candidate.json'), identity = read('release-identity.json');
assert.equal(manifest.version, '1.25.3'); assert.equal(manifest.sourceCommit, freeze.sourceCommit); assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit);
assert.equal(identity.sourceCommit, freeze.sourceCommit); assert.equal(identity.candidate, path.basename(candidate.portableRoot));
assert.equal(hash(bytes('portable-manifest.json')), identity.portableManifestSha256); assert.equal(hash(bytes('source-manifest.json')), identity.sourceManifestSha256);
assert.equal(identity.formalPid, 36552); assert.deepEqual(identity.installer, candidate.installer);
const sourceTree = execFileSync('git', ['ls-tree', '-r', '-z', '--full-tree', freeze.sourceCommit], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }).split('\0').filter(Boolean);
const sourceBlobs = new Map(sourceTree.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[2]]; }));
for (const row of freeze.inputs) {
const value = bytes('source-bytes/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : '')); assert.equal(hash(value), row.sha256, row.path);
const oid = sourceBlobs.get(row.path); assert.ok(oid, row.path);
if (blob(value) !== oid) { assert.equal(value.includes(0), false); assert.equal(blob(Buffer.from(value.toString('utf8').replaceAll('\r\n', '\n'))), oid, row.path); }
}
for (const row of freeze.compiled) assert.equal(hash(bytes('compiled/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : ''))), row.sha256, row.path);
const groups = {
'source-reasoning': 6, 'source-reasoning-desktop': 6, 'source-catalog': 8, 'source-catalog-desktop': 8,
'source-a': 7, 'source-a-desktop': 3, 'source-b': 9, 'source-b-desktop': 9, 'source-c': 6, 'source-c-desktop': 6, 'source-d': 5, 'source-d-desktop': 5,
'portable-reasoning': 6, 'portable-reasoning-desktop': 6, 'portable-catalog': 8, 'portable-catalog-desktop': 8, 'portable-fidelity': 6, 'portable-messages': 7,
'portable-a': 7, 'portable-a-desktop': 3, 'portable-b': 9, 'portable-b-desktop': 9, 'portable-c': 6, 'portable-c-desktop': 6, 'portable-d': 5, 'portable-d-desktop': 5,
'portable-files': 9, 'portable-results': 16, 'portable-reading': 10,
};
const stages = ['rebuild', 'archives', 'portable', ...Object.keys(groups), 'install'];
const receipts = Object.fromEntries(stages.map(stage => {
const suffix = ['portable-fidelity', 'portable-messages'].includes(stage) ? '-reviewed' : '';
const receipt = read('release-verified/receipts/' + stage + suffix + '.json'); assert.equal(receipt.success, true, stage);
if (suffix) {
assert.equal(hash(bytes(localOriginal(receipt.resultPath))), receipt.result.metadataAdapter.originalResultSha256);
assert.equal(receipt.result.metadataAdapter.compiledFilesVerified, 22); assert.equal(receipt.result.metadataAdapter.originalResultUnmodified, true);
}
assert.equal(receipt.sourceCommit, freeze.sourceCommit); assert.equal(receipt.candidate, candidate.portableRoot); assert.equal(receipt.realModelRequests, 0);
assert.equal(hash(bytes('release-verified/logs/' + path.basename(receipt.log))), receipt.logSha256); assert.equal(receipt.result.success, true);
if (groups[stage]) {
assert.equal((receipt.result.checks ?? receipt.result.passed).length, groups[stage], stage);
assert.deepEqual(receipt.result.errors, []); assert.equal(receipt.result.realModelRequests, 0);
if (stage !== 'portable-reading') assert.equal(receipt.result.version, '1.25.3');
}
return [stage, receipt];
}));
const build = receipts.rebuild.result; assert.equal(build.inputCount, freeze.inputs.length + 5); assert.equal(build.compiled.length, 22); assert.deepEqual(build.mismatches, []);
for (const row of freeze.inputs) assert.equal(build.inputs.find(item => item.path === row.path)?.sha256, row.sha256);
for (const row of build.compiled) assert.deepEqual(row.actual, row.expected);
for (const kind of ['portable', 'source', 'evidence']) {
const archive = receipts.archives.result.archives.find(row => row.kind === kind);
assert.equal(archive.crcPassed, true); assert.equal(archive.manifestAndDirectoryMatch, true); assert.equal(archive.sha256, candidate[kind].sha256);
}
const installation = receipts.install.result; assert.equal(installation.results.length, 12); assert.ok(installation.results.every(row => row.passed));
const installedGroups = [16, 9, 7, 3, 9, 9, 6, 6, 5, 5, 6, 6, 8, 8]; assert.equal(installation.installedUI.proofs.length, installedGroups.length);
for (const [i, item] of installation.installedUI.proofs.entries()) {
const proof = item.proof; assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.deepEqual(proof.errors, []); assert.equal(proof.realModelRequests, 0);
assert.equal((proof.checks ?? proof.passed).length, installedGroups[i]); assert.equal(hash(bytes(localOriginal(item.log))), item.sha256);
}
const stats = {};
for (const name of ['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms']) {
const match = bytes('development/full-after-upgrade-fix.txt').toString('utf8').match(new RegExp('^[ℹ#] ' + name + ' ([0-9.]+)\\s*$', 'm'));
assert.ok(match, 'Complete test footer: ' + name); stats[name] = Number(match[1]);
}
assert.deepEqual(stats, identity.fullSuiteResult); assert.ok(stats.tests > 900); assert.equal(stats.pass, stats.tests);
for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[name], 0);
assert.equal(hash(bytes('development/full-after-upgrade-fix.txt')), identity.fullLogSha256);
assert.equal(read('development/preinstall-review.json').success, true);
const delta = read('development/candidate-input-delta.json');
assert.equal(delta.success, true); assert.equal(delta.before, identity.fullSuiteSourceCommit); assert.equal(delta.after, freeze.sourceCommit); assert.equal(delta.compiledExact, 22);
assert.deepEqual(delta.changes.map(row => row.path), ['scripts/reasoning-release.mjs', 'scripts/verify-controlled-release.mjs']);
const testedFreeze = read('release-fixed/freeze.json'); assert.equal(testedFreeze.sourceCommit, delta.before); assert.deepEqual(testedFreeze.compiled, freeze.compiled);
for (const row of freeze.inputs) {
const old = testedFreeze.inputs.find(item => item.path === row.path); assert.ok(old);
if (!delta.changes.some(item => item.path === row.path)) assert.equal(row.sha256, old.sha256, row.path);
}
const firstPortable = read('release-fixed/receipts/portable.json'); assert.equal(firstPortable.success, false);
assert.equal(hash(bytes('release-fixed/logs/portable.log')), firstPortable.logSha256);
assert.match(bytes('release-fixed/logs/portable.log').toString('utf8'), /includes\(expectedVersion\)/);
const oldMetadata = read('release-verified/receipts/portable-fidelity.json'); assert.equal(oldMetadata.success, false); assert.match(oldMetadata.error, /undefined/);
assert.equal(read('release-final/freeze.json').sourceCommit, 'bb4b497713894bdd9016e18a0009cbc333d2be93');
const originalFull = bytes('development/full-final.txt').toString('utf8');
for (const pattern of [/^ℹ tests 926\s*$/m, /^ℹ pass 923\s*$/m, /^ℹ fail 3\s*$/m, /^ℹ cancelled 0\s*$/m]) assert.match(originalFull, pattern);
assert.match(originalFull, /This migration accepts a supported existing Workbench/);
assert.equal(read('release/freeze.json').sourceCommit.startsWith('109e5124'), true);
assert.ok(bytes('development/full-source.txt').length > 0); assert.ok(names.has('development/full-source-cancelled-scope.json'));
const developmentCatalogFailures = [...names].filter(name => name.startsWith('validation/development-catalog-web-') && name.endsWith('/result.json')).filter(name => read(name).success === false);
assert.ok(developmentCatalogFailures.length > 0, 'Preserve the first UI assertion failure.');
const before = read('formal/state-before.json');
for (const stage of ['upgrade', 'final']) {
const proof = read('formal/' + stage + '.json'), state = read('formal/' + stage + '-state-after.json');
assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.equal(proof.pid, 36552); assert.equal(proof.inspectorClosed, true); assert.equal(proof.desktopWindows, 0);
for (const key of ['pid', 'hasKey', 'configSha256', 'projectsSha256', 'sessionsSha256', 'eventCursor', 'exports', 'persistent', 'modelCenterSha256']) assert.deepEqual(state[key], before[key], key);
assert.deepEqual(state.database.tables, before.database.tables); assert.deepEqual(state.database.sessionIds, before.database.sessionIds);
}
assert.equal(read('formal/installer-exit.json').code, 0);
for (const file of ['formal-web/result.json', 'formal-native/result.json']) {
const proof = read(file); assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.equal(proof.pid ?? proof.formalPid, 36552); assert.equal(proof.realModelRequests, 0);
}
assert.equal(read('formal-web/result.json').realCatalogRequests, 0); assert.deepEqual(read('formal-web/result.json').rejectedWrites, []);
const native = read('formal-native/result.json');
assert.equal(native.realCatalogRequests, 0); assert.equal(native.configurationSaved, false); assert.equal(native.keyReadOrResaved, false);
assert.equal(native.closedNativeWindows, true); assert.equal(read('formal-native/launch.json').code, 0);
for (const stage of ['01-task', '02-model-center', '03-upstream-form', '04-deepseek-reasoning']) {
const observation = read('formal-native/' + stage + '.json');
assert.ok(observation.screenshots.length > 0);
for (const screenshot of observation.screenshots) {
assert.ok(['image/jpeg', 'image/png'].includes(screenshot.mime));
assert.ok(names.has('formal-native/' + screenshot.file));
const value = bytes('formal-native/' + screenshot.file);
if (screenshot.mime === 'image/jpeg') assert.equal(value.subarray(0, 3).toString('hex'), 'ffd8ff');
else assert.equal(value.subarray(0, 8).toString('hex'), '89504e470d0a1a0a');
}
}
const identityPath = 'docs/model-catalog-1.25.3-release-identity.json';
assert.equal(hash(fs.readFileSync(identityPath)), hash(bytes('release-identity.json')));
const followRoot = path.resolve('docs/model-catalog-1.25.3-qa-follow-up');
const followBytes = name => fs.readFileSync(path.join(followRoot, name));
const followRead = name => JSON.parse(followBytes(name));
const followManifest = followRead('manifest.json'), followNames = new Set(); let followTotal = 0;
for (const row of followManifest.entries) {
assert.equal(path.basename(row.path), row.path); assert.equal(followNames.has(row.path), false); followNames.add(row.path);
const value = followBytes(row.path); assert.equal(value.length, row.bytes); assert.equal(hash(value), row.sha256, row.path); followTotal += value.length;
}
assert.equal(followManifest.count, followNames.size); assert.equal(followManifest.bytes, followTotal);
const follow = followRead('result.json'), fidelity = followRead('fidelity-result.json');
assert.equal(follow.success, true); assert.equal(follow.candidateSourceCommit, freeze.sourceCommit); assert.equal(follow.candidate, identity.candidate);
assert.equal(follow.sealedManifestUnmodified, hash(bytes('manifest.json'))); assert.equal(follow.observedHealthVersion, '1.25.3');
assert.equal(follow.realModelRequests, 0); assert.equal(follow.realCatalogRequests, 0); assert.equal(follow.actualVersionGateChecks.length, 6);
assert.equal(fidelity.success, true); assert.equal(fidelity.version, '1.25.3'); assert.equal(fidelity.checks.length, 6); assert.deepEqual(fidelity.errors, []);
assert.equal(fidelity.realModelRequests, 0); assert.equal(fidelity.screenshots.length, 10);
for (const screenshot of fidelity.screenshots) assert.equal(hash(followBytes(path.basename(screenshot.path))), screenshot.sha256);
assert.equal(follow.originalMessagesResultSha256, receipts['portable-messages'].result.metadataAdapter.originalResultSha256);
let indexedFiles = 0, indexedBytes = 0;
if (index) {
const rows = execFileSync('git', ['ls-files', '--stage', '-z', '--', 'docs/model-catalog-1.25.3-evidence', 'docs/model-catalog-1.25.3-qa-follow-up', identityPath], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }).split('\0').filter(Boolean);
const blobs = new Map(rows.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[1]]; }));
function checkIndex(file, value) { assert.equal(blobs.get(file), blob(value), file + ' indexed bytes'); indexedFiles++; indexedBytes += value.length; }
for (const name of [...names, 'manifest.json']) checkIndex('docs/model-catalog-1.25.3-evidence/' + name, bytes(name));
for (const name of [...followNames, 'manifest.json']) checkIndex('docs/model-catalog-1.25.3-qa-follow-up/' + name, followBytes(name));
checkIndex(identityPath, fs.readFileSync(identityPath));
}
console.log(JSON.stringify({ success: true, files: names.size, bytes: total, sourceCommit: freeze.sourceCommit, inputs: freeze.inputs.length,
independentInputs: build.inputCount, compiled: 22, fullSuitePassed: stats.pass, fullSuiteFailed: stats.fail,
sourceUIGroups: identity.sourceUIGroups, portableUIGroups: identity.portableUIGroups, isolatedInstallGroups: 12, installedUIGroups: 103,
firstFailuresPreserved: true, realModelRequests: 0, realCatalogRequests: 0, indexChecked: index,
qaFollowUpFiles: followNames.size, qaFollowUpBytes: followTotal, qaFollowUpUIGroups: fidelity.checks.length, indexedFiles, indexedBytes }));