162 lines
15 KiB
JavaScript
162 lines
15 KiB
JavaScript
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import assert from 'node:assert/strict';
|
||
import { createHash } from 'node:crypto';
|
||
import { execFileSync } from 'node:child_process';
|
||
// Read-only historical verification: no server startup or model/catalog requests.
|
||
// The installed release, complete source test run and subsequent QA-only fixes
|
||
// retain distinct source identities and immutable original receipts.
|
||
const root = path.resolve('docs/model-catalog-1.25.3-evidence'), index = process.argv.includes('--index');
|
||
assert.ok(process.argv.slice(2).every(value => value === '--index'));
|
||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||
const blob = bytes => createHash('sha1').update(Buffer.from('blob ' + bytes.length + '\0')).update(bytes).digest('hex');
|
||
const bytes = relative => fs.readFileSync(path.join(root, relative));
|
||
const read = relative => JSON.parse(bytes(relative).toString('utf8').replace(/^\uFEFF/, ''));
|
||
const manifest = read('manifest.json'), names = new Set(), originalFiles = new Map(); let total = 0;
|
||
for (const row of manifest.entries) {
|
||
assert.ok(row.path && !path.isAbsolute(row.path) && !row.path.includes('\\') && !row.path.split('/').includes('..'));
|
||
assert.equal(names.has(row.path), false); names.add(row.path);
|
||
const value = bytes(row.path); assert.equal(value.length, row.bytes); assert.equal(hash(value), row.sha256, row.path); total += value.length;
|
||
originalFiles.set(path.resolve(row.source), row.path);
|
||
}
|
||
assert.equal(manifest.count, names.size); assert.equal(manifest.bytes, total);
|
||
const localOriginal = file => { const name = originalFiles.get(path.resolve(file)); assert.ok(name, file); return name; };
|
||
const freeze = read('release-verified/freeze.json'), candidate = read('release-verified/candidate.json'), identity = read('release-identity.json');
|
||
assert.equal(manifest.version, '1.25.3'); assert.equal(manifest.sourceCommit, freeze.sourceCommit); assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit);
|
||
assert.equal(identity.sourceCommit, freeze.sourceCommit); assert.equal(identity.candidate, path.basename(candidate.portableRoot));
|
||
assert.equal(hash(bytes('portable-manifest.json')), identity.portableManifestSha256); assert.equal(hash(bytes('source-manifest.json')), identity.sourceManifestSha256);
|
||
assert.equal(identity.formalPid, 36552); assert.deepEqual(identity.installer, candidate.installer);
|
||
const sourceTree = execFileSync('git', ['ls-tree', '-r', '-z', '--full-tree', freeze.sourceCommit], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }).split('\0').filter(Boolean);
|
||
const sourceBlobs = new Map(sourceTree.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[2]]; }));
|
||
for (const row of freeze.inputs) {
|
||
const value = bytes('source-bytes/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : '')); assert.equal(hash(value), row.sha256, row.path);
|
||
const oid = sourceBlobs.get(row.path); assert.ok(oid, row.path);
|
||
if (blob(value) !== oid) { assert.equal(value.includes(0), false); assert.equal(blob(Buffer.from(value.toString('utf8').replaceAll('\r\n', '\n'))), oid, row.path); }
|
||
}
|
||
for (const row of freeze.compiled) assert.equal(hash(bytes('compiled/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : ''))), row.sha256, row.path);
|
||
const groups = {
|
||
'source-reasoning': 6, 'source-reasoning-desktop': 6, 'source-catalog': 8, 'source-catalog-desktop': 8,
|
||
'source-a': 7, 'source-a-desktop': 3, 'source-b': 9, 'source-b-desktop': 9, 'source-c': 6, 'source-c-desktop': 6, 'source-d': 5, 'source-d-desktop': 5,
|
||
'portable-reasoning': 6, 'portable-reasoning-desktop': 6, 'portable-catalog': 8, 'portable-catalog-desktop': 8, 'portable-fidelity': 6, 'portable-messages': 7,
|
||
'portable-a': 7, 'portable-a-desktop': 3, 'portable-b': 9, 'portable-b-desktop': 9, 'portable-c': 6, 'portable-c-desktop': 6, 'portable-d': 5, 'portable-d-desktop': 5,
|
||
'portable-files': 9, 'portable-results': 16, 'portable-reading': 10,
|
||
};
|
||
const stages = ['rebuild', 'archives', 'portable', ...Object.keys(groups), 'install'];
|
||
const receipts = Object.fromEntries(stages.map(stage => {
|
||
const suffix = ['portable-fidelity', 'portable-messages'].includes(stage) ? '-reviewed' : '';
|
||
const receipt = read('release-verified/receipts/' + stage + suffix + '.json'); assert.equal(receipt.success, true, stage);
|
||
if (suffix) {
|
||
assert.equal(hash(bytes(localOriginal(receipt.resultPath))), receipt.result.metadataAdapter.originalResultSha256);
|
||
assert.equal(receipt.result.metadataAdapter.compiledFilesVerified, 22); assert.equal(receipt.result.metadataAdapter.originalResultUnmodified, true);
|
||
}
|
||
assert.equal(receipt.sourceCommit, freeze.sourceCommit); assert.equal(receipt.candidate, candidate.portableRoot); assert.equal(receipt.realModelRequests, 0);
|
||
assert.equal(hash(bytes('release-verified/logs/' + path.basename(receipt.log))), receipt.logSha256); assert.equal(receipt.result.success, true);
|
||
if (groups[stage]) {
|
||
assert.equal((receipt.result.checks ?? receipt.result.passed).length, groups[stage], stage);
|
||
assert.deepEqual(receipt.result.errors, []); assert.equal(receipt.result.realModelRequests, 0);
|
||
if (stage !== 'portable-reading') assert.equal(receipt.result.version, '1.25.3');
|
||
}
|
||
return [stage, receipt];
|
||
}));
|
||
const build = receipts.rebuild.result; assert.equal(build.inputCount, freeze.inputs.length + 5); assert.equal(build.compiled.length, 22); assert.deepEqual(build.mismatches, []);
|
||
for (const row of freeze.inputs) assert.equal(build.inputs.find(item => item.path === row.path)?.sha256, row.sha256);
|
||
for (const row of build.compiled) assert.deepEqual(row.actual, row.expected);
|
||
for (const kind of ['portable', 'source', 'evidence']) {
|
||
const archive = receipts.archives.result.archives.find(row => row.kind === kind);
|
||
assert.equal(archive.crcPassed, true); assert.equal(archive.manifestAndDirectoryMatch, true); assert.equal(archive.sha256, candidate[kind].sha256);
|
||
}
|
||
const installation = receipts.install.result; assert.equal(installation.results.length, 12); assert.ok(installation.results.every(row => row.passed));
|
||
const installedGroups = [16, 9, 7, 3, 9, 9, 6, 6, 5, 5, 6, 6, 8, 8]; assert.equal(installation.installedUI.proofs.length, installedGroups.length);
|
||
for (const [i, item] of installation.installedUI.proofs.entries()) {
|
||
const proof = item.proof; assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.deepEqual(proof.errors, []); assert.equal(proof.realModelRequests, 0);
|
||
assert.equal((proof.checks ?? proof.passed).length, installedGroups[i]); assert.equal(hash(bytes(localOriginal(item.log))), item.sha256);
|
||
}
|
||
const stats = {};
|
||
for (const name of ['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms']) {
|
||
const match = bytes('development/full-after-upgrade-fix.txt').toString('utf8').match(new RegExp('^[ℹ#] ' + name + ' ([0-9.]+)\\s*$', 'm'));
|
||
assert.ok(match, 'Complete test footer: ' + name); stats[name] = Number(match[1]);
|
||
}
|
||
assert.deepEqual(stats, identity.fullSuiteResult); assert.ok(stats.tests > 900); assert.equal(stats.pass, stats.tests);
|
||
for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[name], 0);
|
||
assert.equal(hash(bytes('development/full-after-upgrade-fix.txt')), identity.fullLogSha256);
|
||
assert.equal(read('development/preinstall-review.json').success, true);
|
||
const delta = read('development/candidate-input-delta.json');
|
||
assert.equal(delta.success, true); assert.equal(delta.before, identity.fullSuiteSourceCommit); assert.equal(delta.after, freeze.sourceCommit); assert.equal(delta.compiledExact, 22);
|
||
assert.deepEqual(delta.changes.map(row => row.path), ['scripts/reasoning-release.mjs', 'scripts/verify-controlled-release.mjs']);
|
||
const testedFreeze = read('release-fixed/freeze.json'); assert.equal(testedFreeze.sourceCommit, delta.before); assert.deepEqual(testedFreeze.compiled, freeze.compiled);
|
||
for (const row of freeze.inputs) {
|
||
const old = testedFreeze.inputs.find(item => item.path === row.path); assert.ok(old);
|
||
if (!delta.changes.some(item => item.path === row.path)) assert.equal(row.sha256, old.sha256, row.path);
|
||
}
|
||
const firstPortable = read('release-fixed/receipts/portable.json'); assert.equal(firstPortable.success, false);
|
||
assert.equal(hash(bytes('release-fixed/logs/portable.log')), firstPortable.logSha256);
|
||
assert.match(bytes('release-fixed/logs/portable.log').toString('utf8'), /includes\(expectedVersion\)/);
|
||
const oldMetadata = read('release-verified/receipts/portable-fidelity.json'); assert.equal(oldMetadata.success, false); assert.match(oldMetadata.error, /undefined/);
|
||
assert.equal(read('release-final/freeze.json').sourceCommit, 'bb4b497713894bdd9016e18a0009cbc333d2be93');
|
||
const originalFull = bytes('development/full-final.txt').toString('utf8');
|
||
for (const pattern of [/^ℹ tests 926\s*$/m, /^ℹ pass 923\s*$/m, /^ℹ fail 3\s*$/m, /^ℹ cancelled 0\s*$/m]) assert.match(originalFull, pattern);
|
||
assert.match(originalFull, /This migration accepts a supported existing Workbench/);
|
||
assert.equal(read('release/freeze.json').sourceCommit.startsWith('109e5124'), true);
|
||
assert.ok(bytes('development/full-source.txt').length > 0); assert.ok(names.has('development/full-source-cancelled-scope.json'));
|
||
const developmentCatalogFailures = [...names].filter(name => name.startsWith('validation/development-catalog-web-') && name.endsWith('/result.json')).filter(name => read(name).success === false);
|
||
assert.ok(developmentCatalogFailures.length > 0, 'Preserve the first UI assertion failure.');
|
||
const before = read('formal/state-before.json');
|
||
for (const stage of ['upgrade', 'final']) {
|
||
const proof = read('formal/' + stage + '.json'), state = read('formal/' + stage + '-state-after.json');
|
||
assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.equal(proof.pid, 36552); assert.equal(proof.inspectorClosed, true); assert.equal(proof.desktopWindows, 0);
|
||
for (const key of ['pid', 'hasKey', 'configSha256', 'projectsSha256', 'sessionsSha256', 'eventCursor', 'exports', 'persistent', 'modelCenterSha256']) assert.deepEqual(state[key], before[key], key);
|
||
assert.deepEqual(state.database.tables, before.database.tables); assert.deepEqual(state.database.sessionIds, before.database.sessionIds);
|
||
}
|
||
assert.equal(read('formal/installer-exit.json').code, 0);
|
||
for (const file of ['formal-web/result.json', 'formal-native/result.json']) {
|
||
const proof = read(file); assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.equal(proof.pid ?? proof.formalPid, 36552); assert.equal(proof.realModelRequests, 0);
|
||
}
|
||
assert.equal(read('formal-web/result.json').realCatalogRequests, 0); assert.deepEqual(read('formal-web/result.json').rejectedWrites, []);
|
||
const native = read('formal-native/result.json');
|
||
assert.equal(native.realCatalogRequests, 0); assert.equal(native.configurationSaved, false); assert.equal(native.keyReadOrResaved, false);
|
||
assert.equal(native.closedNativeWindows, true); assert.equal(read('formal-native/launch.json').code, 0);
|
||
for (const stage of ['01-task', '02-model-center', '03-upstream-form', '04-deepseek-reasoning']) {
|
||
const observation = read('formal-native/' + stage + '.json');
|
||
assert.ok(observation.screenshots.length > 0);
|
||
for (const screenshot of observation.screenshots) {
|
||
assert.ok(['image/jpeg', 'image/png'].includes(screenshot.mime));
|
||
assert.ok(names.has('formal-native/' + screenshot.file));
|
||
const value = bytes('formal-native/' + screenshot.file);
|
||
if (screenshot.mime === 'image/jpeg') assert.equal(value.subarray(0, 3).toString('hex'), 'ffd8ff');
|
||
else assert.equal(value.subarray(0, 8).toString('hex'), '89504e470d0a1a0a');
|
||
}
|
||
}
|
||
const identityPath = 'docs/model-catalog-1.25.3-release-identity.json';
|
||
assert.equal(hash(fs.readFileSync(identityPath)), hash(bytes('release-identity.json')));
|
||
const followRoot = path.resolve('docs/model-catalog-1.25.3-qa-follow-up');
|
||
const followBytes = name => fs.readFileSync(path.join(followRoot, name));
|
||
const followRead = name => JSON.parse(followBytes(name));
|
||
const followManifest = followRead('manifest.json'), followNames = new Set(); let followTotal = 0;
|
||
for (const row of followManifest.entries) {
|
||
assert.equal(path.basename(row.path), row.path); assert.equal(followNames.has(row.path), false); followNames.add(row.path);
|
||
const value = followBytes(row.path); assert.equal(value.length, row.bytes); assert.equal(hash(value), row.sha256, row.path); followTotal += value.length;
|
||
}
|
||
assert.equal(followManifest.count, followNames.size); assert.equal(followManifest.bytes, followTotal);
|
||
const follow = followRead('result.json'), fidelity = followRead('fidelity-result.json');
|
||
assert.equal(follow.success, true); assert.equal(follow.candidateSourceCommit, freeze.sourceCommit); assert.equal(follow.candidate, identity.candidate);
|
||
assert.equal(follow.sealedManifestUnmodified, hash(bytes('manifest.json'))); assert.equal(follow.observedHealthVersion, '1.25.3');
|
||
assert.equal(follow.realModelRequests, 0); assert.equal(follow.realCatalogRequests, 0); assert.equal(follow.actualVersionGateChecks.length, 6);
|
||
assert.equal(fidelity.success, true); assert.equal(fidelity.version, '1.25.3'); assert.equal(fidelity.checks.length, 6); assert.deepEqual(fidelity.errors, []);
|
||
assert.equal(fidelity.realModelRequests, 0); assert.equal(fidelity.screenshots.length, 10);
|
||
for (const screenshot of fidelity.screenshots) assert.equal(hash(followBytes(path.basename(screenshot.path))), screenshot.sha256);
|
||
assert.equal(follow.originalMessagesResultSha256, receipts['portable-messages'].result.metadataAdapter.originalResultSha256);
|
||
let indexedFiles = 0, indexedBytes = 0;
|
||
if (index) {
|
||
const rows = execFileSync('git', ['ls-files', '--stage', '-z', '--', 'docs/model-catalog-1.25.3-evidence', 'docs/model-catalog-1.25.3-qa-follow-up', identityPath], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }).split('\0').filter(Boolean);
|
||
const blobs = new Map(rows.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[1]]; }));
|
||
function checkIndex(file, value) { assert.equal(blobs.get(file), blob(value), file + ' indexed bytes'); indexedFiles++; indexedBytes += value.length; }
|
||
for (const name of [...names, 'manifest.json']) checkIndex('docs/model-catalog-1.25.3-evidence/' + name, bytes(name));
|
||
for (const name of [...followNames, 'manifest.json']) checkIndex('docs/model-catalog-1.25.3-qa-follow-up/' + name, followBytes(name));
|
||
checkIndex(identityPath, fs.readFileSync(identityPath));
|
||
}
|
||
console.log(JSON.stringify({ success: true, files: names.size, bytes: total, sourceCommit: freeze.sourceCommit, inputs: freeze.inputs.length,
|
||
independentInputs: build.inputCount, compiled: 22, fullSuitePassed: stats.pass, fullSuiteFailed: stats.fail,
|
||
sourceUIGroups: identity.sourceUIGroups, portableUIGroups: identity.portableUIGroups, isolatedInstallGroups: 12, installedUIGroups: 103,
|
||
firstFailuresPreserved: true, realModelRequests: 0, realCatalogRequests: 0, indexChecked: index,
|
||
qaFollowUpFiles: followNames.size, qaFollowUpBytes: followTotal, qaFollowUpUIGroups: fidelity.checks.length, indexedFiles, indexedBytes }));
|