import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; // Read-only historical verification: no server startup or model/catalog requests. // The installed release, complete source test run and subsequent QA-only fixes // retain distinct source identities and immutable original receipts. const root = path.resolve('docs/model-catalog-1.25.3-evidence'), index = process.argv.includes('--index'); assert.ok(process.argv.slice(2).every(value => value === '--index')); const hash = bytes => createHash('sha256').update(bytes).digest('hex'); const blob = bytes => createHash('sha1').update(Buffer.from('blob ' + bytes.length + '\0')).update(bytes).digest('hex'); const bytes = relative => fs.readFileSync(path.join(root, relative)); const read = relative => JSON.parse(bytes(relative).toString('utf8').replace(/^\uFEFF/, '')); const manifest = read('manifest.json'), names = new Set(), originalFiles = new Map(); let total = 0; for (const row of manifest.entries) { assert.ok(row.path && !path.isAbsolute(row.path) && !row.path.includes('\\') && !row.path.split('/').includes('..')); assert.equal(names.has(row.path), false); names.add(row.path); const value = bytes(row.path); assert.equal(value.length, row.bytes); assert.equal(hash(value), row.sha256, row.path); total += value.length; originalFiles.set(path.resolve(row.source), row.path); } assert.equal(manifest.count, names.size); assert.equal(manifest.bytes, total); const localOriginal = file => { const name = originalFiles.get(path.resolve(file)); assert.ok(name, file); return name; }; const freeze = read('release-verified/freeze.json'), candidate = read('release-verified/candidate.json'), identity = read('release-identity.json'); assert.equal(manifest.version, '1.25.3'); assert.equal(manifest.sourceCommit, freeze.sourceCommit); assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit); assert.equal(identity.sourceCommit, freeze.sourceCommit); assert.equal(identity.candidate, path.basename(candidate.portableRoot)); assert.equal(hash(bytes('portable-manifest.json')), identity.portableManifestSha256); assert.equal(hash(bytes('source-manifest.json')), identity.sourceManifestSha256); assert.equal(identity.formalPid, 36552); assert.deepEqual(identity.installer, candidate.installer); const sourceTree = execFileSync('git', ['ls-tree', '-r', '-z', '--full-tree', freeze.sourceCommit], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }).split('\0').filter(Boolean); const sourceBlobs = new Map(sourceTree.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[2]]; })); for (const row of freeze.inputs) { const value = bytes('source-bytes/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : '')); assert.equal(hash(value), row.sha256, row.path); const oid = sourceBlobs.get(row.path); assert.ok(oid, row.path); if (blob(value) !== oid) { assert.equal(value.includes(0), false); assert.equal(blob(Buffer.from(value.toString('utf8').replaceAll('\r\n', '\n'))), oid, row.path); } } for (const row of freeze.compiled) assert.equal(hash(bytes('compiled/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : ''))), row.sha256, row.path); const groups = { 'source-reasoning': 6, 'source-reasoning-desktop': 6, 'source-catalog': 8, 'source-catalog-desktop': 8, 'source-a': 7, 'source-a-desktop': 3, 'source-b': 9, 'source-b-desktop': 9, 'source-c': 6, 'source-c-desktop': 6, 'source-d': 5, 'source-d-desktop': 5, 'portable-reasoning': 6, 'portable-reasoning-desktop': 6, 'portable-catalog': 8, 'portable-catalog-desktop': 8, 'portable-fidelity': 6, 'portable-messages': 7, 'portable-a': 7, 'portable-a-desktop': 3, 'portable-b': 9, 'portable-b-desktop': 9, 'portable-c': 6, 'portable-c-desktop': 6, 'portable-d': 5, 'portable-d-desktop': 5, 'portable-files': 9, 'portable-results': 16, 'portable-reading': 10, }; const stages = ['rebuild', 'archives', 'portable', ...Object.keys(groups), 'install']; const receipts = Object.fromEntries(stages.map(stage => { const suffix = ['portable-fidelity', 'portable-messages'].includes(stage) ? '-reviewed' : ''; const receipt = read('release-verified/receipts/' + stage + suffix + '.json'); assert.equal(receipt.success, true, stage); if (suffix) { assert.equal(hash(bytes(localOriginal(receipt.resultPath))), receipt.result.metadataAdapter.originalResultSha256); assert.equal(receipt.result.metadataAdapter.compiledFilesVerified, 22); assert.equal(receipt.result.metadataAdapter.originalResultUnmodified, true); } assert.equal(receipt.sourceCommit, freeze.sourceCommit); assert.equal(receipt.candidate, candidate.portableRoot); assert.equal(receipt.realModelRequests, 0); assert.equal(hash(bytes('release-verified/logs/' + path.basename(receipt.log))), receipt.logSha256); assert.equal(receipt.result.success, true); if (groups[stage]) { assert.equal((receipt.result.checks ?? receipt.result.passed).length, groups[stage], stage); assert.deepEqual(receipt.result.errors, []); assert.equal(receipt.result.realModelRequests, 0); if (stage !== 'portable-reading') assert.equal(receipt.result.version, '1.25.3'); } return [stage, receipt]; })); const build = receipts.rebuild.result; assert.equal(build.inputCount, freeze.inputs.length + 5); assert.equal(build.compiled.length, 22); assert.deepEqual(build.mismatches, []); for (const row of freeze.inputs) assert.equal(build.inputs.find(item => item.path === row.path)?.sha256, row.sha256); for (const row of build.compiled) assert.deepEqual(row.actual, row.expected); for (const kind of ['portable', 'source', 'evidence']) { const archive = receipts.archives.result.archives.find(row => row.kind === kind); assert.equal(archive.crcPassed, true); assert.equal(archive.manifestAndDirectoryMatch, true); assert.equal(archive.sha256, candidate[kind].sha256); } const installation = receipts.install.result; assert.equal(installation.results.length, 12); assert.ok(installation.results.every(row => row.passed)); const installedGroups = [16, 9, 7, 3, 9, 9, 6, 6, 5, 5, 6, 6, 8, 8]; assert.equal(installation.installedUI.proofs.length, installedGroups.length); for (const [i, item] of installation.installedUI.proofs.entries()) { const proof = item.proof; assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.deepEqual(proof.errors, []); assert.equal(proof.realModelRequests, 0); assert.equal((proof.checks ?? proof.passed).length, installedGroups[i]); assert.equal(hash(bytes(localOriginal(item.log))), item.sha256); } const stats = {}; for (const name of ['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms']) { const match = bytes('development/full-after-upgrade-fix.txt').toString('utf8').match(new RegExp('^[ℹ#] ' + name + ' ([0-9.]+)\\s*$', 'm')); assert.ok(match, 'Complete test footer: ' + name); stats[name] = Number(match[1]); } assert.deepEqual(stats, identity.fullSuiteResult); assert.ok(stats.tests > 900); assert.equal(stats.pass, stats.tests); for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[name], 0); assert.equal(hash(bytes('development/full-after-upgrade-fix.txt')), identity.fullLogSha256); assert.equal(read('development/preinstall-review.json').success, true); const delta = read('development/candidate-input-delta.json'); assert.equal(delta.success, true); assert.equal(delta.before, identity.fullSuiteSourceCommit); assert.equal(delta.after, freeze.sourceCommit); assert.equal(delta.compiledExact, 22); assert.deepEqual(delta.changes.map(row => row.path), ['scripts/reasoning-release.mjs', 'scripts/verify-controlled-release.mjs']); const testedFreeze = read('release-fixed/freeze.json'); assert.equal(testedFreeze.sourceCommit, delta.before); assert.deepEqual(testedFreeze.compiled, freeze.compiled); for (const row of freeze.inputs) { const old = testedFreeze.inputs.find(item => item.path === row.path); assert.ok(old); if (!delta.changes.some(item => item.path === row.path)) assert.equal(row.sha256, old.sha256, row.path); } const firstPortable = read('release-fixed/receipts/portable.json'); assert.equal(firstPortable.success, false); assert.equal(hash(bytes('release-fixed/logs/portable.log')), firstPortable.logSha256); assert.match(bytes('release-fixed/logs/portable.log').toString('utf8'), /includes\(expectedVersion\)/); const oldMetadata = read('release-verified/receipts/portable-fidelity.json'); assert.equal(oldMetadata.success, false); assert.match(oldMetadata.error, /undefined/); assert.equal(read('release-final/freeze.json').sourceCommit, 'bb4b497713894bdd9016e18a0009cbc333d2be93'); const originalFull = bytes('development/full-final.txt').toString('utf8'); for (const pattern of [/^ℹ tests 926\s*$/m, /^ℹ pass 923\s*$/m, /^ℹ fail 3\s*$/m, /^ℹ cancelled 0\s*$/m]) assert.match(originalFull, pattern); assert.match(originalFull, /This migration accepts a supported existing Workbench/); assert.equal(read('release/freeze.json').sourceCommit.startsWith('109e5124'), true); assert.ok(bytes('development/full-source.txt').length > 0); assert.ok(names.has('development/full-source-cancelled-scope.json')); const developmentCatalogFailures = [...names].filter(name => name.startsWith('validation/development-catalog-web-') && name.endsWith('/result.json')).filter(name => read(name).success === false); assert.ok(developmentCatalogFailures.length > 0, 'Preserve the first UI assertion failure.'); const before = read('formal/state-before.json'); for (const stage of ['upgrade', 'final']) { const proof = read('formal/' + stage + '.json'), state = read('formal/' + stage + '-state-after.json'); assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.equal(proof.pid, 36552); assert.equal(proof.inspectorClosed, true); assert.equal(proof.desktopWindows, 0); for (const key of ['pid', 'hasKey', 'configSha256', 'projectsSha256', 'sessionsSha256', 'eventCursor', 'exports', 'persistent', 'modelCenterSha256']) assert.deepEqual(state[key], before[key], key); assert.deepEqual(state.database.tables, before.database.tables); assert.deepEqual(state.database.sessionIds, before.database.sessionIds); } assert.equal(read('formal/installer-exit.json').code, 0); for (const file of ['formal-web/result.json', 'formal-native/result.json']) { const proof = read(file); assert.equal(proof.success, true); assert.equal(proof.version, '1.25.3'); assert.equal(proof.pid ?? proof.formalPid, 36552); assert.equal(proof.realModelRequests, 0); } assert.equal(read('formal-web/result.json').realCatalogRequests, 0); assert.deepEqual(read('formal-web/result.json').rejectedWrites, []); const native = read('formal-native/result.json'); assert.equal(native.realCatalogRequests, 0); assert.equal(native.configurationSaved, false); assert.equal(native.keyReadOrResaved, false); assert.equal(native.closedNativeWindows, true); assert.equal(read('formal-native/launch.json').code, 0); for (const stage of ['01-task', '02-model-center', '03-upstream-form', '04-deepseek-reasoning']) { const observation = read('formal-native/' + stage + '.json'); assert.ok(observation.screenshots.length > 0); for (const screenshot of observation.screenshots) { assert.ok(['image/jpeg', 'image/png'].includes(screenshot.mime)); assert.ok(names.has('formal-native/' + screenshot.file)); const value = bytes('formal-native/' + screenshot.file); if (screenshot.mime === 'image/jpeg') assert.equal(value.subarray(0, 3).toString('hex'), 'ffd8ff'); else assert.equal(value.subarray(0, 8).toString('hex'), '89504e470d0a1a0a'); } } const identityPath = 'docs/model-catalog-1.25.3-release-identity.json'; assert.equal(hash(fs.readFileSync(identityPath)), hash(bytes('release-identity.json'))); const followRoot = path.resolve('docs/model-catalog-1.25.3-qa-follow-up'); const followBytes = name => fs.readFileSync(path.join(followRoot, name)); const followRead = name => JSON.parse(followBytes(name)); const followManifest = followRead('manifest.json'), followNames = new Set(); let followTotal = 0; for (const row of followManifest.entries) { assert.equal(path.basename(row.path), row.path); assert.equal(followNames.has(row.path), false); followNames.add(row.path); const value = followBytes(row.path); assert.equal(value.length, row.bytes); assert.equal(hash(value), row.sha256, row.path); followTotal += value.length; } assert.equal(followManifest.count, followNames.size); assert.equal(followManifest.bytes, followTotal); const follow = followRead('result.json'), fidelity = followRead('fidelity-result.json'); assert.equal(follow.success, true); assert.equal(follow.candidateSourceCommit, freeze.sourceCommit); assert.equal(follow.candidate, identity.candidate); assert.equal(follow.sealedManifestUnmodified, hash(bytes('manifest.json'))); assert.equal(follow.observedHealthVersion, '1.25.3'); assert.equal(follow.realModelRequests, 0); assert.equal(follow.realCatalogRequests, 0); assert.equal(follow.actualVersionGateChecks.length, 6); assert.equal(fidelity.success, true); assert.equal(fidelity.version, '1.25.3'); assert.equal(fidelity.checks.length, 6); assert.deepEqual(fidelity.errors, []); assert.equal(fidelity.realModelRequests, 0); assert.equal(fidelity.screenshots.length, 10); for (const screenshot of fidelity.screenshots) assert.equal(hash(followBytes(path.basename(screenshot.path))), screenshot.sha256); assert.equal(follow.originalMessagesResultSha256, receipts['portable-messages'].result.metadataAdapter.originalResultSha256); let indexedFiles = 0, indexedBytes = 0; if (index) { const rows = execFileSync('git', ['ls-files', '--stage', '-z', '--', 'docs/model-catalog-1.25.3-evidence', 'docs/model-catalog-1.25.3-qa-follow-up', identityPath], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }).split('\0').filter(Boolean); const blobs = new Map(rows.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[1]]; })); function checkIndex(file, value) { assert.equal(blobs.get(file), blob(value), file + ' indexed bytes'); indexedFiles++; indexedBytes += value.length; } for (const name of [...names, 'manifest.json']) checkIndex('docs/model-catalog-1.25.3-evidence/' + name, bytes(name)); for (const name of [...followNames, 'manifest.json']) checkIndex('docs/model-catalog-1.25.3-qa-follow-up/' + name, followBytes(name)); checkIndex(identityPath, fs.readFileSync(identityPath)); } console.log(JSON.stringify({ success: true, files: names.size, bytes: total, sourceCommit: freeze.sourceCommit, inputs: freeze.inputs.length, independentInputs: build.inputCount, compiled: 22, fullSuitePassed: stats.pass, fullSuiteFailed: stats.fail, sourceUIGroups: identity.sourceUIGroups, portableUIGroups: identity.portableUIGroups, isolatedInstallGroups: 12, installedUIGroups: 103, firstFailuresPreserved: true, realModelRequests: 0, realCatalogRequests: 0, indexChecked: index, qaFollowUpFiles: followNames.size, qaFollowUpBytes: followTotal, qaFollowUpUIGroups: fidelity.checks.length, indexedFiles, indexedBytes }));