115 lines
11 KiB
JavaScript
115 lines
11 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
const root = path.resolve('docs/reliability-1.25.2-evidence'), index = process.argv.includes('--index');
|
|
assert.ok(process.argv.slice(2).every(value => value === '--index'));
|
|
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
const blob = bytes => createHash('sha1').update(Buffer.from('blob ' + bytes.length + '\0')).update(bytes).digest('hex');
|
|
const bytes = relative => fs.readFileSync(path.join(root, relative));
|
|
const read = relative => JSON.parse(bytes(relative).toString('utf8').replace(/^\uFEFF/, ''));
|
|
const manifest = read('manifest.json'), names = new Set(), originalFiles = new Map(); let total = 0;
|
|
for (const row of manifest.entries) {
|
|
assert.ok(row.path && !path.isAbsolute(row.path) && !row.path.includes('\\') && !row.path.split('/').includes('..'));
|
|
assert.equal(names.has(row.path), false); names.add(row.path);
|
|
const value = bytes(row.path); assert.equal(value.length, row.bytes); assert.equal(hash(value), row.sha256, row.path); total += value.length;
|
|
originalFiles.set(path.resolve(row.source), row.path);
|
|
}
|
|
assert.equal(manifest.count, names.size); assert.equal(manifest.bytes, total);
|
|
const localOriginal = file => { const name = originalFiles.get(path.resolve(file)); assert.ok(name, file); return name; };
|
|
const freeze = read('release-final/freeze.json'), candidate = read('release-final/candidate.json'), identity = read('release-identity.json');
|
|
assert.equal(manifest.version, '1.25.2'); assert.equal(manifest.sourceCommit, freeze.sourceCommit); assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit);
|
|
assert.equal(identity.sourceCommit, freeze.sourceCommit); assert.equal(identity.candidate, path.basename(candidate.portableRoot));
|
|
assert.equal(hash(bytes('portable-manifest.json')), identity.portableManifestSha256); assert.equal(hash(bytes('source-manifest.json')), identity.sourceManifestSha256);
|
|
assert.equal(identity.formalPid, 36552); assert.deepEqual(identity.installer, candidate.installer);
|
|
const sourceTree = execFileSync('git', ['ls-tree', '-r', '-z', '--full-tree', freeze.sourceCommit], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }).split('\0').filter(Boolean);
|
|
const sourceBlobs = new Map(sourceTree.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[2]]; }));
|
|
for (const row of freeze.inputs) {
|
|
const value = bytes('source-bytes/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : '')); assert.equal(hash(value), row.sha256, row.path);
|
|
const oid = sourceBlobs.get(row.path); assert.ok(oid, row.path);
|
|
if (blob(value) !== oid) { assert.equal(value.includes(0), false); assert.equal(blob(Buffer.from(value.toString('utf8').replaceAll('\r\n', '\n'))), oid, row.path); }
|
|
}
|
|
for (const row of freeze.compiled) assert.equal(hash(bytes('compiled/' + row.path + (row.path.endsWith('.mjs') ? '.txt' : ''))), row.sha256, row.path);
|
|
const stages = ['rebuild', 'archives', 'portable', 'install', 'portable-files', 'portable-results', 'portable-reading',
|
|
'portable-a', 'portable-a-desktop', 'portable-b', 'portable-b-desktop', 'portable-c', 'portable-c-desktop', 'portable-d', 'portable-d-desktop'];
|
|
const receipts = Object.fromEntries(stages.map(stage => {
|
|
const receipt = read('release-final/receipts/' + stage + '.json'); assert.equal(receipt.success, true, stage);
|
|
assert.equal(receipt.sourceCommit, freeze.sourceCommit); assert.equal(receipt.candidate, candidate.portableRoot);
|
|
assert.equal(hash(bytes('release-final/logs/' + path.basename(receipt.log))), receipt.logSha256); return [stage, receipt];
|
|
}));
|
|
const build = receipts.rebuild.result; assert.equal(build.inputCount, freeze.inputs.length + 5); assert.equal(build.compiled.length, 22); assert.deepEqual(build.mismatches, []);
|
|
for (const row of freeze.inputs) assert.equal(build.inputs.find(item => item.path === row.path)?.sha256, row.sha256);
|
|
for (const row of build.compiled) assert.deepEqual(row.actual, row.expected);
|
|
for (const kind of ['portable', 'source', 'evidence']) {
|
|
const archive = receipts.archives.result.archives.find(row => row.kind === kind);
|
|
assert.equal(archive.crcPassed, true); assert.equal(archive.manifestAndDirectoryMatch, true); assert.equal(archive.sha256, candidate[kind].sha256);
|
|
}
|
|
const firstPortable = read('release/receipts/portable.json'); assert.equal(firstPortable.success, false); assert.equal(hash(bytes('release/logs/portable.log')), firstPortable.logSha256);
|
|
assert.match(bytes('release/logs/portable.log').toString(), /尚未发送initialize/);
|
|
const failedNative = read('validation/portable-first/original-engine-report.json');
|
|
assert.equal(failedNative.native.modelRequests, 0); assert.equal(failedNative.native.toolRequests, 0); assert.equal(failedNative.native.cleanup.treeConfirmed, true);
|
|
const secondPortable = read('release/receipts/portable-recheck.json'); assert.equal(secondPortable.success, false);
|
|
assert.equal(hash(bytes('release/logs/portable-recheck.log')), secondPortable.logSha256);
|
|
const secondNative = read('validation/portable-second/original-engine-report.json').native;
|
|
assert.equal(secondNative.modelRequests, 0); assert.equal(secondNative.toolRequests, 0); assert.equal(secondNative.cleanup.treeConfirmed, true);
|
|
const late = read('validation/portable-second/late-ready.json'); assert.equal(late.pid, secondNative.process.pid);
|
|
assert.ok(Date.parse(late.readyAt) - Date.parse(secondNative.process.startedAt) > 15000);
|
|
for (const label of ['diagnostic','detached']) {
|
|
const native = read('validation/compiled-startup-' + label + '/result.json'); assert.equal(native.success, true); assert.equal(native.realModelRequests, 0);
|
|
for (const run of native.originalRuns) assert.ok(Date.parse(run.original.native.readiness.readyAt) - Date.parse(run.original.native.process.startedAt) < 15000);
|
|
}
|
|
const delta = read('development/candidate-input-delta.json'); assert.equal(delta.success, true); assert.equal(delta.after, freeze.sourceCommit); assert.equal(delta.compiledExact, 22);
|
|
assert.deepEqual(delta.changes.map(row => row.path), ['tests/network-diagnostics.test.mjs']);
|
|
const installation = receipts.install.result; assert.equal(installation.results.length, 12); assert.ok(installation.results.every(row => row.passed));
|
|
assert.equal(installation.installedUI.proofs.length, 10);
|
|
for (const [i, item] of installation.installedUI.proofs.entries()) {
|
|
const proof = item.proof; assert.equal(proof.success, true); assert.equal(proof.version, '1.25.2'); assert.deepEqual(proof.errors, []); assert.equal(proof.realModelRequests, 0);
|
|
assert.equal((proof.checks ?? proof.passed).length, [16, 9, 7, 3, 9, 9, 6, 6, 5, 5][i]); assert.equal(hash(bytes(localOriginal(item.log))), item.sha256);
|
|
}
|
|
const sourceUI = read('development/source-ui.json'); assert.equal(sourceUI.results.length, 8); let uiGroups = 0;
|
|
for (const item of sourceUI.results) {
|
|
assert.equal(hash(bytes(localOriginal(item.log))), item.logSha256); assert.equal(hash(bytes(localOriginal(item.resultFile))), item.resultSha256);
|
|
const proof = read(localOriginal(item.resultFile)); assert.equal(proof.success, true); assert.equal(proof.passed.length, item.groups); uiGroups += item.groups;
|
|
}
|
|
assert.equal(uiGroups, 50);
|
|
for (const [stage, groups] of [['portable-a', 7], ['portable-a-desktop', 3], ['portable-b', 9], ['portable-b-desktop', 9], ['portable-c', 6], ['portable-c-desktop', 6], ['portable-d', 5], ['portable-d-desktop', 5]]) {
|
|
const proof = receipts[stage].result; assert.equal(proof.version, '1.25.2'); assert.equal(proof.passed.length, groups); assert.deepEqual(proof.errors, []); assert.equal(proof.realModelRequests, 0);
|
|
}
|
|
const full = bytes('development/full-release-final.tap').toString();
|
|
for (const expression of [/\btests 899\b/, /\bpass 898\b/, /\bfail 1\b/, /\bcancelled 0\b/, /\bskipped 0\b/]) assert.match(full, expression);
|
|
assert.match(full, /tests\/api\.test\.mjs:101:3/); assert.match(full, /Error: Timed out waiting for test state\./);
|
|
const suiteReview = read('development/full-suite-review.json');
|
|
assert.equal(suiteReview.success, true); assert.equal(suiteReview.sourceCommit, freeze.sourceCommit);
|
|
assert.equal(hash(bytes('development/full-release-final.tap')), suiteReview.wholeSuite.sha256);
|
|
assert.equal(hash(bytes(localOriginal(suiteReview.supplement.output))), suiteReview.supplement.outputSha256);
|
|
assert.equal(hash(bytes(localOriginal(suiteReview.supplement.log))), suiteReview.supplement.logSha256);
|
|
assert.equal(suiteReview.supplement.success, true); assert.equal(suiteReview.supplement.sourceCommit, freeze.sourceCommit);
|
|
assert.equal(suiteReview.supplement.passed, 1); assert.equal(suiteReview.supplement.failed, 0);
|
|
const supplement = bytes(localOriginal(suiteReview.supplement.output)).toString();
|
|
for (const expression of [/\btests 1\b/, /\bpass 1\b/, /\bfail 0\b/, /\bcancelled 0\b/, /\bskipped 0\b/]) assert.match(supplement, expression);
|
|
for (const item of suiteReview.unchangedAssertions) assert.equal(freeze.inputs.find(row => row.path === item.path)?.sha256, item.sha256);
|
|
assert.equal(identity.wholeSuiteAllPass, false); assert.deepEqual(identity.fullSuiteResult, suiteReview.wholeSuite);
|
|
assert.match(bytes('development/full.tap').toString(), /\bfail 1\b/);
|
|
assert.match(bytes('development/full-final.tap').toString(), /'tcp' !== 'tls'/);
|
|
const before = read('formal/state-before.json');
|
|
for (const stage of ['upgrade', 'final']) {
|
|
const proof = read('formal/' + stage + '.json'), state = read('formal/' + stage + '-state-after.json');
|
|
assert.equal(proof.success, true); assert.equal(proof.version, '1.25.2'); assert.equal(proof.pid, 36552); assert.equal(proof.inspectorClosed, true); assert.equal(proof.desktopWindows, 0);
|
|
for (const key of ['pid', 'hasKey', 'configSha256', 'projectsSha256', 'sessionsSha256', 'eventCursor', 'exports', 'persistent', 'modelCenterSha256']) assert.deepEqual(state[key], before[key], key);
|
|
assert.deepEqual(state.database.tables, before.database.tables); assert.deepEqual(state.database.sessionIds, before.database.sessionIds);
|
|
}
|
|
assert.equal(read('formal/installer-exit.json').code, 0);
|
|
for (const file of ['formal-web/result.json', 'formal-native/result.json']) {
|
|
const proof = read(file); assert.equal(proof.success, true); assert.equal(proof.version, '1.25.2'); assert.equal(proof.pid, 36552); assert.equal(proof.realModelRequests, 0);
|
|
}
|
|
assert.deepEqual(read('formal-web/result.json').rejectedWrites, []);
|
|
if (index) {
|
|
const rows = execFileSync('git', ['ls-files', '--stage', '-z', '--', 'docs/reliability-1.25.2-evidence'], { encoding: 'utf8', maxBuffer: 8 * 1024 * 1024 }).split('\0').filter(Boolean);
|
|
const blobs = new Map(rows.map(row => { const [meta, file] = row.split('\t'); return [file, meta.split(' ')[1]]; }));
|
|
for (const name of [...names, 'manifest.json']) assert.equal(blobs.get('docs/reliability-1.25.2-evidence/' + name), blob(bytes(name)), name + ' indexed bytes');
|
|
}
|
|
console.log(JSON.stringify({ success: true, files: names.size, bytes: total, sourceCommit: freeze.sourceCommit, inputs: freeze.inputs.length,
|
|
independentInputs: build.inputCount, compiled: 22, fullSuitePassed: 898, fullSuiteFailed: 1, exactSupplementPassed: 1, wholeSuiteAllPass: false, sourceUIGroups: 50, isolatedInstallGroups: 12, installedUIGroups: 75,
|
|
firstFailuresPreserved: true, realModelRequests: 0, indexChecked: index }));
|