import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { DatabaseSync } from 'node:sqlite'; import { execFileSync } from 'node:child_process'; const version = '1.14.0', read = name => JSON.parse(fs.readFileSync(name, 'utf8')); const hash = bytes => createHash('sha256').update(bytes).digest('hex'); const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value; const digest = value => hash(JSON.stringify(canonical(value))); const fileIdentity = file => { const bytes = fs.readFileSync(file); return { path: file.replaceAll('\\', '/'), bytes: bytes.length, sha256: hash(bytes) }; }; const source = read('.qa/source-package-validation.json'), compared = read('.qa/controlled-engine-final-inputs-comparison.json'), baseline = read('docs/controlled-engine-final-code-baseline.json'); const qaChange = read('docs/controlled-engine-qa-path-fix-validation.json'); const portableQa = read('docs/controlled-engine-portable-qa-change.json'), finalBaseline = read('docs/controlled-engine-final-release-baseline.json'); assert.equal(portableQa.success, true); assert.equal(portableQa.change.path, 'scripts/verify-release.mjs'); assert.deepEqual(finalBaseline.sourceValidatedInputs, source.inputs); const release = read('releases/latest.json'), archive = read('.qa/archives-validation.json'), portable = read('.qa/release-validation.json'); const ui = read('docs/controlled-engine-all-ui-validation.json'), projects = read('docs/controlled-engine-final-projects-validation.json'); const deployed = read('docs/controlled-engine-upgrade-live-validation.json'), smoke = read('.qa/controlled-engine-deployed-ui-validation.json'); for (const [name, report] of Object.entries({ source, compared, baseline, archive, portable, ui, deployed, smoke })) { assert.equal(report.success, true, name); assert.equal(report.version, version, name); } assert.equal(projects.success, true); assert.equal(projects.targetVersion, version); assert.equal(projects.testedVersion, version); assert.ok(source.tests.tests > 404); assert.equal(source.tests.pass, source.tests.tests); for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(source.tests[name], 0); assert.deepEqual(source.inputs, baseline.inputs); assert.equal(source.inputFiles, baseline.inputs.length); assert.equal(source.buildMatchedFiles, baseline.builds.length); for (const row of [...finalBaseline.inputs, ...baseline.builds, ...baseline.benchmarks]) assert.equal(hash(fs.readFileSync(row.path)), row.sha256, row.path); for (const row of projects.inputs) { if (row.path.replaceAll('\\', '/') === qaChange.change.path) { assert.equal(qaChange.success, true); assert.equal(qaChange.change.path, 'scripts/verify-controlled-release.mjs'); assert.equal(row.sha256, qaChange.change.originalSha256); assert.equal(hash(fs.readFileSync(row.path)), qaChange.change.sha256); const original = execFileSync('git', ['show', qaChange.productCodeCommit + ':' + qaChange.change.path], { windowsHide: true, maxBuffer: 1048576 }); assert.equal(hash(original), row.sha256); } else if (row.path.replaceAll('\\', '/') === portableQa.change.path) { assert.equal(row.sha256, portableQa.change.originalSha256); assert.equal(hash(fs.readFileSync(row.path)), portableQa.change.sha256); const original = execFileSync('git', ['show', portableQa.sourceValidatedCommit + ':' + portableQa.change.path], { windowsHide: true, maxBuffer: 1048576 }); assert.equal(hash(original), row.sha256); } else assert.equal(hash(fs.readFileSync(row.path)), row.sha256, row.path); } assert.equal(ui.groups, 178); assert.equal(ui.results.length, 23); assert.equal(projects.results.length, 3); assert.equal(projects.cases.length, 13); for (const [index, sample] of projects.results.entries()) { assert.equal(sample.success, true); assert.equal(sample.version, version); assert.equal(sample.realModelRequests, 0); assert.equal(sample.parentCheck.status, 'passed'); assert.ok(sample.commit); assert.equal(sample.parentCheck.results[0].tests.passed, [8, 9, 10][index]); assert.ok(sample.checks.some(check => check.status === 'failed' && check.results.some(result => result.tests?.failed > 0))); for (const run of sample.engineRuns) { assert.equal(run.status, 'completed'); assert.equal(run.native.cleanup.treeConfirmed, true); } assert.equal(sample.metrics.unknownRequests, 0); assert.equal(sample.metrics.requests, sample.modelRequests.length); for (const job of sample.browserJobs) { assert.equal(job.status, 'passed'); assert.equal(job.cleanup.exited, true); assert.equal(job.assertions.passed, 8); } } assert.equal(source.actualControlled.success, true); assert.equal(source.actualControlled.cases, 4); assert.equal(source.actualControlled.realModelRequests, 0); assert.equal(source.actualControlled.rootExited, true); assert.equal(portable.actualControlled.success, true); assert.equal(portable.actualControlled.cases.length, 4); assert.equal(portable.actualControlled.contained, true); assert.equal(portable.actualControlled.realModelRequests, 0); for (const native of [source.actualRea, source.actualEngine, source.actualBrowser]) assert.equal(native.success, true); assert.equal(smoke.results.length, 14); assert.equal(smoke.pid, deployed.pid); for (const field of ['pageErrors', 'requestFailures', 'httpErrors', 'blockedWrites']) assert.deepEqual(smoke[field], []); assert.equal(deployed.allRowsAndExportsPreserved, true); assert.equal(deployed.connectionPreserved, true); assert.equal(deployed.modelCenterPreserved, true); assert.equal(deployed.preservesCurrentObservedProcess, true); assert.equal(deployed.realModelRequests, 0); assert.equal(smoke.modelRequests, 0); for (const kind of ['source', 'portable']) { const item = archive.archives.find(value => value.kind === kind); assert.equal(item.sha256, release[kind].sha256); assert.equal(hash(fs.readFileSync(release[kind].path)), item.sha256); } const sourceLogs = source.logs.map(row => { const bytes = fs.readFileSync(row.path); assert.equal(bytes.length, row.bytes); assert.equal(hash(bytes), row.sha256); const suffix = row.stage === 'tests' ? 'tests.tap' : row.stage.replaceAll(/[^a-z0-9-]+/gi, '-').toLowerCase() + '-log.txt'; const destination = 'docs/controlled-engine-source-' + suffix; fs.writeFileSync(destination, bytes); return { ...fileIdentity(destination), sourcePath: row.path, stage: row.stage }; }); const uiLogs = ui.results.map(row => { const destination = 'docs/controlled-engine-final-ui-' + row.script.replace('.mjs', '-log.txt'); fs.copyFileSync(row.log, destination); return { ...fileIdentity(destination), sourcePath: row.log }; }); const images = new Map(); const pngSignature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); function addImage(file, expected, destination) { const bytes = fs.readFileSync(file); assert.ok(bytes.subarray(0, 8).equals(pngSignature), file); if (expected) assert.equal(hash(bytes), expected, file); if (destination) { fs.mkdirSync(path.dirname(destination), { recursive: true }); fs.writeFileSync(destination, bytes); } images.set(destination ?? file, { ...fileIdentity(destination ?? file), sourcePath: file }); } for (const row of smoke.results) addImage(row.screenshot, row.screenshotSha256, 'docs/screenshots/controlled-engine-deployed-' + path.basename(row.screenshot)); for (const item of ui.results) { const proof = read(item.evidence); if (Array.isArray(proof.screenshots)) for (const row of proof.screenshots) { if (row && typeof row === 'object' && row.file && row.sha256) addImage(row.file, row.sha256, 'docs/screenshots/controlled-engine-final-' + item.script.replace('verify-', '').replace('.mjs', '') + '-' + path.basename(row.file)); } } for (const sample of projects.results) for (const job of sample.browserJobs) for (const row of job.screenshots) { assert.ok([1280, 390].includes(row.width)); assert.ok([900, 844].includes(row.height)); const file = path.join(projects.directory, sample.sampleName, 'page-' + row.width + '-' + row.artifactId + '.png'); addImage(file, row.sha256, 'docs/screenshots/controlled-engine-project-' + path.basename(file)); } const db = new DatabaseSync(path.join(deployed.dataRoot, 'workbench.sqlite'), { readOnly: true }); db.exec('BEGIN'); try { const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all().map(row => row.name); assert.deepEqual(tables, Object.keys(deployed.database).sort()); for (const table of tables) { const rows = db.prepare('SELECT * FROM "' + table.replaceAll('"', '""') + '" ORDER BY rowid').all(); assert.equal(rows.length, deployed.database[table].count); assert.equal(digest(rows), deployed.database[table].sha256); } } finally { db.exec('ROLLBACK'); db.close(); } const base = 'http://127.0.0.1:4318', bootstrap = await (await fetch(base + '/api/bootstrap', { signal: AbortSignal.timeout(10000) })).json(); assert.equal(bootstrap.version, version); assert.equal((await (await fetch(base + '/api/health')).json()).pid, deployed.pid); for (const row of deployed.preservedSessionSnapshots) { const value = await (await fetch(base + '/api/sessions/' + row.id + '/export', { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, signal: AbortSignal.timeout(20000) })).json(); for (const kind of ['project', 'session', 'execution']) assert.equal(digest(value[kind]), row[kind]); } await assert.rejects(fetch('http://127.0.0.1:9229/json/list', { signal: AbortSignal.timeout(1000) })); const copies = [ ['.qa/archives-validation.json', 'docs/controlled-engine-archives-validation.json'], ['.qa/release-validation.json', 'docs/controlled-engine-release-validation.json'], ['.qa/source-package-validation.json', 'docs/controlled-engine-source-package-validation.json'], ['.qa/controlled-engine-final-inputs-comparison.json', 'docs/controlled-engine-git-inputs.json'], ['.qa/controlled-engine-source-tested-candidate.json', 'docs/controlled-engine-source-tested-candidate.json'], ['.qa/controlled-engine-source-candidate-archives-validation.json', 'docs/controlled-engine-source-candidate-archives-validation.json'], ['.qa/controlled-engine-deployed-ui-validation.json', 'docs/controlled-engine-deployed-ui-validation.json'], ]; for (const [from, to] of copies) fs.copyFileSync(from, to); const reportNames = fs.readdirSync('docs').filter(name => name.startsWith('controlled-engine-') && /\.(json|tap|txt|mjs)$/.test(name) && name !== 'controlled-engine-release-identity.json').sort(); const reports = reportNames.map(name => fileIdentity('docs/' + name)); const identity = { success: true, version, sealedAt: new Date().toISOString(), codeCommit: finalBaseline.codeCommit, productCodeCommit: finalBaseline.productCodeCommit, evidenceCommitBeforeSeal: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(), tests: source.tests, uiGroups: ui.groups, uiTypes: ui.results.length, projectSamples: projects.results.length, projectCases: projects.cases.length, validationOnlyChange: qaChange, postSourceValidationOnlyChange: portableQa, sourceInputsExactToFinal: compared.sourceInputsExactToFinal, sourceInputFiles: source.inputFiles, buildFiles: baseline.builds.length, benchmarkFiles: baseline.benchmarks.length, comparedFiles: compared.comparedFiles, gitExactInputs: compared.exactBytes, gitLfOnlyInputs: compared.normalized, release, deployedPid: deployed.pid, database: deployed.database, preservedSessionSnapshots: deployed.preservedSessionSnapshots, modelCenter: deployed.modelCenter, allRowsAndExportsPreserved: true, reports, logs: [...sourceLogs, ...uiLogs], images: [...images.values()], realModelRequests: 0, acceptance: { controlledCompiledSource: source.actualControlled, controlledPortable: { cases: portable.actualControlled.cases.length, contained: true, fixtureModelRequests: portable.actualControlled.fixtureModelRequests }, purgeDefaultEnabled: false, purgeReasoningBenefit: 'not-measured-owned-response-fixture', desktopInstaller: 'planned-1.15', phonePairing: 'not-implemented' }, scope: 'Pinned controlled DeepSeek Harness and default-off source-pinned prompt experiment, owned local HTTP fixtures, actual three functional projects/reviews/checks/Edge/Git, complete independent source/portable/UI and original-host deployment. Historical A–E failures/results retain their original version; real inference benefit and other engine adapters remain separately unverified.' }; fs.writeFileSync('docs/controlled-engine-release-identity.json', JSON.stringify(identity, null, 2)); console.log(JSON.stringify({ success: true, tests: source.tests.tests, uiGroups: ui.groups, projects: projects.results.length, reports: reports.length, logs: identity.logs.length, images: identity.images.length, pid: deployed.pid }));