120 lines
13 KiB
JavaScript
120 lines
13 KiB
JavaScript
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import assert from 'node:assert/strict';
|
||
import { createHash } from 'node:crypto';
|
||
import { DatabaseSync } from 'node:sqlite';
|
||
import { execFileSync } from 'node:child_process';
|
||
|
||
const version = '1.14.0', read = name => JSON.parse(fs.readFileSync(name, 'utf8'));
|
||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||
const digest = value => hash(JSON.stringify(canonical(value)));
|
||
const fileIdentity = file => { const bytes = fs.readFileSync(file); return { path: file.replaceAll('\\', '/'), bytes: bytes.length, sha256: hash(bytes) }; };
|
||
const source = read('.qa/source-package-validation.json'), compared = read('.qa/controlled-engine-final-inputs-comparison.json'), baseline = read('docs/controlled-engine-final-code-baseline.json');
|
||
const qaChange = read('docs/controlled-engine-qa-path-fix-validation.json');
|
||
const portableQa = read('docs/controlled-engine-portable-qa-change.json'), finalBaseline = read('docs/controlled-engine-final-release-baseline.json');
|
||
assert.equal(portableQa.success, true); assert.equal(portableQa.change.path, 'scripts/verify-release.mjs');
|
||
assert.deepEqual(finalBaseline.sourceValidatedInputs, source.inputs);
|
||
const release = read('releases/latest.json'), archive = read('.qa/archives-validation.json'), portable = read('.qa/release-validation.json');
|
||
const ui = read('docs/controlled-engine-all-ui-validation.json'), projects = read('docs/controlled-engine-final-projects-validation.json');
|
||
const deployed = read('docs/controlled-engine-upgrade-live-validation.json'), smoke = read('.qa/controlled-engine-deployed-ui-validation.json');
|
||
for (const [name, report] of Object.entries({ source, compared, baseline, archive, portable, ui, deployed, smoke })) { assert.equal(report.success, true, name); assert.equal(report.version, version, name); }
|
||
assert.equal(projects.success, true); assert.equal(projects.targetVersion, version); assert.equal(projects.testedVersion, version);
|
||
assert.ok(source.tests.tests > 404); assert.equal(source.tests.pass, source.tests.tests);
|
||
for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(source.tests[name], 0);
|
||
assert.deepEqual(source.inputs, baseline.inputs); assert.equal(source.inputFiles, baseline.inputs.length); assert.equal(source.buildMatchedFiles, baseline.builds.length);
|
||
for (const row of [...finalBaseline.inputs, ...baseline.builds, ...baseline.benchmarks]) assert.equal(hash(fs.readFileSync(row.path)), row.sha256, row.path);
|
||
for (const row of projects.inputs) {
|
||
if (row.path.replaceAll('\\', '/') === qaChange.change.path) {
|
||
assert.equal(qaChange.success, true); assert.equal(qaChange.change.path, 'scripts/verify-controlled-release.mjs');
|
||
assert.equal(row.sha256, qaChange.change.originalSha256); assert.equal(hash(fs.readFileSync(row.path)), qaChange.change.sha256);
|
||
const original = execFileSync('git', ['show', qaChange.productCodeCommit + ':' + qaChange.change.path], { windowsHide: true, maxBuffer: 1048576 });
|
||
assert.equal(hash(original), row.sha256);
|
||
} else if (row.path.replaceAll('\\', '/') === portableQa.change.path) {
|
||
assert.equal(row.sha256, portableQa.change.originalSha256); assert.equal(hash(fs.readFileSync(row.path)), portableQa.change.sha256);
|
||
const original = execFileSync('git', ['show', portableQa.sourceValidatedCommit + ':' + portableQa.change.path], { windowsHide: true, maxBuffer: 1048576 });
|
||
assert.equal(hash(original), row.sha256);
|
||
} else assert.equal(hash(fs.readFileSync(row.path)), row.sha256, row.path);
|
||
}
|
||
assert.equal(ui.groups, 178); assert.equal(ui.results.length, 23); assert.equal(projects.results.length, 3); assert.equal(projects.cases.length, 13);
|
||
for (const [index, sample] of projects.results.entries()) {
|
||
assert.equal(sample.success, true); assert.equal(sample.version, version); assert.equal(sample.realModelRequests, 0); assert.equal(sample.parentCheck.status, 'passed'); assert.ok(sample.commit);
|
||
assert.equal(sample.parentCheck.results[0].tests.passed, [8, 9, 10][index]);
|
||
assert.ok(sample.checks.some(check => check.status === 'failed' && check.results.some(result => result.tests?.failed > 0)));
|
||
for (const run of sample.engineRuns) { assert.equal(run.status, 'completed'); assert.equal(run.native.cleanup.treeConfirmed, true); }
|
||
assert.equal(sample.metrics.unknownRequests, 0); assert.equal(sample.metrics.requests, sample.modelRequests.length);
|
||
for (const job of sample.browserJobs) { assert.equal(job.status, 'passed'); assert.equal(job.cleanup.exited, true); assert.equal(job.assertions.passed, 8); }
|
||
}
|
||
assert.equal(source.actualControlled.success, true); assert.equal(source.actualControlled.cases, 4); assert.equal(source.actualControlled.realModelRequests, 0); assert.equal(source.actualControlled.rootExited, true);
|
||
assert.equal(portable.actualControlled.success, true); assert.equal(portable.actualControlled.cases.length, 4); assert.equal(portable.actualControlled.contained, true); assert.equal(portable.actualControlled.realModelRequests, 0);
|
||
for (const native of [source.actualRea, source.actualEngine, source.actualBrowser]) assert.equal(native.success, true);
|
||
assert.equal(smoke.results.length, 14); assert.equal(smoke.pid, deployed.pid);
|
||
for (const field of ['pageErrors', 'requestFailures', 'httpErrors', 'blockedWrites']) assert.deepEqual(smoke[field], []);
|
||
assert.equal(deployed.allRowsAndExportsPreserved, true); assert.equal(deployed.connectionPreserved, true); assert.equal(deployed.modelCenterPreserved, true);
|
||
assert.equal(deployed.preservesCurrentObservedProcess, true); assert.equal(deployed.realModelRequests, 0); assert.equal(smoke.modelRequests, 0);
|
||
for (const kind of ['source', 'portable']) {
|
||
const item = archive.archives.find(value => value.kind === kind); assert.equal(item.sha256, release[kind].sha256); assert.equal(hash(fs.readFileSync(release[kind].path)), item.sha256);
|
||
}
|
||
|
||
const sourceLogs = source.logs.map(row => {
|
||
const bytes = fs.readFileSync(row.path); assert.equal(bytes.length, row.bytes); assert.equal(hash(bytes), row.sha256);
|
||
const suffix = row.stage === 'tests' ? 'tests.tap' : row.stage.replaceAll(/[^a-z0-9-]+/gi, '-').toLowerCase() + '-log.txt';
|
||
const destination = 'docs/controlled-engine-source-' + suffix; fs.writeFileSync(destination, bytes);
|
||
return { ...fileIdentity(destination), sourcePath: row.path, stage: row.stage };
|
||
});
|
||
const uiLogs = ui.results.map(row => { const destination = 'docs/controlled-engine-final-ui-' + row.script.replace('.mjs', '-log.txt'); fs.copyFileSync(row.log, destination); return { ...fileIdentity(destination), sourcePath: row.log }; });
|
||
const images = new Map();
|
||
const pngSignature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
|
||
function addImage(file, expected, destination) {
|
||
const bytes = fs.readFileSync(file); assert.ok(bytes.subarray(0, 8).equals(pngSignature), file); if (expected) assert.equal(hash(bytes), expected, file);
|
||
if (destination) { fs.mkdirSync(path.dirname(destination), { recursive: true }); fs.writeFileSync(destination, bytes); }
|
||
images.set(destination ?? file, { ...fileIdentity(destination ?? file), sourcePath: file });
|
||
}
|
||
for (const row of smoke.results) addImage(row.screenshot, row.screenshotSha256, 'docs/screenshots/controlled-engine-deployed-' + path.basename(row.screenshot));
|
||
for (const item of ui.results) {
|
||
const proof = read(item.evidence);
|
||
if (Array.isArray(proof.screenshots)) for (const row of proof.screenshots) {
|
||
if (row && typeof row === 'object' && row.file && row.sha256) addImage(row.file, row.sha256, 'docs/screenshots/controlled-engine-final-' + item.script.replace('verify-', '').replace('.mjs', '') + '-' + path.basename(row.file));
|
||
}
|
||
}
|
||
for (const sample of projects.results) for (const job of sample.browserJobs) for (const row of job.screenshots) {
|
||
assert.ok([1280, 390].includes(row.width)); assert.ok([900, 844].includes(row.height));
|
||
const file = path.join(projects.directory, sample.sampleName, 'page-' + row.width + '-' + row.artifactId + '.png');
|
||
addImage(file, row.sha256, 'docs/screenshots/controlled-engine-project-' + path.basename(file));
|
||
}
|
||
|
||
const db = new DatabaseSync(path.join(deployed.dataRoot, 'workbench.sqlite'), { readOnly: true }); db.exec('BEGIN');
|
||
try {
|
||
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all().map(row => row.name); assert.deepEqual(tables, Object.keys(deployed.database).sort());
|
||
for (const table of tables) { const rows = db.prepare('SELECT * FROM "' + table.replaceAll('"', '""') + '" ORDER BY rowid').all(); assert.equal(rows.length, deployed.database[table].count); assert.equal(digest(rows), deployed.database[table].sha256); }
|
||
} finally { db.exec('ROLLBACK'); db.close(); }
|
||
const base = 'http://127.0.0.1:4318', bootstrap = await (await fetch(base + '/api/bootstrap', { signal: AbortSignal.timeout(10000) })).json();
|
||
assert.equal(bootstrap.version, version); assert.equal((await (await fetch(base + '/api/health')).json()).pid, deployed.pid);
|
||
for (const row of deployed.preservedSessionSnapshots) {
|
||
const value = await (await fetch(base + '/api/sessions/' + row.id + '/export', { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, signal: AbortSignal.timeout(20000) })).json();
|
||
for (const kind of ['project', 'session', 'execution']) assert.equal(digest(value[kind]), row[kind]);
|
||
}
|
||
await assert.rejects(fetch('http://127.0.0.1:9229/json/list', { signal: AbortSignal.timeout(1000) }));
|
||
const copies = [
|
||
['.qa/archives-validation.json', 'docs/controlled-engine-archives-validation.json'], ['.qa/release-validation.json', 'docs/controlled-engine-release-validation.json'],
|
||
['.qa/source-package-validation.json', 'docs/controlled-engine-source-package-validation.json'], ['.qa/controlled-engine-final-inputs-comparison.json', 'docs/controlled-engine-git-inputs.json'],
|
||
['.qa/controlled-engine-source-tested-candidate.json', 'docs/controlled-engine-source-tested-candidate.json'], ['.qa/controlled-engine-source-candidate-archives-validation.json', 'docs/controlled-engine-source-candidate-archives-validation.json'],
|
||
['.qa/controlled-engine-deployed-ui-validation.json', 'docs/controlled-engine-deployed-ui-validation.json'],
|
||
];
|
||
for (const [from, to] of copies) fs.copyFileSync(from, to);
|
||
const reportNames = fs.readdirSync('docs').filter(name => name.startsWith('controlled-engine-') && /\.(json|tap|txt|mjs)$/.test(name) && name !== 'controlled-engine-release-identity.json').sort();
|
||
const reports = reportNames.map(name => fileIdentity('docs/' + name));
|
||
const identity = { success: true, version, sealedAt: new Date().toISOString(), codeCommit: finalBaseline.codeCommit, productCodeCommit: finalBaseline.productCodeCommit,
|
||
evidenceCommitBeforeSeal: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(),
|
||
tests: source.tests, uiGroups: ui.groups, uiTypes: ui.results.length, projectSamples: projects.results.length, projectCases: projects.cases.length, validationOnlyChange: qaChange,
|
||
postSourceValidationOnlyChange: portableQa, sourceInputsExactToFinal: compared.sourceInputsExactToFinal,
|
||
sourceInputFiles: source.inputFiles, buildFiles: baseline.builds.length, benchmarkFiles: baseline.benchmarks.length, comparedFiles: compared.comparedFiles,
|
||
gitExactInputs: compared.exactBytes, gitLfOnlyInputs: compared.normalized, release, deployedPid: deployed.pid, database: deployed.database,
|
||
preservedSessionSnapshots: deployed.preservedSessionSnapshots, modelCenter: deployed.modelCenter, allRowsAndExportsPreserved: true,
|
||
reports, logs: [...sourceLogs, ...uiLogs], images: [...images.values()], realModelRequests: 0,
|
||
acceptance: { controlledCompiledSource: source.actualControlled, controlledPortable: { cases: portable.actualControlled.cases.length, contained: true, fixtureModelRequests: portable.actualControlled.fixtureModelRequests },
|
||
purgeDefaultEnabled: false, purgeReasoningBenefit: 'not-measured-owned-response-fixture', desktopInstaller: 'planned-1.15', phonePairing: 'not-implemented' },
|
||
scope: 'Pinned controlled DeepSeek Harness and default-off source-pinned prompt experiment, owned local HTTP fixtures, actual three functional projects/reviews/checks/Edge/Git, complete independent source/portable/UI and original-host deployment. Historical A–E failures/results retain their original version; real inference benefit and other engine adapters remain separately unverified.' };
|
||
fs.writeFileSync('docs/controlled-engine-release-identity.json', JSON.stringify(identity, null, 2));
|
||
console.log(JSON.stringify({ success: true, tests: source.tests.tests, uiGroups: ui.groups, projects: projects.results.length, reports: reports.length, logs: identity.logs.length, images: identity.images.length, pid: deployed.pid }));
|