Files
Agent-Workbench/docs/controlled-engine-evidence-seal-verifier.mjs
2026-10-31 08:16:56 +08:00

120 lines
13 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
import { execFileSync } from 'node:child_process';
const version = '1.14.0', read = name => JSON.parse(fs.readFileSync(name, 'utf8'));
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
const digest = value => hash(JSON.stringify(canonical(value)));
const fileIdentity = file => { const bytes = fs.readFileSync(file); return { path: file.replaceAll('\\', '/'), bytes: bytes.length, sha256: hash(bytes) }; };
const source = read('.qa/source-package-validation.json'), compared = read('.qa/controlled-engine-final-inputs-comparison.json'), baseline = read('docs/controlled-engine-final-code-baseline.json');
const qaChange = read('docs/controlled-engine-qa-path-fix-validation.json');
const portableQa = read('docs/controlled-engine-portable-qa-change.json'), finalBaseline = read('docs/controlled-engine-final-release-baseline.json');
assert.equal(portableQa.success, true); assert.equal(portableQa.change.path, 'scripts/verify-release.mjs');
assert.deepEqual(finalBaseline.sourceValidatedInputs, source.inputs);
const release = read('releases/latest.json'), archive = read('.qa/archives-validation.json'), portable = read('.qa/release-validation.json');
const ui = read('docs/controlled-engine-all-ui-validation.json'), projects = read('docs/controlled-engine-final-projects-validation.json');
const deployed = read('docs/controlled-engine-upgrade-live-validation.json'), smoke = read('.qa/controlled-engine-deployed-ui-validation.json');
for (const [name, report] of Object.entries({ source, compared, baseline, archive, portable, ui, deployed, smoke })) { assert.equal(report.success, true, name); assert.equal(report.version, version, name); }
assert.equal(projects.success, true); assert.equal(projects.targetVersion, version); assert.equal(projects.testedVersion, version);
assert.ok(source.tests.tests > 404); assert.equal(source.tests.pass, source.tests.tests);
for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(source.tests[name], 0);
assert.deepEqual(source.inputs, baseline.inputs); assert.equal(source.inputFiles, baseline.inputs.length); assert.equal(source.buildMatchedFiles, baseline.builds.length);
for (const row of [...finalBaseline.inputs, ...baseline.builds, ...baseline.benchmarks]) assert.equal(hash(fs.readFileSync(row.path)), row.sha256, row.path);
for (const row of projects.inputs) {
if (row.path.replaceAll('\\', '/') === qaChange.change.path) {
assert.equal(qaChange.success, true); assert.equal(qaChange.change.path, 'scripts/verify-controlled-release.mjs');
assert.equal(row.sha256, qaChange.change.originalSha256); assert.equal(hash(fs.readFileSync(row.path)), qaChange.change.sha256);
const original = execFileSync('git', ['show', qaChange.productCodeCommit + ':' + qaChange.change.path], { windowsHide: true, maxBuffer: 1048576 });
assert.equal(hash(original), row.sha256);
} else if (row.path.replaceAll('\\', '/') === portableQa.change.path) {
assert.equal(row.sha256, portableQa.change.originalSha256); assert.equal(hash(fs.readFileSync(row.path)), portableQa.change.sha256);
const original = execFileSync('git', ['show', portableQa.sourceValidatedCommit + ':' + portableQa.change.path], { windowsHide: true, maxBuffer: 1048576 });
assert.equal(hash(original), row.sha256);
} else assert.equal(hash(fs.readFileSync(row.path)), row.sha256, row.path);
}
assert.equal(ui.groups, 178); assert.equal(ui.results.length, 23); assert.equal(projects.results.length, 3); assert.equal(projects.cases.length, 13);
for (const [index, sample] of projects.results.entries()) {
assert.equal(sample.success, true); assert.equal(sample.version, version); assert.equal(sample.realModelRequests, 0); assert.equal(sample.parentCheck.status, 'passed'); assert.ok(sample.commit);
assert.equal(sample.parentCheck.results[0].tests.passed, [8, 9, 10][index]);
assert.ok(sample.checks.some(check => check.status === 'failed' && check.results.some(result => result.tests?.failed > 0)));
for (const run of sample.engineRuns) { assert.equal(run.status, 'completed'); assert.equal(run.native.cleanup.treeConfirmed, true); }
assert.equal(sample.metrics.unknownRequests, 0); assert.equal(sample.metrics.requests, sample.modelRequests.length);
for (const job of sample.browserJobs) { assert.equal(job.status, 'passed'); assert.equal(job.cleanup.exited, true); assert.equal(job.assertions.passed, 8); }
}
assert.equal(source.actualControlled.success, true); assert.equal(source.actualControlled.cases, 4); assert.equal(source.actualControlled.realModelRequests, 0); assert.equal(source.actualControlled.rootExited, true);
assert.equal(portable.actualControlled.success, true); assert.equal(portable.actualControlled.cases.length, 4); assert.equal(portable.actualControlled.contained, true); assert.equal(portable.actualControlled.realModelRequests, 0);
for (const native of [source.actualRea, source.actualEngine, source.actualBrowser]) assert.equal(native.success, true);
assert.equal(smoke.results.length, 14); assert.equal(smoke.pid, deployed.pid);
for (const field of ['pageErrors', 'requestFailures', 'httpErrors', 'blockedWrites']) assert.deepEqual(smoke[field], []);
assert.equal(deployed.allRowsAndExportsPreserved, true); assert.equal(deployed.connectionPreserved, true); assert.equal(deployed.modelCenterPreserved, true);
assert.equal(deployed.preservesCurrentObservedProcess, true); assert.equal(deployed.realModelRequests, 0); assert.equal(smoke.modelRequests, 0);
for (const kind of ['source', 'portable']) {
const item = archive.archives.find(value => value.kind === kind); assert.equal(item.sha256, release[kind].sha256); assert.equal(hash(fs.readFileSync(release[kind].path)), item.sha256);
}
const sourceLogs = source.logs.map(row => {
const bytes = fs.readFileSync(row.path); assert.equal(bytes.length, row.bytes); assert.equal(hash(bytes), row.sha256);
const suffix = row.stage === 'tests' ? 'tests.tap' : row.stage.replaceAll(/[^a-z0-9-]+/gi, '-').toLowerCase() + '-log.txt';
const destination = 'docs/controlled-engine-source-' + suffix; fs.writeFileSync(destination, bytes);
return { ...fileIdentity(destination), sourcePath: row.path, stage: row.stage };
});
const uiLogs = ui.results.map(row => { const destination = 'docs/controlled-engine-final-ui-' + row.script.replace('.mjs', '-log.txt'); fs.copyFileSync(row.log, destination); return { ...fileIdentity(destination), sourcePath: row.log }; });
const images = new Map();
const pngSignature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
function addImage(file, expected, destination) {
const bytes = fs.readFileSync(file); assert.ok(bytes.subarray(0, 8).equals(pngSignature), file); if (expected) assert.equal(hash(bytes), expected, file);
if (destination) { fs.mkdirSync(path.dirname(destination), { recursive: true }); fs.writeFileSync(destination, bytes); }
images.set(destination ?? file, { ...fileIdentity(destination ?? file), sourcePath: file });
}
for (const row of smoke.results) addImage(row.screenshot, row.screenshotSha256, 'docs/screenshots/controlled-engine-deployed-' + path.basename(row.screenshot));
for (const item of ui.results) {
const proof = read(item.evidence);
if (Array.isArray(proof.screenshots)) for (const row of proof.screenshots) {
if (row && typeof row === 'object' && row.file && row.sha256) addImage(row.file, row.sha256, 'docs/screenshots/controlled-engine-final-' + item.script.replace('verify-', '').replace('.mjs', '') + '-' + path.basename(row.file));
}
}
for (const sample of projects.results) for (const job of sample.browserJobs) for (const row of job.screenshots) {
assert.ok([1280, 390].includes(row.width)); assert.ok([900, 844].includes(row.height));
const file = path.join(projects.directory, sample.sampleName, 'page-' + row.width + '-' + row.artifactId + '.png');
addImage(file, row.sha256, 'docs/screenshots/controlled-engine-project-' + path.basename(file));
}
const db = new DatabaseSync(path.join(deployed.dataRoot, 'workbench.sqlite'), { readOnly: true }); db.exec('BEGIN');
try {
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all().map(row => row.name); assert.deepEqual(tables, Object.keys(deployed.database).sort());
for (const table of tables) { const rows = db.prepare('SELECT * FROM "' + table.replaceAll('"', '""') + '" ORDER BY rowid').all(); assert.equal(rows.length, deployed.database[table].count); assert.equal(digest(rows), deployed.database[table].sha256); }
} finally { db.exec('ROLLBACK'); db.close(); }
const base = 'http://127.0.0.1:4318', bootstrap = await (await fetch(base + '/api/bootstrap', { signal: AbortSignal.timeout(10000) })).json();
assert.equal(bootstrap.version, version); assert.equal((await (await fetch(base + '/api/health')).json()).pid, deployed.pid);
for (const row of deployed.preservedSessionSnapshots) {
const value = await (await fetch(base + '/api/sessions/' + row.id + '/export', { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, signal: AbortSignal.timeout(20000) })).json();
for (const kind of ['project', 'session', 'execution']) assert.equal(digest(value[kind]), row[kind]);
}
await assert.rejects(fetch('http://127.0.0.1:9229/json/list', { signal: AbortSignal.timeout(1000) }));
const copies = [
['.qa/archives-validation.json', 'docs/controlled-engine-archives-validation.json'], ['.qa/release-validation.json', 'docs/controlled-engine-release-validation.json'],
['.qa/source-package-validation.json', 'docs/controlled-engine-source-package-validation.json'], ['.qa/controlled-engine-final-inputs-comparison.json', 'docs/controlled-engine-git-inputs.json'],
['.qa/controlled-engine-source-tested-candidate.json', 'docs/controlled-engine-source-tested-candidate.json'], ['.qa/controlled-engine-source-candidate-archives-validation.json', 'docs/controlled-engine-source-candidate-archives-validation.json'],
['.qa/controlled-engine-deployed-ui-validation.json', 'docs/controlled-engine-deployed-ui-validation.json'],
];
for (const [from, to] of copies) fs.copyFileSync(from, to);
const reportNames = fs.readdirSync('docs').filter(name => name.startsWith('controlled-engine-') && /\.(json|tap|txt|mjs)$/.test(name) && name !== 'controlled-engine-release-identity.json').sort();
const reports = reportNames.map(name => fileIdentity('docs/' + name));
const identity = { success: true, version, sealedAt: new Date().toISOString(), codeCommit: finalBaseline.codeCommit, productCodeCommit: finalBaseline.productCodeCommit,
evidenceCommitBeforeSeal: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(),
tests: source.tests, uiGroups: ui.groups, uiTypes: ui.results.length, projectSamples: projects.results.length, projectCases: projects.cases.length, validationOnlyChange: qaChange,
postSourceValidationOnlyChange: portableQa, sourceInputsExactToFinal: compared.sourceInputsExactToFinal,
sourceInputFiles: source.inputFiles, buildFiles: baseline.builds.length, benchmarkFiles: baseline.benchmarks.length, comparedFiles: compared.comparedFiles,
gitExactInputs: compared.exactBytes, gitLfOnlyInputs: compared.normalized, release, deployedPid: deployed.pid, database: deployed.database,
preservedSessionSnapshots: deployed.preservedSessionSnapshots, modelCenter: deployed.modelCenter, allRowsAndExportsPreserved: true,
reports, logs: [...sourceLogs, ...uiLogs], images: [...images.values()], realModelRequests: 0,
acceptance: { controlledCompiledSource: source.actualControlled, controlledPortable: { cases: portable.actualControlled.cases.length, contained: true, fixtureModelRequests: portable.actualControlled.fixtureModelRequests },
purgeDefaultEnabled: false, purgeReasoningBenefit: 'not-measured-owned-response-fixture', desktopInstaller: 'planned-1.15', phonePairing: 'not-implemented' },
scope: 'Pinned controlled DeepSeek Harness and default-off source-pinned prompt experiment, owned local HTTP fixtures, actual three functional projects/reviews/checks/Edge/Git, complete independent source/portable/UI and original-host deployment. Historical A–E failures/results retain their original version; real inference benefit and other engine adapters remain separately unverified.' };
fs.writeFileSync('docs/controlled-engine-release-identity.json', JSON.stringify(identity, null, 2));
console.log(JSON.stringify({ success: true, tests: source.tests.tests, uiGroups: ui.groups, projects: projects.results.length, reports: reports.length, logs: identity.logs.length, images: identity.images.length, pid: deployed.pid }));