120 lines
14 KiB
JavaScript
120 lines
14 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { spawn, execFileSync } from 'node:child_process';
|
|
const workspace = process.cwd(), read = file => JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
const receipt = file => ({ path: path.resolve(file), bytes: fs.statSync(file).size, sha256: digest(file) });
|
|
const priorFile = '.qa/release-efficiency-source-b-original-result.json', prior = read(priorFile);
|
|
assert.equal(prior.success, false); assert.equal(prior.stage, 'actual contained Electron window');
|
|
assert.match(prior.error, /browsingLayout/); assert.match(prior.error, /openFilePreview/);
|
|
const release = read('releases/latest.json'), archives = read('.qa/archives-validation.json');
|
|
assert.equal(archives.success, true); assert.equal(archives.version, '1.23.0');
|
|
assert.ok(path.basename(release.portableRoot).endsWith('20261102025721172'));
|
|
const sourceRoot = fs.realpathSync.native(archives.extractionRoot), oldSource = fs.realpathSync.native(prior.sourceRoot);
|
|
assert.equal(path.basename(sourceRoot), path.basename(release.sourceRoot));
|
|
const directory = path.join(sourceRoot, '.qa', 'package-continuation-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
|
const files = (root, relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => entry.isDirectory() ? files(root, path.join(relative, entry.name)) : [path.join(relative, entry.name)]);
|
|
const inputFolders = ['src', 'tests', 'scripts', 'benchmarks', 'public', 'examples', 'research'];
|
|
const inputNames = [...inputFolders.flatMap(folder => files(path.join(sourceRoot, folder)).map(relative => path.join(folder, relative))),
|
|
'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', 'mobile.html', '.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].sort();
|
|
const snapshot = root => inputNames.map(relative => ({ path: relative.replaceAll('\\', '/'), bytes: fs.statSync(path.join(root, relative)).size, sha256: digest(path.join(root, relative)) }));
|
|
const inputs = snapshot(sourceRoot); assert.deepEqual(snapshot(oldSource), prior.inputs, 'Original failed-round inputs must still be byte exact.');
|
|
assert.deepEqual(inputs.map(value => value.path), prior.inputs.map(value => value.path));
|
|
const changedInputs = inputs.filter((value, index) => value.sha256 !== prior.inputs[index].sha256 || value.bytes !== prior.inputs[index].bytes).map(value => value.path);
|
|
assert.deepEqual(changedInputs.slice().sort(), ['.gitattributes', 'scripts/verify-desktop-window.mjs']);
|
|
const normalized = file => fs.readFileSync(file, 'utf8').replaceAll('\r\n', '\n');
|
|
const oldAssertion = " const browserMethods = ['1.22.0', '1.22.1'].includes(original.version) ? ['browsingLayout', ...(original.version === '1.22.1' ? ['openFilePreview'] : [])] : [];\n assert.deepEqual(surface.methods, [...browserMethods, 'pickProject', 'projectFolder', 'protocol', 'revealFile', 'reviewTask', 'status', 'stopHost'].sort());";
|
|
const newAssertion = " // This verifier runs the current source and its current preload. Keep an\n // explicit allowlist so later versions retain these reviewed capabilities\n // while any unexpected privileged method still fails acceptance.\n assert.deepEqual(surface.methods, ['browsingLayout', 'openFilePreview', 'pickProject', 'projectFolder', 'protocol', 'revealFile', 'reviewTask', 'status', 'stopHost'].sort());";
|
|
assert.ok(normalized(path.join(oldSource, 'scripts/verify-desktop-window.mjs')).includes(oldAssertion));
|
|
assert.equal(normalized(path.join(sourceRoot, 'scripts/verify-desktop-window.mjs')), normalized(path.join(oldSource, 'scripts/verify-desktop-window.mjs')).replace(oldAssertion, newAssertion));
|
|
const attributeAdditions = normalized(path.join(sourceRoot, '.gitattributes')).slice(normalized(path.join(oldSource, '.gitattributes')).length);
|
|
assert.ok(normalized(path.join(sourceRoot, '.gitattributes')).startsWith(normalized(path.join(oldSource, '.gitattributes'))));
|
|
assert.ok(attributeAdditions.split('\n').filter(Boolean).every(line => line.startsWith('docs/release-efficiency-')));
|
|
const inheritedLogs = [];
|
|
for (const log of prior.logs) {
|
|
const actual = receipt(log.path); assert.equal(actual.sha256, log.sha256); assert.equal(actual.bytes, log.bytes);
|
|
if (log.stage !== prior.stage) inheritedLogs.push({ ...log, inherited: true });
|
|
}
|
|
const stats = Object.fromEntries(['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo'].map(key => [key, 0]));
|
|
const testLogs = inheritedLogs.filter(value => /\.tap$/.test(value.path)); assert.equal(testLogs.length, 12);
|
|
for (const log of testLogs) {
|
|
const counts = Object.fromEntries([...fs.readFileSync(log.path, 'utf8').matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])]));
|
|
assert.ok(counts.tests > 0); assert.equal(counts.tests, counts.pass);
|
|
for (const key of Object.keys(stats)) { assert.ok(Number.isInteger(counts[key])); stats[key] += counts[key]; }
|
|
}
|
|
assert.equal(stats.tests, 703); assert.equal(stats.pass, 703);
|
|
for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0);
|
|
const finalJson = text => {
|
|
const singleLine = text.trim().split(/\r?\n/).findLast(line => line.startsWith('{') && line.endsWith('}'));
|
|
if (singleLine) return JSON.parse(singleLine);
|
|
for (const match of [...text.matchAll(/^\{/gm)].reverse()) { try { return JSON.parse(text.slice(match.index).trim()); } catch {} }
|
|
const evidencePath = [...text.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
|
|
if (evidencePath) {
|
|
assert.ok(path.resolve(evidencePath).startsWith(path.join(oldSource, '.qa') + path.sep));
|
|
return read(path.join(evidencePath, 'result.json'));
|
|
}
|
|
throw Error('Original log has no complete JSON result.');
|
|
};
|
|
const inheritedProofs = {};
|
|
for (const log of inheritedLogs.filter(value => /\.log$/.test(value.path) && !['install.log','build.log'].includes(path.basename(value.path)))) {
|
|
const text = fs.readFileSync(log.path, 'utf8'), summary = finalJson(text);
|
|
assert.ok(summary.success || summary.passed); inheritedProofs[path.basename(log.path)] = summary;
|
|
}
|
|
fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2));
|
|
fs.writeFileSync(path.join(directory, 'test-plan.json'), fs.readFileSync(path.join(prior.directory, 'test-plan.json')));
|
|
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
|
const allowedEnv = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON|AGENT_WORKBENCH_DEEPSEEK_ROOT|AGENT_WORKBENCH_PURGE_ROOT)$/i;
|
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => allowedEnv.test(key)));
|
|
const inheritedPath = Object.entries(env).find(([key]) => key.toLowerCase() === 'path')?.[1] || '';
|
|
for (const key of Object.keys(env)) if (key.toLowerCase() === 'path') delete env[key]; env.PATH = path.dirname(process.execPath) + path.delimiter + inheritedPath;
|
|
const logs = [], stages = []; let stage = 'offline install', python;
|
|
const save = value => { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(value, null, 2)); fs.writeFileSync('.qa/source-package-validation.json', JSON.stringify(value, null, 2)); };
|
|
async function run(args, name) {
|
|
console.log('Source continuation: ' + stage);
|
|
const file = path.join(directory, name); const output = fs.createWriteStream(file); let tail = '';
|
|
const code = await new Promise((resolve, reject) => {
|
|
const child = spawn(process.execPath, args, { cwd: sourceRoot, env, windowsHide: true, stdio: ['ignore','pipe','pipe'] });
|
|
const capture = bytes => { output.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-18000); };
|
|
child.stdout.on('data', capture); child.stderr.on('data', capture);
|
|
const timeout = setTimeout(() => { try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore', timeout: 10000 }); } catch {} }, 30 * 60 * 1000);
|
|
child.once('error', error => { clearTimeout(timeout); output.end(); reject(error); });
|
|
child.once('close', code => { clearTimeout(timeout); output.end(() => resolve(code)); });
|
|
});
|
|
logs.push({ stage, ...receipt(file) }); assert.equal(code, 0, stage + ': ' + tail); stages.push({ stage, success: true }); return tail;
|
|
}
|
|
try {
|
|
assert.equal(process.version, 'v24.22.1');
|
|
python = JSON.parse(execFileSync(env.AGENT_WORKBENCH_TEST_PYTHON, ['-I','-c','import json,sys,pytest;print(json.dumps({"version":sys.version.split()[0],"pytest":pytest.__version__}))'], { env, windowsHide: true, encoding: 'utf8' }));
|
|
assert.equal(python.pytest, '8.4.2'); assert.equal(python.version, '3.10.11');
|
|
await run([npmCli, 'ci','--offline','--ignore-scripts','--no-audit','--no-fund'], 'install.log');
|
|
stage = 'build'; await run([npmCli, 'run','build'], 'build.log');
|
|
const expected = {};
|
|
for (const folder of ['dist-client','dist-server','dist-desktop']) {
|
|
const names = files(path.join(sourceRoot, folder)); assert.deepEqual(names, files(path.join(oldSource, folder))); assert.deepEqual(names, files(path.join(workspace, folder)));
|
|
for (const relative of names) { const next = path.join(folder, relative), sha256 = digest(path.join(sourceRoot, next)); assert.equal(sha256, digest(path.join(oldSource,next))); assert.equal(sha256, digest(path.join(workspace,next))); expected[next.replaceAll('\\','/')] = sha256; }
|
|
}
|
|
const proofs = {};
|
|
const steps = [
|
|
['actual contained Electron window', ['scripts/verify-desktop-window.mjs'], 'desktop-window.log', () => read(path.join(sourceRoot,'.qa/desktop-window-latest.json')), value => assert.equal(value.results.length,7)],
|
|
['actual contained Electron engine UI', ['scripts/verify-controlled-ui.mjs','--desktop'], 'desktop-controlled-ui.log', () => read(read(path.join(sourceRoot,'.qa/desktop-controlled-ui-location.json')).result), value => { assert.equal(value.testedVersion,'1.23.0'); assert.equal(value.assertions.length,9); }],
|
|
['actual contained Electron engineering projects', ['scripts/verify-controlled-projects.mjs','--desktop'], 'desktop-projects.log', () => read(read(path.join(sourceRoot,'.qa/desktop-projects-location.json')).result), value => { assert.equal(value.cases.length,16); assert.deepEqual(value.results.map(row=>row.parentCheck.results[0].tests.passed),[8,9,10]); }],
|
|
['Responses SSE actual Node React Python engineering', ['scripts/verify-controlled-projects.mjs','--responses'], 'responses-engineering.log', () => read(read(path.join(sourceRoot,'.qa/responses-engineering-location.json')).result), value => { assert.equal(value.protocol,'responses-stream'); assert.deepEqual(value.results.map(row=>row.parentCheck.results[0].tests.passed),[8,9,10]); }],
|
|
['screenshot Task Node React Python engineering', ['scripts/verify-vision-projects.mjs'], 'vision-engineering.log', () => read(read(path.join(sourceRoot,'.qa/vision-engineering-location.json')).result), value => assert.deepEqual(value.results.map(row=>row.check.results[0].tests.passed),[8,9,10])],
|
|
['cached index original requests and Node React Python engineering', ['scripts/verify-project-cache-projects.mjs'], 'project-cache-engineering.log', () => read(read(path.join(sourceRoot,'.qa/project-cache-engineering-location.json')).result), value => { assert.equal(value.cases.length,9); assert.deepEqual(value.results.map(row=>row.check.results[0].tests.passed),[8,9,10]); }],
|
|
['compiled fixed engineering evaluations', ['scripts/verify-evaluations-release.mjs','--app-root',sourceRoot], 'evaluations-release.log', output => read(path.join([...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)[1].trim(),'result.json')), value => { assert.equal(value.assertions.length,5); assert.deepEqual(value.pageErrors,[]); }],
|
|
['fixed Node React Python and source-bound module engineering', ['scripts/verify-evaluations-projects.mjs'], 'evaluations-engineering.log', () => read(read(path.join(sourceRoot,'.qa/engineering-evals-projects-location.json')).result), value => { assert.equal(value.assertions.length,14); assert.deepEqual(value.results.map(row=>row.parentCheck.results[0].tests.passed),[8,9,10,9]); }],
|
|
];
|
|
for (const [label,args,name,load,check] of steps) {
|
|
stage = label; const output = await run(args,name), proof = load(output); assert.ok(proof.success || proof.passed); check(proof);
|
|
if (proof.realModelRequests !== undefined) assert.equal(proof.realModelRequests,0); proofs[name] = proof;
|
|
}
|
|
assert.deepEqual(snapshot(sourceRoot), inputs); assert.deepEqual(snapshot(oldSource), prior.inputs);
|
|
const result = { success:true, version:'1.23.0', testedAt:new Date().toISOString(), sourceRoot,directory,nodeVersion:process.version,python,offlineInstall:true,
|
|
execution:'strict-input-bound continuation after original failed desktop assertion; not a new complete execution of the inherited tests',
|
|
originalFailure:receipt(priorFile),originalSourceRoot:oldSource,originalOverallSuccess:false,changedInputs,inputs,inputFiles:inputs.length,
|
|
buildMatchedFiles:Object.keys(expected).length,buildFiles:expected,tests:stats,inheritedLogs,logs,stages,inheritedProofs,proofs,realModelRequests:0 };
|
|
save(result); console.log(JSON.stringify({success:true,sourceRoot,directory,tests:stats,changedInputs,buildMatchedFiles:result.buildMatchedFiles,stages:stages.length}));
|
|
} catch(error) { save({success:false,version:'1.23.0',sourceRoot,directory,stage,error:error.stack,inputs,logs,inheritedLogs,originalFailure:receipt(priorFile),changedInputs}); throw error; }
|