import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash, randomUUID } from 'node:crypto'; import { spawn, execFileSync } from 'node:child_process'; const workspace = process.cwd(), read = file => JSON.parse(fs.readFileSync(file, 'utf8')); const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex'); const receipt = file => ({ path: path.resolve(file), bytes: fs.statSync(file).size, sha256: digest(file) }); const priorFile = '.qa/release-efficiency-source-b-original-result.json', prior = read(priorFile); assert.equal(prior.success, false); assert.equal(prior.stage, 'actual contained Electron window'); assert.match(prior.error, /browsingLayout/); assert.match(prior.error, /openFilePreview/); const release = read('releases/latest.json'), archives = read('.qa/archives-validation.json'); assert.equal(archives.success, true); assert.equal(archives.version, '1.23.0'); assert.ok(path.basename(release.portableRoot).endsWith('20261102025721172')); const sourceRoot = fs.realpathSync.native(archives.extractionRoot), oldSource = fs.realpathSync.native(prior.sourceRoot); assert.equal(path.basename(sourceRoot), path.basename(release.sourceRoot)); const directory = path.join(sourceRoot, '.qa', 'package-continuation-' + randomUUID()); fs.mkdirSync(directory, { recursive: true }); const files = (root, relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => entry.isDirectory() ? files(root, path.join(relative, entry.name)) : [path.join(relative, entry.name)]); const inputFolders = ['src', 'tests', 'scripts', 'benchmarks', 'public', 'examples', 'research']; const inputNames = [...inputFolders.flatMap(folder => files(path.join(sourceRoot, folder)).map(relative => path.join(folder, relative))), 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', 'mobile.html', '.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].sort(); const snapshot = root => inputNames.map(relative => ({ path: relative.replaceAll('\\', '/'), bytes: fs.statSync(path.join(root, relative)).size, sha256: digest(path.join(root, relative)) })); const inputs = snapshot(sourceRoot); assert.deepEqual(snapshot(oldSource), prior.inputs, 'Original failed-round inputs must still be byte exact.'); assert.deepEqual(inputs.map(value => value.path), prior.inputs.map(value => value.path)); const changedInputs = inputs.filter((value, index) => value.sha256 !== prior.inputs[index].sha256 || value.bytes !== prior.inputs[index].bytes).map(value => value.path); assert.deepEqual(changedInputs.slice().sort(), ['.gitattributes', 'scripts/verify-desktop-window.mjs']); const normalized = file => fs.readFileSync(file, 'utf8').replaceAll('\r\n', '\n'); const oldAssertion = " const browserMethods = ['1.22.0', '1.22.1'].includes(original.version) ? ['browsingLayout', ...(original.version === '1.22.1' ? ['openFilePreview'] : [])] : [];\n assert.deepEqual(surface.methods, [...browserMethods, 'pickProject', 'projectFolder', 'protocol', 'revealFile', 'reviewTask', 'status', 'stopHost'].sort());"; const newAssertion = " // This verifier runs the current source and its current preload. Keep an\n // explicit allowlist so later versions retain these reviewed capabilities\n // while any unexpected privileged method still fails acceptance.\n assert.deepEqual(surface.methods, ['browsingLayout', 'openFilePreview', 'pickProject', 'projectFolder', 'protocol', 'revealFile', 'reviewTask', 'status', 'stopHost'].sort());"; assert.ok(normalized(path.join(oldSource, 'scripts/verify-desktop-window.mjs')).includes(oldAssertion)); assert.equal(normalized(path.join(sourceRoot, 'scripts/verify-desktop-window.mjs')), normalized(path.join(oldSource, 'scripts/verify-desktop-window.mjs')).replace(oldAssertion, newAssertion)); const attributeAdditions = normalized(path.join(sourceRoot, '.gitattributes')).slice(normalized(path.join(oldSource, '.gitattributes')).length); assert.ok(normalized(path.join(sourceRoot, '.gitattributes')).startsWith(normalized(path.join(oldSource, '.gitattributes')))); assert.ok(attributeAdditions.split('\n').filter(Boolean).every(line => line.startsWith('docs/release-efficiency-'))); const inheritedLogs = []; for (const log of prior.logs) { const actual = receipt(log.path); assert.equal(actual.sha256, log.sha256); assert.equal(actual.bytes, log.bytes); if (log.stage !== prior.stage) inheritedLogs.push({ ...log, inherited: true }); } const stats = Object.fromEntries(['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo'].map(key => [key, 0])); const testLogs = inheritedLogs.filter(value => /\.tap$/.test(value.path)); assert.equal(testLogs.length, 12); for (const log of testLogs) { const counts = Object.fromEntries([...fs.readFileSync(log.path, 'utf8').matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])])); assert.ok(counts.tests > 0); assert.equal(counts.tests, counts.pass); for (const key of Object.keys(stats)) { assert.ok(Number.isInteger(counts[key])); stats[key] += counts[key]; } } assert.equal(stats.tests, 703); assert.equal(stats.pass, 703); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0); const finalJson = text => { const singleLine = text.trim().split(/\r?\n/).findLast(line => line.startsWith('{') && line.endsWith('}')); if (singleLine) return JSON.parse(singleLine); for (const match of [...text.matchAll(/^\{/gm)].reverse()) { try { return JSON.parse(text.slice(match.index).trim()); } catch {} } const evidencePath = [...text.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); if (evidencePath) { assert.ok(path.resolve(evidencePath).startsWith(path.join(oldSource, '.qa') + path.sep)); return read(path.join(evidencePath, 'result.json')); } throw Error('Original log has no complete JSON result.'); }; const inheritedProofs = {}; for (const log of inheritedLogs.filter(value => /\.log$/.test(value.path) && !['install.log','build.log'].includes(path.basename(value.path)))) { const text = fs.readFileSync(log.path, 'utf8'), summary = finalJson(text); assert.ok(summary.success || summary.passed); inheritedProofs[path.basename(log.path)] = summary; } fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2)); fs.writeFileSync(path.join(directory, 'test-plan.json'), fs.readFileSync(path.join(prior.directory, 'test-plan.json'))); const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js'); const allowedEnv = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON|AGENT_WORKBENCH_DEEPSEEK_ROOT|AGENT_WORKBENCH_PURGE_ROOT)$/i; const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => allowedEnv.test(key))); const inheritedPath = Object.entries(env).find(([key]) => key.toLowerCase() === 'path')?.[1] || ''; for (const key of Object.keys(env)) if (key.toLowerCase() === 'path') delete env[key]; env.PATH = path.dirname(process.execPath) + path.delimiter + inheritedPath; const logs = [], stages = []; let stage = 'offline install', python; const save = value => { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(value, null, 2)); fs.writeFileSync('.qa/source-package-validation.json', JSON.stringify(value, null, 2)); }; async function run(args, name) { console.log('Source continuation: ' + stage); const file = path.join(directory, name); const output = fs.createWriteStream(file); let tail = ''; const code = await new Promise((resolve, reject) => { const child = spawn(process.execPath, args, { cwd: sourceRoot, env, windowsHide: true, stdio: ['ignore','pipe','pipe'] }); const capture = bytes => { output.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-18000); }; child.stdout.on('data', capture); child.stderr.on('data', capture); const timeout = setTimeout(() => { try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore', timeout: 10000 }); } catch {} }, 30 * 60 * 1000); child.once('error', error => { clearTimeout(timeout); output.end(); reject(error); }); child.once('close', code => { clearTimeout(timeout); output.end(() => resolve(code)); }); }); logs.push({ stage, ...receipt(file) }); assert.equal(code, 0, stage + ': ' + tail); stages.push({ stage, success: true }); return tail; } try { assert.equal(process.version, 'v24.22.1'); python = JSON.parse(execFileSync(env.AGENT_WORKBENCH_TEST_PYTHON, ['-I','-c','import json,sys,pytest;print(json.dumps({"version":sys.version.split()[0],"pytest":pytest.__version__}))'], { env, windowsHide: true, encoding: 'utf8' })); assert.equal(python.pytest, '8.4.2'); assert.equal(python.version, '3.10.11'); await run([npmCli, 'ci','--offline','--ignore-scripts','--no-audit','--no-fund'], 'install.log'); stage = 'build'; await run([npmCli, 'run','build'], 'build.log'); const expected = {}; for (const folder of ['dist-client','dist-server','dist-desktop']) { const names = files(path.join(sourceRoot, folder)); assert.deepEqual(names, files(path.join(oldSource, folder))); assert.deepEqual(names, files(path.join(workspace, folder))); for (const relative of names) { const next = path.join(folder, relative), sha256 = digest(path.join(sourceRoot, next)); assert.equal(sha256, digest(path.join(oldSource,next))); assert.equal(sha256, digest(path.join(workspace,next))); expected[next.replaceAll('\\','/')] = sha256; } } const proofs = {}; const steps = [ ['actual contained Electron window', ['scripts/verify-desktop-window.mjs'], 'desktop-window.log', () => read(path.join(sourceRoot,'.qa/desktop-window-latest.json')), value => assert.equal(value.results.length,7)], ['actual contained Electron engine UI', ['scripts/verify-controlled-ui.mjs','--desktop'], 'desktop-controlled-ui.log', () => read(read(path.join(sourceRoot,'.qa/desktop-controlled-ui-location.json')).result), value => { assert.equal(value.testedVersion,'1.23.0'); assert.equal(value.assertions.length,9); }], ['actual contained Electron engineering projects', ['scripts/verify-controlled-projects.mjs','--desktop'], 'desktop-projects.log', () => read(read(path.join(sourceRoot,'.qa/desktop-projects-location.json')).result), value => { assert.equal(value.cases.length,16); assert.deepEqual(value.results.map(row=>row.parentCheck.results[0].tests.passed),[8,9,10]); }], ['Responses SSE actual Node React Python engineering', ['scripts/verify-controlled-projects.mjs','--responses'], 'responses-engineering.log', () => read(read(path.join(sourceRoot,'.qa/responses-engineering-location.json')).result), value => { assert.equal(value.protocol,'responses-stream'); assert.deepEqual(value.results.map(row=>row.parentCheck.results[0].tests.passed),[8,9,10]); }], ['screenshot Task Node React Python engineering', ['scripts/verify-vision-projects.mjs'], 'vision-engineering.log', () => read(read(path.join(sourceRoot,'.qa/vision-engineering-location.json')).result), value => assert.deepEqual(value.results.map(row=>row.check.results[0].tests.passed),[8,9,10])], ['cached index original requests and Node React Python engineering', ['scripts/verify-project-cache-projects.mjs'], 'project-cache-engineering.log', () => read(read(path.join(sourceRoot,'.qa/project-cache-engineering-location.json')).result), value => { assert.equal(value.cases.length,9); assert.deepEqual(value.results.map(row=>row.check.results[0].tests.passed),[8,9,10]); }], ['compiled fixed engineering evaluations', ['scripts/verify-evaluations-release.mjs','--app-root',sourceRoot], 'evaluations-release.log', output => read(path.join([...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)[1].trim(),'result.json')), value => { assert.equal(value.assertions.length,5); assert.deepEqual(value.pageErrors,[]); }], ['fixed Node React Python and source-bound module engineering', ['scripts/verify-evaluations-projects.mjs'], 'evaluations-engineering.log', () => read(read(path.join(sourceRoot,'.qa/engineering-evals-projects-location.json')).result), value => { assert.equal(value.assertions.length,14); assert.deepEqual(value.results.map(row=>row.parentCheck.results[0].tests.passed),[8,9,10,9]); }], ]; for (const [label,args,name,load,check] of steps) { stage = label; const output = await run(args,name), proof = load(output); assert.ok(proof.success || proof.passed); check(proof); if (proof.realModelRequests !== undefined) assert.equal(proof.realModelRequests,0); proofs[name] = proof; } assert.deepEqual(snapshot(sourceRoot), inputs); assert.deepEqual(snapshot(oldSource), prior.inputs); const result = { success:true, version:'1.23.0', testedAt:new Date().toISOString(), sourceRoot,directory,nodeVersion:process.version,python,offlineInstall:true, execution:'strict-input-bound continuation after original failed desktop assertion; not a new complete execution of the inherited tests', originalFailure:receipt(priorFile),originalSourceRoot:oldSource,originalOverallSuccess:false,changedInputs,inputs,inputFiles:inputs.length, buildMatchedFiles:Object.keys(expected).length,buildFiles:expected,tests:stats,inheritedLogs,logs,stages,inheritedProofs,proofs,realModelRequests:0 }; save(result); console.log(JSON.stringify({success:true,sourceRoot,directory,tests:stats,changedInputs,buildMatchedFiles:result.buildMatchedFiles,stages:stages.length})); } catch(error) { save({success:false,version:'1.23.0',sourceRoot,directory,stage,error:error.stack,inputs,logs,inheritedLogs,originalFailure:receipt(priorFile),changedInputs}); throw error; }