import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash, randomUUID } from 'node:crypto'; import { spawn } from 'node:child_process'; if (process.argv.includes('--help')) { console.log('node scripts/verify-tests.mjs — complete regression with frozen input/log hashes'); process.exit(0); } const root = process.cwd(), version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version; const directory = path.resolve('.qa', 'tests-' + version + '-' + randomUUID()); fs.mkdirSync(directory, { recursive: true }); const hash = filename => createHash('sha256').update(fs.readFileSync(filename)).digest('hex'); function filesIn(relative) { return fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => { const filename = path.join(relative, entry.name); if (entry.isSymbolicLink()) throw new Error('Regression input cannot be a link: ' + filename); return entry.isDirectory() ? filesIn(filename) : [filename]; }); } const paths = ['src', 'tests', 'scripts'].flatMap(filesIn).concat(['package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html']).sort(); const snapshot = () => paths.map(filename => ({ path: filename.replaceAll('\\', '/'), bytes: fs.statSync(filename).size, sha256: hash(filename) })); const inputs = snapshot(), tests = fs.readdirSync('tests').filter(name => name.endsWith('.test.mjs')).sort().map(name => path.join('tests', name)); assert.ok(tests.length); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2) + '\n'); const allowed = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON)$/i; const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => allowed.test(name))); const logPath = path.join(directory, 'tests.tap'), output = fs.createWriteStream(logPath), startedAt = new Date().toISOString(); let result, failure; try { await new Promise((resolve, reject) => { const child = spawn(process.execPath, ['--test', '--test-concurrency=2', '--test-reporter=tap', ...tests], { cwd: root, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); child.stdout.pipe(output, { end: false }); child.stderr.pipe(output, { end: false }); const timer = setTimeout(() => child.kill(), 600000); child.once('error', error => { clearTimeout(timer); reject(error); }); child.once('close', code => { clearTimeout(timer); code === 0 ? resolve() : reject(new Error('Regression process exit: ' + code)); }); }); } catch (error) { failure = error; } await new Promise(resolve => output.end(resolve)); try { const text = fs.readFileSync(logPath, 'utf8'), stats = Object.fromEntries([...text.matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])])); if (failure) throw Object.assign(failure, { tail: text.slice(-16000) }); assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0); assert.deepEqual(snapshot(), inputs, 'Inputs changed during regression; this is not final-source evidence.'); result = { success: true, version, startedAt, finishedAt: new Date().toISOString(), tests: stats, directory, nodeVersion: process.version, inputFiles: inputs.length, inputs, log: { path: logPath, bytes: fs.statSync(logPath).size, sha256: hash(logPath) }, credentialsHandling: 'Only runtime/system paths and the independent QA Python path are inherited; no real model configuration or credentials.' }; } catch (error) { result = { success: false, version, startedAt, finishedAt: new Date().toISOString(), directory, error: error.stack, tail: error.tail, inputs, log: { path: logPath, sha256: hash(logPath) } }; process.exitCode = 1; } fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n'); fs.writeFileSync('.qa/tests-validation.json', JSON.stringify(result, null, 2) + '\n'); if (version === '1.8.0') fs.writeFileSync('docs/rea-engine-tests-validation.json', JSON.stringify(result, null, 2) + '\n'); if (version === '1.9.0') fs.writeFileSync('docs/external-engines-tests-validation.json', JSON.stringify(result, null, 2) + '\n'); if (version === '1.10.0') fs.writeFileSync('docs/browser-engine-tests-validation.json', JSON.stringify(result, null, 2) + '\n'); if (version === '1.11.0') fs.writeFileSync('docs/task-plans-tests-validation.json', JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify({ success: result.success, version, tests: result.tests, inputFiles: result.inputFiles, directory, error: result.error, tail: result.tail }, null, 2));