60 lines
5.0 KiB
JavaScript
60 lines
5.0 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { spawn } from 'node:child_process';
|
|
|
|
if (process.argv.includes('--help')) { console.log('node scripts/verify-tests.mjs — complete regression with frozen input/log hashes'); process.exit(0); }
|
|
const root = process.cwd(), version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
|
const directory = path.resolve('.qa', 'tests-' + version + '-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
|
const hash = filename => createHash('sha256').update(fs.readFileSync(filename)).digest('hex');
|
|
function filesIn(relative) {
|
|
return fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
|
|
const filename = path.join(relative, entry.name); if (entry.isSymbolicLink()) throw new Error('Regression input cannot be a link: ' + filename);
|
|
return entry.isDirectory() ? filesIn(filename) : [filename];
|
|
});
|
|
}
|
|
const paths = ['src', 'tests', 'scripts'].flatMap(filesIn).concat(['package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html']).sort();
|
|
const snapshot = () => paths.map(filename => ({ path: filename.replaceAll('\\', '/'), bytes: fs.statSync(filename).size, sha256: hash(filename) }));
|
|
const inputs = snapshot(), tests = fs.readdirSync('tests').filter(name => name.endsWith('.test.mjs')).sort().map(name => path.join('tests', name));
|
|
assert.ok(tests.length); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2) + '\n');
|
|
const allowed = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON)$/i;
|
|
const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => allowed.test(name)));
|
|
const timeoutMs = 30 * 60 * 1000;
|
|
const logPath = path.join(directory, 'tests.tap'), output = fs.createWriteStream(logPath), startedAt = new Date().toISOString(); let result, failure, timedOut = false;
|
|
try {
|
|
await new Promise((resolve, reject) => {
|
|
const child = spawn(process.execPath, ['--test', '--test-concurrency=2', '--test-reporter=tap', ...tests], { cwd: root, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
child.stdout.pipe(output, { end: false }); child.stderr.pipe(output, { end: false });
|
|
const timer = setTimeout(() => {
|
|
timedOut = true;
|
|
if (process.platform === 'win32' && child.pid) {
|
|
const killer = spawn('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' });
|
|
killer.once('error', () => child.kill());
|
|
killer.once('close', code => { if (code !== 0) child.kill(); });
|
|
} else child.kill('SIGTERM');
|
|
}, timeoutMs);
|
|
child.once('error', error => { clearTimeout(timer); reject(error); });
|
|
child.once('close', code => {
|
|
clearTimeout(timer);
|
|
if (timedOut) reject(new Error(`Regression timed out after ${timeoutMs}ms; its process tree was terminated.`));
|
|
else code === 0 ? resolve() : reject(new Error('Regression process exit: ' + code));
|
|
});
|
|
});
|
|
} catch (error) { failure = error; }
|
|
await new Promise(resolve => output.end(resolve));
|
|
try {
|
|
const text = fs.readFileSync(logPath, 'utf8'), stats = Object.fromEntries([...text.matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])]));
|
|
if (failure) throw Object.assign(failure, { tail: text.slice(-16000) });
|
|
assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0);
|
|
assert.deepEqual(snapshot(), inputs, 'Inputs changed during regression; this is not final-source evidence.');
|
|
result = { success: true, version, startedAt, finishedAt: new Date().toISOString(), timeoutMs, tests: stats, directory, nodeVersion: process.version, inputFiles: inputs.length, inputs,
|
|
log: { path: logPath, bytes: fs.statSync(logPath).size, sha256: hash(logPath) }, credentialsHandling: 'Only runtime/system paths and the independent QA Python path are inherited; no real model configuration or credentials.' };
|
|
} catch (error) {
|
|
result = { success: false, version, startedAt, finishedAt: new Date().toISOString(), timeoutMs, timedOut, directory, error: error.stack, tail: error.tail, inputs, log: { path: logPath, sha256: hash(logPath) } }; process.exitCode = 1;
|
|
}
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n'); fs.writeFileSync('.qa/tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
|
// Release evidence is archived explicitly by the release workflow. A development
|
|
// regression must not replace a previously published report for the same version.
|
|
console.log(JSON.stringify({ success: result.success, version, tests: result.tests, inputFiles: result.inputFiles, directory, error: result.error, tail: result.tail }, null, 2));
|