Files
Agent-Workbench/scripts/verify-tests.mjs
T
2026-10-30 20:34:54 +08:00

60 lines
5.0 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { spawn } from 'node:child_process';
if (process.argv.includes('--help')) { console.log('node scripts/verify-tests.mjs — complete regression with frozen input/log hashes'); process.exit(0); }
const root = process.cwd(), version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
const directory = path.resolve('.qa', 'tests-' + version + '-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
const hash = filename => createHash('sha256').update(fs.readFileSync(filename)).digest('hex');
function filesIn(relative) {
return fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
const filename = path.join(relative, entry.name); if (entry.isSymbolicLink()) throw new Error('Regression input cannot be a link: ' + filename);
return entry.isDirectory() ? filesIn(filename) : [filename];
});
}
const paths = ['src', 'tests', 'scripts'].flatMap(filesIn).concat(['package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html']).sort();
const snapshot = () => paths.map(filename => ({ path: filename.replaceAll('\\', '/'), bytes: fs.statSync(filename).size, sha256: hash(filename) }));
const inputs = snapshot(), tests = fs.readdirSync('tests').filter(name => name.endsWith('.test.mjs')).sort().map(name => path.join('tests', name));
assert.ok(tests.length); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2) + '\n');
const allowed = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON)$/i;
const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => allowed.test(name)));
const timeoutMs = 30 * 60 * 1000;
const logPath = path.join(directory, 'tests.tap'), output = fs.createWriteStream(logPath), startedAt = new Date().toISOString(); let result, failure, timedOut = false;
try {
await new Promise((resolve, reject) => {
const child = spawn(process.execPath, ['--test', '--test-concurrency=2', '--test-reporter=tap', ...tests], { cwd: root, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
child.stdout.pipe(output, { end: false }); child.stderr.pipe(output, { end: false });
const timer = setTimeout(() => {
timedOut = true;
if (process.platform === 'win32' && child.pid) {
const killer = spawn('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' });
killer.once('error', () => child.kill());
killer.once('close', code => { if (code !== 0) child.kill(); });
} else child.kill('SIGTERM');
}, timeoutMs);
child.once('error', error => { clearTimeout(timer); reject(error); });
child.once('close', code => {
clearTimeout(timer);
if (timedOut) reject(new Error(`Regression timed out after ${timeoutMs}ms; its process tree was terminated.`));
else code === 0 ? resolve() : reject(new Error('Regression process exit: ' + code));
});
});
} catch (error) { failure = error; }
await new Promise(resolve => output.end(resolve));
try {
const text = fs.readFileSync(logPath, 'utf8'), stats = Object.fromEntries([...text.matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])]));
if (failure) throw Object.assign(failure, { tail: text.slice(-16000) });
assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0);
assert.deepEqual(snapshot(), inputs, 'Inputs changed during regression; this is not final-source evidence.');
result = { success: true, version, startedAt, finishedAt: new Date().toISOString(), timeoutMs, tests: stats, directory, nodeVersion: process.version, inputFiles: inputs.length, inputs,
log: { path: logPath, bytes: fs.statSync(logPath).size, sha256: hash(logPath) }, credentialsHandling: 'Only runtime/system paths and the independent QA Python path are inherited; no real model configuration or credentials.' };
} catch (error) {
result = { success: false, version, startedAt, finishedAt: new Date().toISOString(), timeoutMs, timedOut, directory, error: error.stack, tail: error.tail, inputs, log: { path: logPath, sha256: hash(logPath) } }; process.exitCode = 1;
}
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n'); fs.writeFileSync('.qa/tests-validation.json', JSON.stringify(result, null, 2) + '\n');
// Release evidence is archived explicitly by the release workflow. A development
// regression must not replace a previously published report for the same version.
console.log(JSON.stringify({ success: result.success, version, tests: result.tests, inputFiles: result.inputFiles, directory, error: result.error, tail: result.tail }, null, 2));