Files
Agent-Workbench/scripts/verify-all-ui.mjs
T

83 lines
7.0 KiB
JavaScript

import { releaseEvidencePrefix } from './qa-evidence.mjs';
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
import { gitEnvironment } from '../src/core/git-process.mjs';
import { randomUUID } from 'node:crypto';
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version), 'Review evidence paths when introducing the next release.');
const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = [];
const cases = [
...(version === '1.11.0' ? [['verify-task-plans-ui.mjs', 'docs/task-plans-task-ui-validation.json']] : []),
...(['1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []),
['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'],
['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'],
['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`],
['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`],
['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`],
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []),
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []),
...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []),
...(['1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []),
];
fs.mkdirSync('.qa', { recursive: true });
const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']);
async function startHost(script) {
const data = path.resolve('.qa', 'ui-host-' + script.replace('.mjs', '') + '-' + randomUUID());
fs.mkdirSync(data, { recursive: true });
const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env: { ...gitEnvironment(),
AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let url = '', output = '', errors = '';
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
const deadline = Date.now() + 10000;
while (!url && child.exitCode === null && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 30));
if (!url) { child.kill(); throw new Error('Independent UI host failed: ' + errors); }
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
assert.equal(bootstrap.version, version); assert.equal(bootstrap.config.hasKey, false); assert.equal(bootstrap.config.mode, 'demo');
return { url, data, async close() {
try {
await fetch(url + '/api/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, body: '{}' });
const stopDeadline = Date.now() + 10000;
while (child.exitCode === null && child.signalCode === null && Date.now() < stopDeadline) await new Promise(resolve => setTimeout(resolve, 30));
} finally { if (child.exitCode === null && child.signalCode === null) child.kill(); }
} };
}
for (const [script, proof] of cases) {
console.log('UI regression: ' + script); const started = Date.now(), log = `.qa/${prefix}-${script.replace('.mjs', '')}.log`;
const stream = fs.createWriteStream(log);
let host;
try {
if (externalHost.has(script)) host = await startHost(script);
await new Promise((resolve, reject) => {
const child = spawn(process.execPath, [path.join('scripts', script)], { cwd: process.cwd(), env: { ...gitEnvironment(), ...(host ? { AGENT_WORKBENCH_URL: host.url } : {}) }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let tail = ''; const capture = bytes => { stream.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-10000); };
child.stdout.on('data', capture); child.stderr.on('data', capture);
const timer = setTimeout(() => child.kill(), 240000);
child.once('error', error => { clearTimeout(timer); reject(error); });
child.once('close', code => { clearTimeout(timer); code === 0 ? resolve() : reject(new Error(script + ' failed (' + code + '): ' + tail)); });
});
stream.end(); const evidence = JSON.parse(fs.readFileSync(proof, 'utf8'));
assert.notEqual(evidence.success, false); if (evidence.version) assert.equal(evidence.version, version);
const passed = evidence.passed ?? evidence.cases;
assert.ok(Array.isArray(passed) && passed.length > 0);
const errors = evidence.pageErrors ?? [], failures = evidence.requestFailures ?? [];
assert.equal(errors.length, 0); assert.equal(failures.length, 0);
const saved = script === 'verify-ui.mjs' ? `docs/${prefix}-basic-ui-validation.json`
: ['verify-workspaces-ui.mjs', 'verify-task-plans-ui.mjs'].includes(script) ? proof
: `docs/${prefix}-${script.replace('verify-', '').replace('.mjs', '')}-validation.json`;
if (path.resolve(saved) !== path.resolve(proof)) fs.copyFileSync(proof, saved);
results.push({ script, success: true, groups: passed.length, durationMs: Date.now() - started, evidence: saved, log });
console.log('PASS ' + script + ': ' + passed.length + ' groups');
} catch (error) {
stream.end(); fs.writeFileSync(resultFile, JSON.stringify({ success: false, version, at: new Date().toISOString(), results, error: error.message, script, log }, null, 2) + '\n');
throw error;
} finally { await host?.close(); }
}
const result = { success: true, version, at: new Date().toISOString(), groups: results.reduce((sum, item) => sum + item.groups, 0), results,
scope: 'Every verifier owns an independent port and synthetic data directory. Synthetic provider fixtures verify protocols; configured-model acceptance is separate. No production service or real key read/copied.' };
fs.writeFileSync(resultFile, JSON.stringify(result, null, 2) + '\n');
fs.copyFileSync(resultFile, 'docs/' + prefix + '-all-ui-validation.json'); console.log(JSON.stringify(result, null, 2));