120 lines
15 KiB
JavaScript
120 lines
15 KiB
JavaScript
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { spawn, execFileSync } from 'node:child_process';
|
|
import { gitEnvironment } from '../src/core/git-process.mjs';
|
|
import { randomUUID } from 'node:crypto';
|
|
|
|
if (process.argv.slice(2).some(value => value !== '--help')) throw new Error('Unsupported argument; use --help.');
|
|
if (process.argv.includes('--help')) { console.log('node scripts/verify-all-ui.mjs - run serial desktop/390px checks in independent local hosts; use explicit QA Python/Harness/Purge source paths for 1.15.'); process.exit(0); }
|
|
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
|
assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version), 'Review evidence paths when introducing the next release.');
|
|
const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = [];
|
|
const qaEnvironment = Object.fromEntries(['AGENT_WORKBENCH_TEST_PYTHON', 'AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_178_ROOT', 'AGENT_WORKBENCH_PURGE_181_ROOT']
|
|
.filter(name => process.env[name]).map(name => [name, process.env[name]]));
|
|
for (const [name, value] of Object.entries(qaEnvironment)) assert.ok(path.isAbsolute(value) && fs.existsSync(value), 'Use an existing absolute QA runtime/source path: ' + name);
|
|
const cases = [
|
|
...(['1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-task-agreement-ui.mjs', null], ['verify-task-follow-up-ui.mjs', null]] : []),
|
|
...(['1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-file-results-ui.mjs', null]] : []),
|
|
...(['1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-release-efficiency-ui.mjs', null]] : []),
|
|
...(['1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-local-files.mjs', null]] : []),
|
|
...(['1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version)?[['verify-embedded-browser.mjs',null]]:[]),
|
|
...(['1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version)?[['verify-environment-ui.mjs',null],['verify-model-fidelity-ui.mjs',null],['verify-model-wait-ui.mjs',null],['verify-task-control-ui.mjs',null]]:[]),
|
|
...(['1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-evaluations-ui.mjs', null]] : []),
|
|
...(['1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-project-cache-ui.mjs', null]] : []),
|
|
...(['1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-vision-ui.mjs', null]] : []),
|
|
...(['1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-model-protocols-ui.mjs', null]] : []),
|
|
...(['1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-companion-ui.mjs', null]] : []),
|
|
...(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-controlled-ui.mjs', null], ['verify-purge-ui.mjs', null]] : []),
|
|
...(['1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-workers-ui.mjs', null]] : []),
|
|
...(['1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []),
|
|
...(['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []),
|
|
['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'],
|
|
['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'],
|
|
['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`],
|
|
['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`],
|
|
['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`],
|
|
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []),
|
|
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []),
|
|
...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []),
|
|
...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []),
|
|
...(['1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0', '1.25.1', '1.25.2'].includes(version) ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []),
|
|
];
|
|
fs.mkdirSync('.qa', { recursive: true });
|
|
const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']);
|
|
async function startHost(script) {
|
|
const data = path.resolve('.qa', 'ui-host-' + script.replace('.mjs', '') + '-' + randomUUID());
|
|
fs.mkdirSync(data, { recursive: true });
|
|
const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env: { ...gitEnvironment(),
|
|
AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
let url = '', output = '', errors = '';
|
|
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
|
|
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
|
const deadline = Date.now() + 10000;
|
|
while (!url && child.exitCode === null && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 30));
|
|
if (!url) { child.kill(); throw new Error('Independent UI host failed: ' + errors); }
|
|
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
|
assert.equal(bootstrap.version, version); assert.equal(bootstrap.config.hasKey, false); assert.equal(bootstrap.config.mode, 'demo');
|
|
return { url, data, async close() {
|
|
try {
|
|
await fetch(url + '/api/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, body: '{}' });
|
|
const stopDeadline = Date.now() + 10000;
|
|
while (child.exitCode === null && child.signalCode === null && Date.now() < stopDeadline) await new Promise(resolve => setTimeout(resolve, 30));
|
|
} finally { if (child.exitCode === null && child.signalCode === null) child.kill(); }
|
|
} };
|
|
}
|
|
for (const [script, fixedProof] of cases) {
|
|
console.log('UI regression: ' + script); const started = Date.now(), log = `.qa/${prefix}-${script.replace('.mjs', '')}.log`;
|
|
const stream = fs.createWriteStream(log);
|
|
let host;
|
|
try {
|
|
if (externalHost.has(script)) host = await startHost(script);
|
|
const output = await new Promise((resolve, reject) => {
|
|
const child = spawn(process.execPath, [path.join('scripts', script)], { cwd: process.cwd(), env: { ...gitEnvironment(), ...qaEnvironment, ...(host ? { AGENT_WORKBENCH_URL: host.url } : {}) }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
let tail = ''; const capture = bytes => { stream.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-10000); };
|
|
child.stdout.on('data', capture); child.stderr.on('data', capture);
|
|
let timedOut = false;
|
|
const timer = setTimeout(() => {
|
|
timedOut = true;
|
|
if (process.platform === 'win32' && child.pid) {
|
|
try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000, stdio: 'ignore' }); } catch { child.kill(); }
|
|
} else child.kill();
|
|
}, 240000);
|
|
child.once('error', error => { clearTimeout(timer); reject(error); });
|
|
child.once('close', code => { clearTimeout(timer); code === 0 && !timedOut ? resolve(tail) : reject(new Error(script + (timedOut ? ' timed out' : ' failed (' + code + ')') + ': ' + tail)); });
|
|
});
|
|
stream.end();
|
|
const summary = output.trim().split(/\r?\n/).findLast(line => line.startsWith('{"success":'));
|
|
const detail = summary ? JSON.parse(summary) : null;
|
|
const directory = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1].trim() || detail?.directory || detail?.root;
|
|
const proof = fixedProof || (directory && path.join(directory, 'result.json'));
|
|
assert.ok(proof, 'Verifier must identify its current evidence.');
|
|
if (!fixedProof) {
|
|
const relative = path.relative(path.resolve('.qa'), path.resolve(proof));
|
|
assert.ok(relative && !relative.startsWith('..') && !path.isAbsolute(relative), 'Evidence must belong to this workspace QA directory.');
|
|
}
|
|
const evidence = JSON.parse(fs.readFileSync(proof, 'utf8'));
|
|
assert.notEqual(evidence.success, false); assert.notEqual(evidence.passed, false); if (evidence.version) assert.equal(evidence.version, version);
|
|
const passed = [evidence.passed, evidence.cases, evidence.assertions, evidence.checks, evidence.results].find(Array.isArray);
|
|
assert.ok(Array.isArray(passed) && passed.length > 0);
|
|
const errors = evidence.pageErrors ?? [], failures = evidence.requestFailures ?? [];
|
|
assert.equal(errors.length, 0); assert.equal(failures.length, 0);
|
|
assert.deepEqual(evidence.errors ?? [], []);
|
|
const saved = ['verify-controlled-ui.mjs', 'verify-purge-ui.mjs'].includes(script) ? `docs/${prefix}-${script.replace('verify-', '').replace('-ui.mjs', '')}-final-ui-validation.json`
|
|
: script === 'verify-ui.mjs' ? `docs/${prefix}-basic-ui-validation.json`
|
|
: ['verify-workspaces-ui.mjs', 'verify-task-plans-ui.mjs'].includes(script) ? proof
|
|
: `docs/${prefix}-${script.replace('verify-', '').replace('.mjs', '')}-validation.json`;
|
|
if (path.resolve(saved) !== path.resolve(proof)) fs.copyFileSync(proof, saved);
|
|
results.push({ script, success: true, groups: passed.length, durationMs: Date.now() - started, evidence: saved, log });
|
|
console.log('PASS ' + script + ': ' + passed.length + ' groups');
|
|
} catch (error) {
|
|
stream.end(); fs.writeFileSync(resultFile, JSON.stringify({ success: false, version, at: new Date().toISOString(), results, error: error.message, script, log }, null, 2) + '\n');
|
|
throw error;
|
|
} finally { await host?.close(); }
|
|
}
|
|
const result = { success: true, version, at: new Date().toISOString(), groups: results.reduce((sum, item) => sum + item.groups, 0), results,
|
|
scope: 'Every verifier owns an independent port and synthetic data directory. Synthetic provider fixtures verify protocols; configured-model acceptance is separate. No production service or real key read/copied.' };
|
|
fs.writeFileSync(resultFile, JSON.stringify(result, null, 2) + '\n');
|
|
fs.copyFileSync(resultFile, 'docs/' + prefix + '-all-ui-validation.json'); console.log(JSON.stringify(result, null, 2));
|