import { releaseEvidencePrefix } from './qa-evidence.mjs'; import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { spawn, execFileSync } from 'node:child_process'; import { gitEnvironment } from '../src/core/git-process.mjs'; import { randomUUID } from 'node:crypto'; if (process.argv.slice(2).some(value => value !== '--help')) throw new Error('Unsupported argument; use --help.'); if (process.argv.includes('--help')) { console.log('node scripts/verify-all-ui.mjs - run serial desktop/390px checks in independent local hosts; use explicit QA Python/Harness/Purge source paths for 1.15.'); process.exit(0); } const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version; assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version), 'Review evidence paths when introducing the next release.'); const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = []; const qaEnvironment = Object.fromEntries(['AGENT_WORKBENCH_TEST_PYTHON', 'AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_178_ROOT', 'AGENT_WORKBENCH_PURGE_181_ROOT'] .filter(name => process.env[name]).map(name => [name, process.env[name]])); for (const [name, value] of Object.entries(qaEnvironment)) assert.ok(path.isAbsolute(value) && fs.existsSync(value), 'Use an existing absolute QA runtime/source path: ' + name); const cases = [ ...(version === '1.19.0' ? [['verify-project-cache-ui.mjs', null]] : []), ...(['1.18.0', '1.19.0'].includes(version) ? [['verify-vision-ui.mjs', null]] : []), ...(['1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-model-protocols-ui.mjs', null]] : []), ...(['1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-companion-ui.mjs', null]] : []), ...(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-controlled-ui.mjs', null], ['verify-purge-ui.mjs', null]] : []), ...(['1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-workers-ui.mjs', null]] : []), ...(['1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []), ...(['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []), ['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'], ['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'], ['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`], ['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`], ['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`], ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []), ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []), ...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []), ...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []), ...(['1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []), ]; fs.mkdirSync('.qa', { recursive: true }); const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']); async function startHost(script) { const data = path.resolve('.qa', 'ui-host-' + script.replace('.mjs', '') + '-' + randomUUID()); fs.mkdirSync(data, { recursive: true }); const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env: { ...gitEnvironment(), AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); let url = '', output = '', errors = ''; child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } }); child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); }); const deadline = Date.now() + 10000; while (!url && child.exitCode === null && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 30)); if (!url) { child.kill(); throw new Error('Independent UI host failed: ' + errors); } const bootstrap = await (await fetch(url + '/api/bootstrap')).json(); assert.equal(bootstrap.version, version); assert.equal(bootstrap.config.hasKey, false); assert.equal(bootstrap.config.mode, 'demo'); return { url, data, async close() { try { await fetch(url + '/api/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, body: '{}' }); const stopDeadline = Date.now() + 10000; while (child.exitCode === null && child.signalCode === null && Date.now() < stopDeadline) await new Promise(resolve => setTimeout(resolve, 30)); } finally { if (child.exitCode === null && child.signalCode === null) child.kill(); } } }; } for (const [script, fixedProof] of cases) { console.log('UI regression: ' + script); const started = Date.now(), log = `.qa/${prefix}-${script.replace('.mjs', '')}.log`; const stream = fs.createWriteStream(log); let host; try { if (externalHost.has(script)) host = await startHost(script); const output = await new Promise((resolve, reject) => { const child = spawn(process.execPath, [path.join('scripts', script)], { cwd: process.cwd(), env: { ...gitEnvironment(), ...qaEnvironment, ...(host ? { AGENT_WORKBENCH_URL: host.url } : {}) }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); let tail = ''; const capture = bytes => { stream.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-10000); }; child.stdout.on('data', capture); child.stderr.on('data', capture); let timedOut = false; const timer = setTimeout(() => { timedOut = true; if (process.platform === 'win32' && child.pid) { try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000, stdio: 'ignore' }); } catch { child.kill(); } } else child.kill(); }, 240000); child.once('error', error => { clearTimeout(timer); reject(error); }); child.once('close', code => { clearTimeout(timer); code === 0 && !timedOut ? resolve(tail) : reject(new Error(script + (timedOut ? ' timed out' : ' failed (' + code + ')') + ': ' + tail)); }); }); stream.end(); const directory = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1].trim(); const proof = fixedProof || (directory && path.join(directory, 'result.json')); assert.ok(proof, 'Verifier must identify its current evidence.'); if (!fixedProof) { const relative = path.relative(path.resolve('.qa'), path.resolve(proof)); assert.ok(relative && !relative.startsWith('..') && !path.isAbsolute(relative), 'Evidence must belong to this workspace QA directory.'); } const evidence = JSON.parse(fs.readFileSync(proof, 'utf8')); assert.notEqual(evidence.success, false); assert.notEqual(evidence.passed, false); if (evidence.version) assert.equal(evidence.version, version); const passed = [evidence.passed, evidence.cases, evidence.assertions].find(Array.isArray); assert.ok(Array.isArray(passed) && passed.length > 0); const errors = evidence.pageErrors ?? [], failures = evidence.requestFailures ?? []; assert.equal(errors.length, 0); assert.equal(failures.length, 0); const saved = ['verify-controlled-ui.mjs', 'verify-purge-ui.mjs'].includes(script) ? `docs/${prefix}-${script.replace('verify-', '').replace('-ui.mjs', '')}-final-ui-validation.json` : script === 'verify-ui.mjs' ? `docs/${prefix}-basic-ui-validation.json` : ['verify-workspaces-ui.mjs', 'verify-task-plans-ui.mjs'].includes(script) ? proof : `docs/${prefix}-${script.replace('verify-', '').replace('.mjs', '')}-validation.json`; if (path.resolve(saved) !== path.resolve(proof)) fs.copyFileSync(proof, saved); results.push({ script, success: true, groups: passed.length, durationMs: Date.now() - started, evidence: saved, log }); console.log('PASS ' + script + ': ' + passed.length + ' groups'); } catch (error) { stream.end(); fs.writeFileSync(resultFile, JSON.stringify({ success: false, version, at: new Date().toISOString(), results, error: error.message, script, log }, null, 2) + '\n'); throw error; } finally { await host?.close(); } } const result = { success: true, version, at: new Date().toISOString(), groups: results.reduce((sum, item) => sum + item.groups, 0), results, scope: 'Every verifier owns an independent port and synthetic data directory. Synthetic provider fixtures verify protocols; configured-model acceptance is separate. No production service or real key read/copied.' }; fs.writeFileSync(resultFile, JSON.stringify(result, null, 2) + '\n'); fs.copyFileSync(resultFile, 'docs/' + prefix + '-all-ui-validation.json'); console.log(JSON.stringify(result, null, 2));