Files
Agent-Workbench/scripts/verify-delivery-ui.mjs

227 lines
22 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { spawn, execFileSync } from 'node:child_process';
import { chromium, expect } from 'playwright/test';
import { Store } from '../src/server/store.mjs';
import { GitDelivery } from '../src/core/git-delivery.mjs';
import { gitEnvironment } from '../src/core/git-process.mjs';
import { checkedDelivery } from '../tests/fixtures/delivery-project.mjs';
// Owns an independent host and synthetic projects. It never connects to 4318.
const workspace = process.cwd(), root = path.resolve('.qa', 'delivery-ui-' + randomUUID()), data = path.join(root, 'data');
const projectRoot = path.join(root, '四模块交付工程'), recoveryRoot = path.join(root, '提交恢复工程');
const allowed = ['src/normalize.mjs', 'src/summary.mjs', 'src/render.mjs', 'src/cli.mjs'];
const bytesHash = value => createHash('sha256').update(value).digest('hex');
fs.mkdirSync(root, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
fs.cpSync(path.join(workspace, 'benchmarks/node-delivery-v1/baseline'), projectRoot, { recursive: true });
fs.writeFileSync(path.join(projectRoot, '.gitignore'), 'dist/\n');
fs.writeFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), 'Owner baseline notes\n');
fs.writeFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Owner baseline staged file\n');
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', projectRoot, ...args], { env: gitEnvironment(), encoding: 'utf8', windowsHide: true });
const list = directory => fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => entry.isDirectory() ? list(path.join(directory, entry.name)) : [path.relative(projectRoot, path.join(directory, entry.name)).replaceAll('\\', '/')]);
const baselineFiles = list(projectRoot);
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); git(['config', 'user.name', 'Workbench UI QA']); git(['config', 'user.email', 'ui@workbench.invalid']);
git(['add', '--', ...baselineFiles]); git(['commit', '--no-gpg-sign', '-m', 'Sealed four-module UI input']);
const baselineHead = git(['rev-parse', 'HEAD']).trim();
fs.appendFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), 'Keep owner unstaged notes\n');
fs.writeFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Keep owner staged content\n'); git(['add', '--', 'OWNER-STAGED.txt']);
fs.appendFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Keep later owner working copy\n');
fs.writeFileSync(path.join(projectRoot, 'OWNER-UNTRACKED.txt'), 'Keep owner untracked file\n');
const protectedFiles = [...baselineFiles.filter(relative => !allowed.includes(relative)), 'OWNER-UNTRACKED.txt'];
const protectedHashes = Object.fromEntries(protectedFiles.map(relative => [relative, bytesHash(fs.readFileSync(path.join(projectRoot, relative)))]));
const initialHashes = Object.fromEntries(allowed.map(relative => [relative, bytesHash(fs.readFileSync(path.join(projectRoot, relative)))]));
const repairs = Object.fromEntries(allowed.map(relative => [relative, fs.readFileSync(path.join(workspace, 'examples/delivery-report', relative), 'utf8')]));
const store = new Store(data), primary = store.addProject(projectRoot);
fs.mkdirSync(recoveryRoot);
fs.writeFileSync(path.join(recoveryRoot, 'README.md'), '# Recovery UI fixture\n');
const recoveryProject = store.addProject(recoveryRoot), recoverySession = store.createSession(recoveryProject.id), seedCleanup = [];
const f = { root, projectRoot: recoveryRoot, data, store, project: recoveryProject, session: recoverySession, onCleanup: fn => seedCleanup.push(fn) };
await checkedDelivery(f);
recoverySession.title = 'Git中断核对样本'; store.save(recoverySession);
const recoveryAdapter = new GitDelivery(store, { faultInjector: phase => { if (phase === 'committed') throw new Error('Controlled lost acknowledgement after actual publication'); } });
const recoveryPlan = await recoveryAdapter.plan(recoverySession);
for (const cleanup of seedCleanup) await cleanup(); store.close();
const exited = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/delivery-crash-worker.mjs'), data, recoverySession.id, recoveryPlan.id, 'committed'],
{ windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let exitOutput = '', exitErrors = '';
exited.stdout.on('data', bytes => { exitOutput += bytes.toString('utf8'); }); exited.stderr.on('data', bytes => { exitErrors += bytes.toString('utf8'); });
assert.equal(await new Promise(resolve => exited.once('close', resolve)), 73, exitErrors);
const crashed = JSON.parse(exitOutput.trim()), recoveredStore = new Store(data), recoveryJob = structuredClone(recoveredStore.owned('delivery_job', crashed.jobId, recoverySession.id));
assert.equal(recoveryJob.status, 'unknown'); recoveredStore.close();
let modelMode = 'initial', modelStage = 0;
const model = http.createServer(async (request, response) => {
try {
const chunks = []; for await (const chunk of request) chunks.push(chunk);
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'));
const last = input.messages.findLastIndex(item => item.role === 'user');
const results = input.messages.slice(last + 1).filter(item => item.role === 'tool').map(item => JSON.parse(item.content));
// The request intentionally seals old complete tool groups when bounded.
// Consume the original fields in its visible feedback, not history length.
const feedback = input.messages.filter(item => item.role === 'system').flatMap(item => {
for (const line of item.content.split('\n')) try { const value = JSON.parse(line); if (value.data?.kind === 'latest-tool-feedback') return value.data.entries.map(entry => entry.result); } catch {}
return [];
});
const plan = [...results, ...feedback].findLast(value => value.checks && value.fingerprint);
let calls = [];
if (modelStage === 0) calls = [['inspect_project', {}]];
else if (modelStage === 1) calls = [['plan_checks', {}]];
else if (modelStage === 2) { assert.ok(plan?.id, 'The actual check plan must remain visible in raw results or original-field feedback.'); calls = [['run_checks', { plan_id: plan.id }]]; }
else if (modelStage === 3 && modelMode === 'initial') calls = allowed.map(relative => ['propose_file', { path: relative, content: repairs[relative], base_sha256: initialHashes[relative], reason: 'Controlled four-module repair after actual failure logs' }]);
else if (modelStage === 3 && modelMode === 'final') calls = [['plan_delivery', { message: 'feat: prepare verified four-module delivery' }]];
else if (modelStage === 4 && modelMode === 'final') calls = [['delivery_report', {}]];
modelStage++;
response.writeHead(200, { 'Content-Type': 'application/json' });
response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: calls.length ? '' : modelMode === 'initial' ? '四个文件已准备提案,等待审查。' : '实际14项测试和构建通过,Git预览已准备,等待确认。',
...(calls.length ? { tool_calls: calls.map(([name, args]) => ({ id: randomUUID(), type: 'function', function: { name, arguments: JSON.stringify(args) } })) } : {}) },
finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 40, completion_tokens: 20, total_tokens: 60 } }));
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
});
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
const env = { ...process.env, AGENT_WORKBENCH_APP_DIR: workspace, AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' }; delete env.AGENT_WORKBENCH_API_KEY;
const host = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: workspace, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let url = '', hostOutput = '', hostErrors = '';
host.stdout.on('data', bytes => { hostOutput += bytes.toString('utf8'); for (const line of hostOutput.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
host.stderr.on('data', bytes => { hostErrors += bytes.toString('utf8'); });
const until = async predicate => { const deadline = Date.now() + 20000; while (!(await predicate())) { if (Date.now() > deadline) throw new Error('QA host state timeout: ' + hostErrors); await new Promise(resolve => setTimeout(resolve, 20)); } };
await until(() => url || host.exitCode !== null); assert.ok(url, hostErrors); assert.notEqual(new URL(url).port, '4318');
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
const api = async (route, input) => {
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const result = await response.json(); assert.ok(response.ok, result.error); return result;
};
await api('/config', { ...bootstrap.config, mode: 'compatible', model: 'delivery-ui-controlled-fixture', baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', apiKey: 'delivery-ui-volatile-fixture-key' });
const browser = await chromium.launch({ executablePath: 'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe', headless: true });
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
const page = await context.newPage(), passed = [], pageErrors = [], requestFailures = [], confirmations = [];
page.on('pageerror', error => pageErrors.push(error.message));
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
page.on('request', request => { if (request.method() === 'POST' && /\/delivery-jobs$/.test(new URL(request.url()).pathname)) confirmations.push(request.postDataJSON().planId); });
const mark = text => { passed.push(text); console.log('PASS ' + text); };
const screenshot = name => page.screenshot({ path: '.qa/screenshots/' + name + '.png', scale: 'css' });
const task = () => page.getByLabel('任务记录', { exact: true });
const delivery = () => task().getByLabel('Git交付', { exact: true });
const confirm = () => page.getByRole('dialog', { name: '确认本地Git提交', exact: true });
const artifact = () => page.getByRole('dialog', { name: '执行结果工件', exact: true });
let sessionId = '';
const state = () => api('/sessions/' + sessionId);
const waitStatus = status => expect.poll(async () => (await state()).status, { timeout: 30000 }).toBe(status);
try {
await page.goto(url); await page.waitForLoadState('networkidle');
await page.getByLabel('当前项目').selectOption(primary.id);
await page.getByRole('button', { name: '新建任务', exact: true }).click();
await page.getByLabel('任务内容').fill('修复四模块并交付Git报告'); await page.getByRole('button', { name: '发送任务' }).click();
await expect.poll(async () => (await api('/sessions')).find(value => value.title === '修复四模块并交付Git报告')?.id).toBeTruthy();
sessionId = (await api('/sessions')).find(value => value.title === '修复四模块并交付Git报告').id;
await waitStatus('awaiting_approval'); await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitStatus('needs_review');
const reviewed = await state(); assert.equal(reviewed.execution.checks[0].results[0].tests.failed, 13);
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(task()).toContainText('13');
assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineHead);
mark('actual failing tests and four source proposals retain the original project and task');
await page.locator('.inspector-tabs').getByRole('button', { name: /改动/ }).click();
for (const relative of allowed) {
await page.locator('.change-list').getByRole('button').filter({ hasText: relative }).click();
await page.getByRole('button', { name: '应用这项改动', exact: true }).click();
}
await waitStatus('paused'); modelMode = 'final'; modelStage = 0; await page.getByRole('button', { name: '继续原任务', exact: true }).click();
await waitStatus('awaiting_approval'); await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitStatus('needs_review');
const verified = await state(); assert.equal(verified.execution.task.id, reviewed.execution.task.id); assert.equal(verified.execution.checks.at(-1).status, 'passed');
assert.equal(verified.execution.checks.at(-1).results[0].tests.passed, 14); assert.equal(verified.execution.deliveries.length, 0);
assert.equal(verified.execution.deliveryPlans.at(-1).preparedBy, 'agent');
assert.equal(verified.execution.run.stopReason, 'git_review');
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(task()).toContainText('测试 14/14 通过');
await page.locator('.session-select').filter({ hasText: '修复四模块并交付Git报告' }).click();
await expect(task()).toContainText('测试 14/14 通过');
await expect(delivery().getByLabel('等待Git确认')).toBeVisible();
await expect(page.getByRole('button', { name: '继续原任务', exact: true })).toHaveCount(0);
await delivery().getByRole('button', { name: '查看待确认预览', exact: true }).click();
await expect(confirm()).toBeVisible(); await expect(confirm()).toContainText('14/14');
await screenshot('delivery-git-review-boundary');
await confirm().getByRole('button', { name: '返回修改', exact: true }).click();
mark('reviewed changes continue the same Task through 14 real tests and model-prepared Git review');
await delivery().getByLabel('提交说明', { exact: true }).fill('feat: ship the reviewed four-module delivery');
await delivery().getByText('Git署名', { exact: true }).click(); await delivery().getByLabel('提交者姓名').fill('UI Reviewer'); await delivery().getByLabel('提交者邮箱').fill('reviewer@workbench.invalid');
await delivery().getByRole('button', { name: '预览Git提交', exact: true }).click(); await expect(confirm()).toBeVisible();
await expect(confirm()).toContainText('UI Reviewer'); await expect(confirm()).toContainText('14/14'); await expect(confirm()).toContainText(baselineHead.slice(0, 12)); await expect(confirm()).toContainText('不执行仓库hooks');
for (const relative of allowed) { await confirm().locator('summary').filter({ hasText: relative }).click(); await expect(confirm().getByLabel('提交差异 ' + relative, { exact: true })).toBeVisible(); }
await confirm().locator('.delivery-confirm').evaluate(element => { element.scrollTop = 0; });
await screenshot('delivery-desktop-review');
await page.keyboard.press('Escape'); await expect(confirm()).not.toBeVisible(); assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineHead);
mark('review shows actual paths, diffs, tests, author and policy; cancelling leaves Git unchanged');
await delivery().getByRole('button', { name: '预览Git提交', exact: true }).click(); await expect(confirm()).toBeVisible();
git(['add', '--', 'OWNER-NOTES.md']); const protectedStage = git(['show', ':OWNER-STAGED.txt']), stagedNotes = git(['show', ':OWNER-NOTES.md']);
await confirm().getByRole('button', { name: '确认创建本地提交', exact: true }).click();
await expect.poll(async () => (await state()).execution.deliveries.at(-1)?.status, { timeout: 20000 }).toBe('stale');
await expect(delivery()).toContainText('提交预览后'); assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineHead);
assert.equal(git(['show', ':OWNER-STAGED.txt']), protectedStage); assert.equal(git(['show', ':OWNER-NOTES.md']), stagedNotes);
mark('an index change after confirmation preview refuses the stale job and preserves both staged owners');
await delivery().getByRole('button', { name: '预览Git提交', exact: true }).click(); await expect(confirm()).toBeVisible();
await page.setViewportSize({ width: 390, height: 844 }); await expect(confirm()).toBeVisible();
await confirm().locator('summary').filter({ hasText: 'src/normalize.mjs' }).click();
await screenshot('delivery-mobile-review');
const box = await confirm().boundingBox(), confirmButton = await confirm().getByRole('button', { name: '确认创建本地提交', exact: true }).boundingBox();
assert.ok(box.x >= 0 && box.x + box.width <= 391 && box.y >= 0 && box.y + box.height <= 845);
assert.ok(confirmButton.x >= 0 && confirmButton.x + confirmButton.width <= 391);
assert.ok(await confirm().evaluate(element => element.scrollWidth <= element.clientWidth + 1));
mark('390px confirmation keeps file hashes, diffs and actions inside the modal');
await page.setViewportSize({ width: 1440, height: 1000 });
await confirm().getByRole('button', { name: '确认创建本地提交', exact: true }).dblclick();
await expect.poll(async () => (await state()).execution.deliveries.at(-1)?.status, { timeout: 25000 }).toBe('completed');
const delivered = await state(), job = delivered.execution.deliveries.at(-1);
assert.equal(confirmations.filter(value => value === job.planId).length, 1);
assert.equal(job.workspacePreserved, true); assert.equal(job.indexPreserved, true); assert.equal(git(['rev-list', '--count', 'HEAD']).trim(), '2');
assert.equal(git(['show', ':OWNER-STAGED.txt']), protectedStage); assert.equal(git(['show', ':OWNER-NOTES.md']), stagedNotes);
for (const [relative, expected] of Object.entries(protectedHashes)) assert.equal(bytesHash(fs.readFileSync(path.join(projectRoot, relative))), expected);
assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split('\n').sort(), allowed.slice().sort());
await expect(delivery()).toContainText(job.commit); await expect(delivery()).toContainText('实际提交已核对'); await screenshot('delivery-desktop-result');
mark('double confirmation creates one four-file commit, retains owner bytes and publishes report with completion');
const jobCard = delivery().locator('.delivery-job').filter({ hasText: job.commit });
await jobCard.getByRole('button', { name: '查看交付报告', exact: true }).click(); await expect(artifact()).toContainText('检查通过并已提交'); await expect(artifact()).toContainText('14/14');
await screenshot('delivery-desktop-report'); await artifact().getByRole('button', { name: '关闭窗口' }).click();
for (const [label, id, extension] of [['下载Markdown', job.reportMarkdownArtifactId, '.md'], ['下载JSON', job.reportArtifactId, '.json']]) {
const event = page.waitForEvent('download'); await jobCard.getByRole('button', { name: label, exact: true }).click(); const download = await event;
const target = path.join(root, 'download' + extension); await download.saveAs(target);
const meta = await api('/artifacts/' + id + '?session=' + sessionId); assert.equal(bytesHash(fs.readFileSync(target)), meta.sha256);
if (extension === '.json') assert.equal(JSON.parse(fs.readFileSync(target, 'utf8')).status, 'verified_committed');
}
mark('report viewing and Markdown/JSON downloads preserve verified artifact bytes and truthful acceptance limits');
await page.setViewportSize({ width: 390, height: 844 }); await page.getByRole('button', { name: '查看任务记录' }).click();
await delivery().getByRole('button', { name: '查看最新报告', exact: true }).click(); await expect(artifact()).toBeVisible(); await screenshot('delivery-mobile-report');
await artifact().getByRole('button', { name: '关闭窗口' }).click(); await page.getByRole('button', { name: '关闭检视区' }).click();
const composer = await page.getByLabel('任务内容').boundingBox(); assert.ok(composer.y >= 0 && composer.y + composer.height <= 844);
assert.ok(await page.locator('body').evaluate(element => element.scrollWidth <= 390)); mark('mobile report preview closes cleanly and keeps the task input fixed');
await page.setViewportSize({ width: 1440, height: 1000 });
if (!await page.getByLabel('当前项目').isVisible()) await page.getByRole('button', { name: '展开项目栏' }).click();
await page.getByLabel('当前项目').selectOption(recoveryProject.id); await page.locator('.session-select').filter({ hasText: 'Git中断核对样本' }).click();
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(delivery()).toContainText('结果待核对');
await expect(delivery().getByRole('button', { name: '预览Git提交', exact: true })).toBeDisabled(); await screenshot('delivery-unknown-result');
await delivery().getByRole('button', { name: '核对提交结果', exact: true }).click();
await expect.poll(async () => (await api('/sessions/' + recoverySession.id)).execution.deliveries[0].status, { timeout: 15000 }).toBe('completed');
const recovered = await api('/sessions/' + recoverySession.id);
assert.equal(recovered.execution.deliveries[0].commit, recoveryJob.expectedCommit); assert.equal(fs.existsSync(path.join(recoveryRoot, '.git/index.lock')), false);
await expect(delivery()).toContainText('没有重新执行提交'); await screenshot('delivery-reconciled-result');
mark('unknown Git results block new submission and the explicit reconcile action restores the existing commit');
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), browser: browser.version(), passed, pageErrors, requestFailures,
projectRoot, projectId: primary.id, sessionId, taskId: reviewed.execution.task.id, commit: job.commit, recoveryCommit: recoveryJob.expectedCommit,
testCounts: { baselineFailed: 13, passed: 14 }, model: 'controlled local HTTP fixture', confirmations: confirmations.length };
fs.writeFileSync('.qa/delivery-ui-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
} catch (error) {
await screenshot('delivery-ui-failure').catch(() => {}); fs.writeFileSync('.qa/delivery-ui-failure-dom.html', await page.content().catch(() => ''));
fs.writeFileSync('.qa/delivery-ui-validation.json', JSON.stringify({ success: false, error: error.message, passed, pageErrors, requestFailures, root, url }, null, 2));
throw error;
} finally {
await browser.close(); await api('/shutdown', {}).catch(() => {}); await until(() => host.exitCode !== null).catch(() => host.kill());
model.closeAllConnections(); await new Promise(resolve => model.close(resolve));
}