Files
Agent-Workbench/scripts/verify-turn-progress-ui.mjs

159 lines
14 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import net from 'node:net';
import assert from 'node:assert/strict';
import { randomUUID, createHash } from 'node:crypto';
import { chromium, _electron } from 'playwright';
import { fixture, compatibleFixture, launchWorkbench, until, call, completion } from '../tests/helpers.mjs';
import { HostController } from '../src/core/desktop/host-controller.mjs';
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
import { projectEnvironment } from '../src/core/process-environment.mjs';
// Existing UI-verifier convention: a real browser/Electron, isolated data and a
// localhost model protocol fixture. No configured provider or real key is used.
const desktop = process.argv.includes('--desktop');
const appRoot = path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT || '.');
const directory = path.resolve('.qa/requirements-progress', (desktop ? 'shadow-electron-' : 'shadow-ui-') + randomUUID());
fs.mkdirSync(directory, { recursive: true });
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t);
let scenario = 'pages', requests = 0;
const fixtureRequests = [];
const model = await compatibleFixture(t, () => {
const request = ++requests;
fixtureRequests.push({ scenario, request, receivedAt: new Date().toISOString() });
if (scenario === 'network') return { status: 502, body: { error: { message: 'Owned shadow gateway failure, original response retained' } } };
if (scenario === 'approval') return { body: request === 1 ? completion([
call('read-before-approval', 'read_file', { path: 'README.md', limit: 10 }),
call('owned-command', 'run_command', { command: 'node --version', reason: '所属只读环境检查,需原审批' }),
]) : completion([], '原审批已拒绝,说明与原工具结果保留。') };
return { body: completion([call('page-' + request, 'read_file', { path: 'README.md', offset: request === 1 ? 0 : 10, limit: 10 })]) };
});
f.session.title = '单轮观察分页原任务'; f.store.save(f.session);
const networkSession = f.store.createSession(f.project.id); networkSession.title = '单轮观察原网络失败'; f.store.save(networkSession);
f.store.saveConfig({ mode: 'compatible', model: 'owned-shadow-ui', baseUrl: model.baseUrl, maxSteps: 3 }); f.store.close();
const passed = [], screenshots = [], errors = [], snapshots = [], sourceArtifacts = new Map(); let browser, application, controller, host, page, window, result;
const mark = name => { passed.push(name); console.log('PASS ' + name); };
const panel = () => page.getByLabel('单轮观察', { exact: true });
async function visiblePanel() {
if (!await page.getByLabel('选择运行记录', { exact: true }).isVisible()) await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
await panel().waitFor();
if (!await panel().evaluate(element => element.open)) await panel().locator('summary').click();
}
async function select(session) {
if (!await page.getByLabel('当前项目').isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
await page.getByLabel('当前项目').selectOption(f.project.id);
await page.locator('.session-list button').filter({ hasText: session.title }).click();
}
async function resize(width, height) {
if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); }
else await page.setViewportSize({ width, height });
}
async function shot(name) {
await panel().scrollIntoViewIfNeeded(); const file = path.join(directory, name + '.png');
await page.screenshot({ path: file, scale: 'css' }); screenshots.push(file);
}
async function state(session = f.session) { return host.api('/sessions/' + session.id); }
async function originalArtifact(artifact, session) {
let content = '', offset = 0, page;
do { page = await host.api('/artifacts/' + artifact.id + '?session=' + session.id + '&offset=' + offset + '&limit=16000'); content += page.content; offset += page.content.length; }
while (page.truncated && page.content.length);
const bytes = Buffer.from(content); assert.equal(createHash('sha256').update(bytes).digest('hex'), artifact.sha256);
assert.equal(bytes.length, artifact.bytes); return content;
}
async function saved(name, session = f.session) {
const snapshot = await host.api('/sessions/' + session.id + '/export');
for (const artifact of snapshot.execution.artifacts.filter(row => ['turn-progress-shadow', 'tool-result', 'model-http-response', 'model-stream-response', 'delivery-report', 'delivery-report-markdown'].includes(row.kind))) {
if (sourceArtifacts.has(artifact.id)) continue;
const content = await originalArtifact(artifact, session), relative = 'artifacts/' + artifact.id + '.txt';
fs.mkdirSync(path.join(directory, 'artifacts'), { recursive: true }); fs.writeFileSync(path.join(directory, relative), content);
sourceArtifacts.set(artifact.id, { id: artifact.id, kind: artifact.kind, sha256: artifact.sha256, bytes: artifact.bytes, path: relative });
}
fs.writeFileSync(path.join(directory, name + '.json'), JSON.stringify(snapshot, null, 2)); snapshots.push(name + '.json'); return snapshot;
}
try {
if (desktop) {
const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve));
const port = socket.address().port; await new Promise(resolve => socket.close(resolve)); assert.notEqual(port, 4318);
controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 });
await controller.start(); host = { url: controller.url, bootstrap: await controller.api('/bootstrap'),
api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) };
const executable = desktopExecutableIdentity(appRoot).path;
application = await _electron.launch({ executablePath: executable,
args: [...(executable.toLowerCase().endsWith('agentworkbench.exe') ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]), '--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window'],
cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
page = await application.firstWindow(); window = await application.browserWindow(page);
await window.evaluate(win => { win.showInactive(); win.setContentSize(1440, 1080); }); await until(() => page.evaluate(() => innerWidth === 1440));
assert.equal(await page.evaluate(() => typeof globalThis.process), 'undefined');
} else {
host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318');
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ viewport: { width: 1440, height: 1080 } }); await page.goto(host.url);
}
assert.equal(host.bootstrap.config.hasKey, false); page.on('pageerror', error => errors.push(error.message));
await page.locator('.connection.connected').waitFor(); await select(f.session);
await host.api('/sessions/' + f.session.id + '/messages', { content: '只研究README.md原说明,不修改文件;本机固定分页观察夹具。' });
await until(async () => (await state()).status === 'paused', 25000);
const first = await state(), firstRun = first.execution.run;
assert.equal(requests, 3); assert.equal(firstRun.stopReason, 'step_limit'); await visiblePanel();
await until(async () => (await panel().textContent()).includes('重复读取候选'));
assert.match(await panel().textContent(), /第 1 轮 · 新增外显证据/); assert.match(await panel().textContent(), /第 3 轮 · 重复读取候选/);
assert.equal(firstRun.progressShadow.turns[1].delta.read_characters, 10); assert.equal(firstRun.progressShadow.turns[2].delta.read_characters, 10);
await shot('desktop-pages'); await saved('pages-original'); mark('actual runtime pages grow then repeat under the original three-round limit');
await panel().locator('li').nth(1).getByRole('button', { name: '查看原观察收据', exact: true }).click();
const dialog = page.getByRole('dialog', { name: '执行结果工件', exact: true }); await dialog.waitFor();
assert.match(await dialog.textContent(), /evidence_growth/); assert.match(await dialog.textContent(), /result_artifact_id/);
await dialog.getByRole('button', { name: '关闭窗口', exact: true }).click(); mark('original observation receipt opens with the actual tool-result provenance');
await page.reload(); await page.locator('.connection.connected').waitFor(); await visiblePanel();
assert.equal(await page.getByLabel('选择运行记录').inputValue(), firstRun.id); assert.match(await panel().textContent(), /重复读取候选/);
mark('refresh restores the same Task, Run and persisted shadow observations');
await resize(390, 844); await visiblePanel();
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), true);
assert.ok((await panel().getByRole('button', { name: '查看原观察收据', exact: true }).first().boundingBox()).height >= 44);
await shot('narrow-pages'); mark('390px keeps observation facts and original receipt actions reachable without page overflow');
await resize(1440, 1080); scenario = 'approval'; requests = 0;
await host.api('/sessions/' + f.session.id + '/resume', { content: '继续核对所属环境;命令只经原审批执行。' });
await until(async () => (await state()).status === 'awaiting_approval', 25000); await visiblePanel();
await until(async () => (await panel().textContent()).includes('等待所属操作')); await shot('waiting-approval');
const waiting = await state(); assert.equal(waiting.execution.run.progressShadow.turns.at(-1).classification, 'waiting');
await saved('approval-waiting'); mark('an actual pending approval stays waiting and does not consume the earlier same-turn read twice');
await page.getByRole('button', { name: '拒绝执行', exact: true }).click();
await until(async () => (await state()).status === 'completed', 25000); const second = await state();
assert.equal(requests, 2); assert.equal(second.taskId, first.taskId); assert.notEqual(second.execution.run.id, firstRun.id);
assert.equal(second.execution.run.progressShadow.turns.find(row => row.phase === 'settled').delta.repeated_reads, 0);
await until(async () => (await panel().textContent()).includes('新失败 1')); await shot('denied-approval');
assert.match(await panel().textContent(), /失败类别:权限或审批拒绝/);
await saved('approval-denied'); mark('refusing the original approval preserves the permission failure and the original tool pairs');
await page.getByLabel('选择运行记录').selectOption(firstRun.id);
await until(async () => (await panel().textContent()).includes('第 3 轮 · 重复读取候选'));
await shot('historical-run'); await page.getByLabel('选择运行记录').selectOption(second.execution.run.id);
await until(async () => !(await panel().textContent()).includes('第 3 轮')); mark('selecting historical and current Runs keeps their independent observations');
await host.api('/sessions/' + f.session.id + '/delivery-reports', {});
const reportExport = await saved('report-original');
const reportArtifact = reportExport.execution.artifacts.filter(row => row.kind === 'delivery-report').at(-1);
const report = JSON.parse(await originalArtifact(reportArtifact, f.session));
assert.equal(report.turnObservations.length, 2); assert.ok(report.turnObservations.every(row => row.evidence_verified));
fs.writeFileSync(path.join(directory, 'report.json'), JSON.stringify(report, null, 2)); mark('delivery report uses both Runs and independently verifies the original observation receipts');
scenario = 'network'; requests = 0; await select(networkSession);
await host.api('/sessions/' + networkSession.id + '/messages', { content: '读取原工程,保存失败来源;本机502夹具。' });
await until(async () => (await state(networkSession)).status === 'failed', 25000); await visiblePanel();
await until(async () => (await panel().textContent()).includes('新增失败证据'));
assert.match(await panel().textContent(), /失败类别:上游服务/);
const failure = await state(networkSession); assert.equal(failure.lastFailure.httpStatus, 502); assert.equal(failure.execution.run.stopReason, null);
await shot('network-original-failure'); await saved('network-original', networkSession);
mark('original HTTP 502 remains a model failure and is not relabeled as no progress'); assert.deepEqual(errors, []);
result = { success: true, version: host.bootstrap.version, appRoot, passed, errors, screenshots, snapshots, actualElectron: desktop, installationValidated: false, realModelRequests: 0, officialServiceTouched: false };
} catch (error) {
try { await saved('failure-session'); } catch (captureError) {
fs.writeFileSync(path.join(directory, 'failure-session-error.txt'), captureError.stack);
}
if (page) { try { await page.screenshot({ path: path.join(directory, 'failure.png') }); fs.writeFileSync(path.join(directory, 'failure-dom.txt'), await page.locator('body').innerText()); } catch {} }
result = { success: false, passed, errors, screenshots, snapshots, error: error.stack, actualElectron: desktop, realModelRequests: 0, officialServiceTouched: false };
} finally {
fs.writeFileSync(path.join(directory, 'fixture-requests.json'), JSON.stringify(fixtureRequests, null, 2));
if (application) await application.close(); if (browser) await browser.close();
if (controller && await controller.health()) await controller.stop();
for (const dispose of disposers.reverse()) await dispose();
fs.writeFileSync(path.join(directory, 'source-artifacts.json'), JSON.stringify([...sourceArtifacts.values()], null, 2));
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
}
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;