Files
Agent-Workbench/scripts/verify-request-observation-ui.mjs

81 lines
7.3 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import net from 'node:net';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { chromium, _electron } from 'playwright';
import { fixture, launchWorkbench, until, call, completion, compatibleFixture } from '../tests/helpers.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { HostController } from '../src/core/desktop/host-controller.mjs';
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
import { projectEnvironment } from '../src/core/process-environment.mjs';
assert.ok(process.argv.slice(2).every(value => value === '--desktop'));
const desktop = process.argv.includes('--desktop'), appRoot = path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT || '.');
const directory = path.resolve('.qa/tool-prefix-c', (desktop ? 'electron-' : 'web-') + randomUUID());
fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory);
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true }), provider = new Provider();
let round = 0;
const model = await compatibleFixture(t, () => ({ ...completion(round++ ? [] : [call('owned-ui-list', 'list_files', { path: '' })], '已核对原事实。'), usage: { prompt_tokens: 1000, completion_tokens: 12 } }));
const hub = provider.modelHub(f.store), connection = hub.upsert({ name: '本机界面夹具', model: 'owned-ui', baseUrl: model.baseUrl, requiresKey: false });
hub.select({ type: 'connection', id: connection.id });
const runtime = new Runtime(f.store, provider);
const runs = [];
for (const content of ['读取项目并回答,不修改文件,不运行工程检查。', '前面的错误是怎么回事?', '继续']) {
runtime.start(f.session.id, content); runs.push(f.session.runId); await runtime.active.get(f.session.id).promise;
}
f.store.close();
const marks = [], screenshots = [], errors = [], launches = [];
let host, controller, browser, app, page, window, result;
const panel = () => page.getByLabel('请求上下文', { exact: true });
const contains = async text => until(async () => (await panel().innerText()).includes(text));
const mark = value => { marks.push(value); console.log('PASS ' + value); };
async function shot(name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, fullPage: true }); screenshots.push(file); }
async function resize(width) {
if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height: 1000 }); await until(() => page.evaluate(w => innerWidth === w, width)); }
else await page.setViewportSize({ width, height: 1000 });
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), 'Unexpected horizontal overflow.');
}
async function openRecord() { await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await panel().waitFor({ state: 'visible' }); }
try {
if (desktop) {
const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve)); assert.notEqual(port, 4318);
controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start();
const executablePath = desktopExecutableIdentity(appRoot).path;
app = await _electron.launch({ executablePath, args: [...(executablePath.toLowerCase().endsWith('agentworkbench.exe') ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]),
'--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port)], env: projectEnvironment({ AGENT_WORKBENCH_QA: '1', AGENT_WORKBENCH_NODE: process.execPath }) });
page = await app.firstWindow(); window = await app.browserWindow(page);
} else {
host = await launchWorkbench({ after: fn => disposers.push(fn) }, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318');
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage(); await page.goto(host.url);
}
page.on('pageerror', error => errors.push(error.message));
page.on('request', request => { if (/\/sessions\/[^/]+\/messages$/.test(request.url())) launches.push(request.url()); });
await page.waitForLoadState('networkidle'); await resize(1440); await openRecord();
await contains('主任务'); await contains('用户消息实例'); await contains('首次变化位置'); await contains('上游未报告');
await panel().locator('summary').click();
await until(async () => await panel().getByLabel('本轮工具可见性').isVisible());
const toolText = await panel().getByLabel('本轮工具可见性').innerText();
assert.ok(toolText.includes('list_files') && toolText.includes('本轮可见') && toolText.includes('缺少当前任务资源') && toolText.includes('schema'));
await shot('main-tools-1440'); mark('Main request explains tool visibility, missing resources, generation, changed message and unknown cache.');
await page.getByLabel('选择运行记录', { exact: true }).selectOption(runs[1]); await contains('独立诊断');
await shot('diagnostic-1440'); mark('Historical diagnostic is identified separately from the main path.');
await page.getByLabel('选择运行记录', { exact: true }).selectOption(runs[0]);
await page.getByLabel('选择模型请求', { exact: true }).selectOption({ index: 0 });
await contains('首次请求或新路线');
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click();
const artifact = page.getByRole('dialog', { name: '执行结果工件', exact: true }); await artifact.waitFor({ state: 'visible' });
const content = await artifact.locator('pre').innerText(); assert.doesNotMatch(content, /requestObservation|schemaSha256|generation/);
await page.keyboard.press('Escape'); mark('Historical request selector opens original wire without injecting observations or executing tools.');
await page.keyboard.press('Escape'); await page.reload(); await page.waitForLoadState('networkidle'); await openRecord();
await contains('主任务'); await contains('用户消息实例'); await resize(390); await openRecord();
await panel().locator('summary').click(); await panel().locator('summary').scrollIntoViewIfNeeded();
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1));
await shot('main-tools-390'); mark('Reload retains records; 390px expanded tool details fit the existing task panel.');
assert.deepEqual(errors, []); assert.equal(launches.length, 0);
result = { success: true, desktop, appRoot, marks, screenshots, errors, fixtureModelRequests: model.requests.length, uiModelRequests: 0, realModelRequests: 0 };
} catch (error) { if (page) await shot('failure').catch(() => {}); result = { success: false, desktop, error: error.stack, marks, screenshots, errors }; }
finally { await app?.close(); await browser?.close(); await controller?.stop(); for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); }
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;