Files
Agent-Workbench/scripts/verify-diagnostics-ui.mjs
T

186 lines
17 KiB
JavaScript

import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import { randomUUID } from 'node:crypto';
import { chromium } from 'playwright';
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4319';
const root = fs.mkdtempSync(path.resolve('.qa', 'diagnostics-ui-'));
const projectRoot = path.join(root, 'project'); fs.mkdirSync(projectRoot);
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Diagnostics UI fixture\n');
fs.mkdirSync('.qa/screenshots', { recursive: true });
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
assert.equal(bootstrap.config.mode, 'demo', 'Diagnostics UI validation requires an independent demo service.');
assert.equal(bootstrap.config.hasKey, false, 'Diagnostics UI validation must not replace a configured model connection.');
const api = async (route, input) => {
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const value = await response.json(); assert.ok(response.ok, value.error); return value;
};
let mode = 'failure', requestCount = 0, sessionId = '', page, diagnosticReads = 0, diagnosticProjectionUsed = false;
const passed = [], pageErrors = [], networkErrors = [];
const model = http.createServer(async (request, response) => {
try {
const chunks = []; for await (const chunk of request) chunks.push(chunk);
if (request.url === '/v1/models') { response.writeHead(200, { 'Content-Type': 'application/json' }); return response.end(JSON.stringify({ data: [{ id: 'ui-diagnostics-fixture' }] })); }
const input = JSON.parse(Buffer.concat(chunks).toString('utf8')); requestCount++;
assert.equal(request.headers.authorization, 'Bearer ui-fixture-memory-key');
const answer = content => { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content }, finish_reason: 'stop' }] })); };
const tool = (name, args) => { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: '', tool_calls: [{ id: 'ui-' + randomUUID(), type: 'function', function: { name, arguments: JSON.stringify(args) } }] }, finish_reason: 'tool_calls' }] })); };
const failure = (status, headers = {}) => { response.writeHead(status, { 'Content-Type': 'application/json', ...headers }); response.end(JSON.stringify({ error: { message: 'Controlled model-service failure' } })); };
if (!input.tools) return answer('OK');
const lastUser = input.messages.findLastIndex(message => message.role === 'user');
const results = input.messages.slice(lastUser + 1).filter(message => message.role === 'tool');
if (mode === 'failure') return results.length ? failure(503) : tool('read_file', { path: 'README.md' });
if (mode === 'recover') { assert.equal(input.messages.at(-1).role, 'tool'); return answer('**模型请求已恢复**,已完成的读取结果保留。'); }
if (mode === 'question') {
assert.match(input.messages[0].content, /model_connection/); assert.match(input.messages[0].content, /503/);
// Large owned snapshots may be projected into mandatory latest-tool
// feedback. The fixture must consume that saved result like the model,
// rather than replaying an inspection whose raw group was archived.
const feedback = input.messages.filter(message => message.role === 'system').map(message => {
try { return JSON.parse(message.content.split('\n').at(-1)); } catch { return {}; }
}).find(value => value.latest_tools)?.data.entries.find(entry => entry.tool === 'inspect_session');
const snapshot = results.length ? JSON.parse(results.at(-1).content) : feedback?.result;
if (!snapshot) { diagnosticReads++; return tool('inspect_session', {}); }
assert.equal(snapshot.session.id, sessionId); assert.equal(snapshot.lastFailure.source, 'model_connection');
if (feedback) { diagnosticProjectionUsed = true; assert.equal(feedback.recorded_state, 'completed'); assert.match(snapshot.lastFailure.message, /HTTP 503/); }
else assert.equal(snapshot.lastFailure.httpStatus, 503);
return answer('已读取工作台执行记录。上次错误发生在请求模型服务时:**HTTP 503**。项目读取已经成功,文件未改动。');
}
if (mode === 'proposal') {
if (!results.length) return tool('propose_file', { path: 'REVIEWED.md', content: '# Reviewed through the UI\n', reason: 'UI review fixture' });
await new Promise(resolve => setTimeout(resolve, 700)); return answer('改动已准备好,请查看差异后应用。');
}
if (mode === 'command') {
if (!results.length) return tool('run_command', { command: "@'\nconsole.log('中文输入与输出正确');\n'@ | node --input-type=module\nexit $LASTEXITCODE", reason: '验证中文通过项目终端进入 Node。' });
const result = JSON.parse(results.at(-1).content); assert.equal(result.status, 'completed'); assert.match(result.stdout, /中文输入与输出正确/);
return answer('**UTF-8 验证通过**:中文输入与输出正确。');
}
if (mode === 'authentication') return failure(401);
if (mode === 'backoff') return failure(503, { 'Retry-After': '3' });
if (mode === 'transient') return requestCount === 1 ? failure(503) : answer('临时连接错误已自动恢复。');
throw new Error('Unknown fixture mode');
} catch (error) { if (!response.headersSent) response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
});
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
const browser = await chromium.launch({ executablePath: path.join(process.env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Microsoft', 'Edge', 'Application', 'msedge.exe'), headless: true });
const mark = text => { passed.push(text); console.log('PASS ' + text); };
const state = async () => api('/sessions/' + sessionId);
const waitState = async status => {
await page.waitForFunction(expected => document.querySelector('.task-toolbar .status')?.classList.contains('status-' + expected), status);
};
const checkViewport = async () => {
const bounds = await page.evaluate(() => {
const header = document.querySelector('.work-header').getBoundingClientRect(), composer = document.querySelector('.composer-wrap').getBoundingClientRect();
return { width: innerWidth, height: innerHeight, scrollWidth: document.documentElement.scrollWidth, scrollHeight: document.documentElement.scrollHeight, headerTop: header.top, composerBottom: composer.bottom };
});
assert.ok(bounds.scrollWidth <= bounds.width + 1 && bounds.scrollHeight <= bounds.height + 1, JSON.stringify(bounds));
assert.ok(bounds.headerTop >= -1 && bounds.composerBottom <= bounds.height + 1, JSON.stringify(bounds));
};
const screenshot = async name => { await checkViewport(); await page.screenshot({ path: '.qa/screenshots/' + name + '.png', scale: 'css' }); };
const task = async content => {
const newTask = page.getByRole('button', { name: '新建任务', exact: true });
if (!await newTask.isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
await newTask.click();
await page.getByLabel('任务内容').fill(content); await page.getByRole('button', { name: '发送任务', exact: true }).click();
await page.waitForFunction(() => !!localStorage.getItem('awb-session')); sessionId = await page.evaluate(() => localStorage.getItem('awb-session'));
};
const settings = async () => {
await page.getByRole('button', { name: '选择与管理模型', exact: true }).click();
await page.getByRole('dialog', { name: '模型中心', exact: true }).locator('.mc-connection').filter({ has: page.getByRole('heading', { name: '原模型连接', exact: true }) }).getByRole('button', { name: '编辑', exact: true }).click();
return page.getByRole('dialog', { name: '原连接与演示设置', exact: true });
};
try {
page = await browser.newPage({ viewport: { width: 1440, height: 960 }, deviceScaleFactor: 1 }); page.setDefaultTimeout(15000);
page.on('pageerror', error => pageErrors.push(error.message));
page.on('requestfailed', request => { if (!request.url().includes('/api/events')) networkErrors.push(request.failure()?.errorText); });
await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.locator('.connection.connected').waitFor();
await page.getByRole('button', { name: '打开项目', exact: true }).click();
const projectDialog = page.getByRole('dialog', { name: '打开项目' });
await projectDialog.getByLabel('项目文件夹路径').fill(projectRoot); await projectDialog.getByRole('button', { name: '打开项目', exact: true }).click();
await projectDialog.waitFor({ state: 'hidden' });
const connection = await settings(); await connection.getByRole('button', { name: '兼容模型服务' }).click();
await connection.getByLabel('接口根地址', { exact: true }).fill('http://127.0.0.1:' + model.address().port + '/v1');
await connection.getByLabel('API Key(按服务要求填写)').fill('ui-fixture-memory-key');
await connection.getByLabel('模型名称').fill('ui-diagnostics-fixture');
await connection.getByRole('button', { name: '验证模型响应', exact: true }).click();
await connection.getByText(/模型响应验证通过/).waitFor(); assert.equal((await api('/diagnostics')).connection.mode, 'demo');
await connection.getByRole('button', { name: '保存并切换原连接', exact: true }).click(); await connection.waitFor({ state: 'hidden' });
mark('model response validation checks the tentative connection before saving');
requestCount = 0; await task('制造读取成功后的模型服务错误'); await waitState('failed');
const failed = await state(); assert.equal(failed.failure.source, 'model_connection'); assert.equal(failed.failure.httpStatus, 503); assert.equal(failed.failure.attempts, 3);
assert.equal(failed.events.filter(event => event.tool === 'read_file').length, 1);
await page.locator('.failure-card').waitFor(); await screenshot('formal-model-error');
await page.locator('.failure-card').getByRole('button', { name: '查看运行诊断' }).click();
const diagnostic = page.getByRole('dialog', { name: '运行诊断' }); await diagnostic.getByText('Windows PowerShell 5.1', { exact: true }).waitFor();
await diagnostic.getByText(/HTTP 503/).first().waitFor(); await screenshot('formal-diagnostics');
assert.ok(!(await diagnostic.innerText()).includes('ui-fixture-memory-key'));
await diagnostic.getByRole('button', { name: '关闭窗口', exact: true }).click();
mark('a model failure after successful tools shows source, attempts and local runtime diagnosis');
mode = 'recover'; await page.locator('.failure-card').getByRole('button', { name: '重试模型请求', exact: true }).click(); await waitState('completed');
const recovered = await state(); assert.equal(recovered.events.filter(event => event.tool === 'read_file').length, 1);
assert.equal(recovered.messages.filter(message => message.role === 'user').length, 1);
mark('explicit retry preserves the completed tool result without replaying the task');
mode = 'question'; await page.getByLabel('任务内容').fill('刚才工作台报错怎么回事?请查执行记录。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
await page.getByText(/已读取工作台执行记录/).waitFor(); await waitState('completed');
const answered = await state(); assert.ok(answered.events.some(event => event.tool === 'inspect_session' && event.phase === 'completed'));
assert.equal(diagnosticReads, 1, 'Read the actual saved diagnosis once, including when its raw group is projected.');
assert.equal(answered.events.filter(event => event.tool === 'inspect_session').length, 1);
assert.equal(answered.events.filter(event => event.tool === 'search_files').length, 0);
assert.ok(!await page.locator('.message-assistant .message-text').last().innerText().then(text => text.includes('**')));
await screenshot('formal-diagnostic-answer'); mark('an error question receives the actual session diagnosis and formatted answer');
mode = 'proposal'; await task('准备一个可审查文件'); await page.getByRole('button', { name: '审核 1 项', exact: true }).waitFor();
await page.getByRole('button', { name: '审核 1 项', exact: true }).click();
const apply = page.getByRole('button', { name: '应用这项改动', exact: true }); assert.ok(await apply.isDisabled());
assert.equal(fs.existsSync(path.join(projectRoot, 'REVIEWED.md')), false);
await waitState('needs_review'); assert.equal(await page.getByRole('button', { name: '继续任务', exact: true }).count(), 0);
await screenshot('formal-review'); await apply.click(); await waitState('completed');
assert.equal(fs.readFileSync(path.join(projectRoot, 'REVIEWED.md'), 'utf8'), '# Reviewed through the UI\n');
mark('review state exposes the review action and prevents application during a running request');
mode = 'command'; await task('验证中文终端输入输出'); await waitState('awaiting_approval');
await page.locator('.approval-shell').getByText(/PowerShell 5.1/).waitFor(); await screenshot('formal-command-approval');
await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitState('completed');
assert.match((await state()).events.find(event => event.tool === 'run_command').output, /中文输入与输出正确/);
mark('approved Unicode commands execute in the declared shell and retain Chinese output');
mode = 'authentication'; requestCount = 0; await task('验证身份失败的处理'); await waitState('failed'); assert.equal(requestCount, 1);
assert.match(await page.locator('.failure-card').innerText(), /HTTP 401/);
await page.setViewportSize({ width: 390, height: 844 }); await screenshot('formal-mobile-error');
await page.locator('.failure-card').getByRole('button', { name: '查看运行诊断' }).click();
await page.getByRole('dialog', { name: '运行诊断' }).getByText('Windows PowerShell 5.1', { exact: true }).waitFor();
const box = await page.getByRole('dialog', { name: '运行诊断' }).boundingBox(); assert.ok(box.x >= 0 && box.x + box.width <= 391);
await screenshot('formal-mobile-diagnostics'); await page.getByRole('button', { name: '关闭窗口', exact: true }).click();
await page.setViewportSize({ width: 1440, height: 960 }); mark('authentication errors do not retry and error/diagnostic controls fit a 390 px viewport');
mode = 'backoff'; requestCount = 0; await task('停止自动重试'); await page.getByText('正在重试模型连接', { exact: true }).waitFor();
await page.getByRole('button', { name: '停止', exact: true }).click(); await waitState('paused'); assert.equal(requestCount, 1);
assert.equal((await state()).events.find(event => event.kind === 'model').phase, 'interrupted');
assert.equal((await api('/diagnostics')).connection.lastRequest.status, 'interrupted');
await page.locator('.task-toolbar').getByRole('button', { name: '运行诊断', exact: true }).click();
const stoppedDiagnostic = page.getByRole('dialog', { name: '运行诊断' });
await stoppedDiagnostic.getByText(/最近请求:已停止/).waitFor();
await screenshot('formal-stopped-diagnostics');
await stoppedDiagnostic.getByRole('button', { name: '关闭窗口', exact: true }).click();
mark('stop cancels pending retry and clears running model state');
mode = 'transient'; requestCount = 0; await task('验证自动恢复'); await waitState('completed'); assert.equal(requestCount, 2);
await screenshot('formal-desktop'); mark('a transient service failure recovers within the bounded retry policy');
assert.deepEqual(pageErrors, []); assert.deepEqual(networkErrors, []);
const result = { success: true, version: bootstrap.version, url, date: new Date().toISOString(), passed, pageErrors, requestFailures: networkErrors, diagnosticReads, diagnosticProjectionUsed, modelValidation: 'controlled local HTTP fixture; actual configured model is verified separately' };
fs.writeFileSync('.qa/diagnostics-ui-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
} catch (error) {
if (page) await page.screenshot({ path: '.qa/screenshots/formal-ui-failure.png', scale: 'css' }).catch(() => {});
console.error(error); throw error;
} finally {
await browser.close();
await api('/config', { ...bootstrap.config, mode: 'demo', apiKey: '' }).catch(() => {});
model.closeAllConnections(); await new Promise(resolve => model.close(resolve));
}