142 lines
15 KiB
JavaScript
142 lines
15 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import http from 'node:http';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { chromium, expect } from 'playwright/test';
|
|
|
|
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4319';
|
|
if (!/^http:\/\/127\.0\.0\.1:(?!4318(?:\/|$))\d+\/?$/.test(url)) throw new Error('Use an independent local QA service.');
|
|
const root = path.resolve('.qa', 'execution-ui-' + randomUUID()), projectRoot = path.join(root, 'project'), otherRoot = path.join(root, 'other-project');
|
|
fs.mkdirSync(path.join(projectRoot, 'test'), { recursive: true }); fs.mkdirSync(otherRoot, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
|
fs.writeFileSync(path.join(projectRoot, 'package.json'), JSON.stringify({ name: 'execution-ui-project', type: 'module', scripts: { test: 'node --test test/*.test.mjs', build: 'node --check src.mjs' } }));
|
|
fs.writeFileSync(path.join(projectRoot, 'src.mjs'), 'export const add = (a, b) => a + b;\n');
|
|
fs.writeFileSync(path.join(projectRoot, 'test/cases.test.mjs'), "import test from 'node:test'; import assert from 'node:assert/strict'; import { add } from '../src.mjs'; test('addition', () => assert.equal(add(2, 3), 5)); test('negative values', () => assert.equal(add(-1, 3), 2));");
|
|
fs.writeFileSync(path.join(otherRoot, 'README.md'), '# Secondary project\n');
|
|
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
|
assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
|
const api = async (route, input) => {
|
|
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
|
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
|
};
|
|
let mode = 'checks', expectedDisconnect = false, eventsPaused = false, page, selectedId = '';
|
|
const eventConnections = new Set();
|
|
const proxy = http.createServer((request, response) => {
|
|
const eventStream = request.url.startsWith('/api/events?');
|
|
if (eventStream && eventsPaused) { response.writeHead(503); response.end(); return; }
|
|
const headers = { ...request.headers, host: new URL(url).host };
|
|
if (headers.origin) headers.origin = url;
|
|
const upstream = http.request(url + request.url, { method: request.method, headers }, result => { response.writeHead(result.statusCode, result.headers); result.pipe(response); });
|
|
const connection = { response, upstream }; if (eventStream) eventConnections.add(connection);
|
|
response.once('close', () => { eventConnections.delete(connection); upstream.destroy(); });
|
|
upstream.once('error', () => { if (!response.destroyed) { response.writeHead(502); response.end(); } });
|
|
request.pipe(upstream);
|
|
});
|
|
await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve));
|
|
const browserUrl = 'http://127.0.0.1:' + proxy.address().port;
|
|
const passed = [], errors = [], requestFailures = [];
|
|
const mark = message => { passed.push(message); console.log('PASS ' + message); };
|
|
const model = http.createServer(async (request, response) => {
|
|
try {
|
|
const parts = []; for await (const part of request) parts.push(part);
|
|
const input = JSON.parse(Buffer.concat(parts).toString());
|
|
const lastUser = input.messages.findLastIndex(message => message.role === 'user');
|
|
const results = input.messages.slice(lastUser + 1).filter(message => message.role === 'tool').map(message => JSON.parse(message.content));
|
|
const plan = results.find(value => value.id && value.checks && value.fingerprint);
|
|
let call;
|
|
if (mode === 'checks') call = !results.length ? ['inspect_project', {}] : results.length === 1 ? ['plan_checks', {}] : results.length === 2 ? ['run_checks', { plan_id: plan.id }] : null;
|
|
if (mode === 'budget') call = ['list_files', { path: '' }];
|
|
if (mode === 'proposal') call = !results.length ? ['propose_file', { path: 'src.mjs', content: 'export const add = (a, b) => a + b + 0;\n', reason: 'Verify reviewed task continuation' }] : null;
|
|
response.writeHead(200, { 'Content-Type': 'application/json' });
|
|
response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: call ? '' : mode === 'answer' ? '依据已保存记录完成本次回应。' : '已记录实际工程检查和日志。', ...(call ? { tool_calls: [{ id: randomUUID(), type: 'function', function: { name: call[0], arguments: JSON.stringify(call[1]) } }] } : {}) }, finish_reason: call ? 'tool_calls' : 'stop' }], usage: mode === 'budget' ? { prompt_tokens: 1000, completion_tokens: 100 } : { prompt_tokens: 40, completion_tokens: 20 } }));
|
|
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
|
|
});
|
|
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
|
const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
|
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
|
|
const state = () => api('/sessions/' + selectedId);
|
|
const waitState = async status => { await expect.poll(async () => (await state()).status, { timeout: 20000 }).toBe(status); };
|
|
const screenshot = name => page.screenshot({ path: '.qa/screenshots/' + name + '.png', scale: 'css' });
|
|
const taskPanel = () => page.getByLabel('任务记录', { exact: true });
|
|
async function newTask(content) {
|
|
if (!await page.getByRole('button', { name: '新建任务', exact: true }).isVisible()) await page.getByRole('button', { name: '展开项目栏' }).click();
|
|
await page.getByRole('button', { name: '新建任务', exact: true }).click(); await page.getByLabel('任务内容').fill(content); await page.getByRole('button', { name: '发送任务' }).click();
|
|
await expect.poll(async () => (await api('/sessions')).find(value => value.title === content)?.id, { timeout: 10000 }).toBeTruthy();
|
|
selectedId = (await api('/sessions')).find(value => value.title === content).id;
|
|
}
|
|
try {
|
|
const project = await api('/projects', { root: projectRoot }), other = await api('/projects', { root: otherRoot });
|
|
await api('/config', { ...bootstrap.config, mode: 'compatible', model: 'execution-ui-fixture', baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', apiKey: 'execution-ui-fixture-key' });
|
|
page = await context.newPage(); page.on('pageerror', error => errors.push(error.message)); page.on('requestfailed', request => { if (!expectedDisconnect) requestFailures.push(request.url() + ': ' + request.failure()?.errorText); });
|
|
await page.goto(browserUrl); await page.waitForLoadState('networkidle');
|
|
await page.getByLabel('当前项目').selectOption(project.id);
|
|
await newTask('验证真实工程检查'); await waitState('awaiting_approval');
|
|
await expect(page.locator('.approval-card')).toContainText('npm run test'); await expect(page.locator('.approval-card')).toContainText('npm run build');
|
|
assert.equal((await state()).execution.checks.length, 0);
|
|
await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitState('completed');
|
|
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(taskPanel()).toContainText('验证真实工程检查');
|
|
await expect(taskPanel()).toContainText('execution-ui-fixture'); await expect(taskPanel()).toContainText('240 tokens'); await expect(taskPanel()).toContainText('测试 2/2 通过');
|
|
assert.equal((await state()).execution.checks.at(-1).status, 'passed');
|
|
await screenshot('execution-desktop'); mark('model-selected checks wait for approval and display actual counted results');
|
|
|
|
await taskPanel().getByRole('button', { name: '查看检查日志' }).first().click();
|
|
const artifact = page.getByRole('dialog', { name: '执行结果工件' }); await expect(artifact).toContainText('tests 2'); await expect(artifact).toContainText('SHA256');
|
|
await screenshot('execution-artifact'); await artifact.getByRole('button', { name: '关闭窗口' }).click(); mark('verified artifact preview shows the real check log and hash');
|
|
|
|
fs.appendFileSync(path.join(projectRoot, 'src.mjs'), '\n// Source changed after verification.\n');
|
|
await taskPanel().getByRole('button', { name: '重新识别检查计划' }).click(); await expect(taskPanel().locator('.project-checks > .record-heading .status')).toHaveText('需重新检查');
|
|
await taskPanel().getByRole('button', { name: '执行这些检查' }).click(); const confirmation = page.getByRole('dialog', { name: '执行工程检查' });
|
|
await expect(confirmation).toContainText('node --test'); await expect(confirmation).toContainText('node --check');
|
|
await confirmation.getByRole('button', { name: '确认执行检查' }).click();
|
|
await expect.poll(async () => (await state()).execution.checks.at(-1).status, { timeout: 15000 }).toBe('passed');
|
|
await expect(taskPanel().locator('.project-checks > .record-heading .status')).toHaveText('检查通过'); mark('changed source invalidates old checks and manual confirmation runs the new plan');
|
|
|
|
await page.setViewportSize({ width: 390, height: 844 }); await page.getByRole('button', { name: '查看任务记录' }).click();
|
|
await expect(taskPanel()).toBeVisible(); await taskPanel().getByRole('button', { name: '查看检查日志' }).first().click();
|
|
await expect(artifact).toBeVisible(); await screenshot('execution-mobile-artifact'); await artifact.getByRole('button', { name: '关闭窗口' }).click();
|
|
await screenshot('execution-mobile-task'); await page.getByRole('button', { name: '关闭检视区' }).click();
|
|
const composerBox = await page.getByLabel('任务内容').boundingBox(), headerBox = await page.locator('.work-header').boundingBox();
|
|
assert.ok(composerBox.y >= 0 && composerBox.y + composerBox.height <= 844); assert.ok(headerBox.y >= 0); assert.ok((await page.locator('body').evaluate(node => node.scrollWidth)) <= 390);
|
|
mark('390px task drawer, log preview, close controls and fixed composer remain operable');
|
|
|
|
await page.setViewportSize({ width: 1440, height: 1000 });
|
|
expectedDisconnect = true; eventsPaused = true; for (const connection of eventConnections) { connection.response.destroy(); connection.upstream.destroy(); } await expect(page.locator('.connection')).not.toHaveClass(/connected/);
|
|
mode = 'answer'; const offlineSession = await api('/sessions', { projectId: other.id }); await api('/sessions/' + offlineSession.id + '/messages', { content: '断线期间的其他项目任务' });
|
|
await expect.poll(async () => (await api('/sessions/' + offlineSession.id)).status).toBe('completed');
|
|
eventsPaused = false; await expect(page.locator('.connection')).toHaveClass(/connected/, { timeout: 15000 });
|
|
await page.getByRole('button', { name: '展开项目栏' }).click(); await page.getByLabel('当前项目').selectOption(other.id);
|
|
await expect(page.locator('.session-list')).toContainText('断线期间的其他项目任务'); await page.getByLabel('当前项目').selectOption(project.id); expectedDisconnect = false;
|
|
mark('reconnection synchronizes missed session changes in other projects');
|
|
|
|
await page.getByRole('button', { name: '模型连接', exact: true }).click(); const settings = page.getByRole('dialog', { name: '模型连接' });
|
|
await settings.getByText('上下文与任务预算', { exact: true }).click(); await settings.getByLabel('单次运行token预算').fill('1000'); await settings.getByLabel('运行时间预算').fill('2'); await settings.getByLabel('任务累计token预算').fill('50000');
|
|
await settings.getByRole('button', { name: '保存连接' }).click(); mode = 'budget'; await newTask('保留预算暂停的任务目标'); await waitState('paused');
|
|
const original = await state(); assert.equal(original.execution.run.stopReason, 'token_limit'); assert.equal(original.approvals.length, 0);
|
|
await page.getByRole('button', { name: '模型连接', exact: true }).click(); await settings.getByText('上下文与任务预算', { exact: true }).click(); await settings.getByLabel('单次运行token预算').fill('16000'); await settings.getByRole('button', { name: '保存连接' }).click();
|
|
mode = 'answer'; await page.getByRole('button', { name: '继续任务', exact: true }).click(); await waitState('completed');
|
|
const resumed = await state(); assert.equal(resumed.execution.task.id, original.execution.task.id); assert.equal(resumed.execution.runs.length, 2);
|
|
await page.getByRole('button', { name: '查看任务记录' }).click(); await taskPanel().getByLabel('选择运行记录').selectOption(original.execution.run.id);
|
|
await expect(taskPanel()).toContainText('到达本次 token 预算'); await expect(taskPanel()).toContainText('1,000 tokens'); await screenshot('execution-budget-history');
|
|
mark('budget controls pause before tools and preserve the goal across recorded runs');
|
|
|
|
mode = 'proposal'; await newTask('应用后继续同一个工程任务'); await waitState('needs_review'); const reviewed = await state();
|
|
await page.locator('.inspector-tabs').getByRole('button', { name: /改动/ }).click(); await page.getByRole('button', { name: '应用这项改动', exact: true }).click();
|
|
await waitState('completed'); mode = 'checks'; await page.getByRole('button', { name: '继续检查与交付', exact: true }).click(); await waitState('awaiting_approval');
|
|
await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitState('completed');
|
|
const delivered = await state(); assert.equal(delivered.execution.task.id, reviewed.execution.task.id); assert.equal(delivered.execution.task.goal, '应用后继续同一个工程任务'); assert.equal(delivered.execution.checks.at(-1).status, 'passed');
|
|
mark('applied proposals continue the original task into actual engineering verification');
|
|
|
|
assert.deepEqual(errors, []); assert.deepEqual(requestFailures, []);
|
|
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), browser: browser.version(), passed, pageErrors: errors, requestFailures, projectRoot, model: 'controlled local HTTP fixture' };
|
|
fs.writeFileSync('.qa/execution-ui-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
|
|
} catch (error) {
|
|
if (page) { await screenshot('execution-ui-failure').catch(() => {}); fs.writeFileSync('.qa/execution-failure-dom.html', await page.content().catch(() => '')); }
|
|
fs.writeFileSync('.qa/execution-ui-validation.json', JSON.stringify({ success: false, error: error.message, passed, pageErrors: errors, requestFailures }, null, 2)); throw error;
|
|
} finally {
|
|
await context.setOffline(false).catch(() => {}); await browser.close();
|
|
if (selectedId) await api('/sessions/' + selectedId + '/stop', {}).catch(() => {});
|
|
await api('/config', { ...bootstrap.config, mode: 'demo', apiKey: '' }).catch(() => {});
|
|
model.closeAllConnections(); await new Promise(resolve => model.close(resolve));
|
|
proxy.closeAllConnections(); await new Promise(resolve => proxy.close(resolve));
|
|
}
|