Files
Agent-Workbench/scripts/verify-mcp-ui.mjs
T
2026-10-30 20:34:54 +08:00

112 lines
9.5 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { chromium, expect } from 'playwright/test';
import { fixture, launchWorkbench, compatibleFixture, completion, call, until } from '../tests/helpers.mjs';
const directory = path.resolve('.qa', 'mcp-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t); f.store.close();
const entry = path.join(f.root, 'registered-mcp.mjs'); fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
let registration;
const model = await compatibleFixture(t, () => completion([call('ui-unknown-call', 'call_registered_mcp', {
registration_id: registration.id, configuration_revision: registration.configurationRevision, contract_sha256: registration.contract.sha256,
tool: 'hold', arguments: { text: 'effect-can-be-unknown' },
})]));
const host = await launchWorkbench(t, f.data); const browser = await chromium.launch({ channel: 'msedge', headless: true });
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
const pageErrors = [], requestFailures = [], assertions = [];
page.on('pageerror', error => pageErrors.push(error.message));
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
const mark = value => { assertions.push(value); console.log('PASS ' + value); };
try {
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
await page.goto(host.url); await page.waitForLoadState('networkidle');
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
const dialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); await expect(dialog).toBeVisible();
const form = dialog.locator('.mcp-register');
await form.getByLabel('显示名称', { exact: true }).fill('只读与工程工具');
await form.getByLabel('可执行文件绝对路径', { exact: true }).fill(process.execPath);
await form.getByLabel('Node 脚本入口(可选)', { exact: true }).fill(entry);
await form.getByRole('button', { name: '登记后保持停用', exact: true }).click();
const row = dialog.locator('.mcp-registration').filter({ hasText: '只读与工程工具' });
await expect(row).toContainText('已停用'); await expect(row).toContainText('尚未探测');
mark('MCP registration stays disabled and shows the hashed local executable and Node entry');
await row.getByRole('button', { name: '实际探测', exact: true }).click();
const confirm = page.getByRole('dialog', { name: '启动 MCP 实际探测' });
await expect(confirm).toContainText('不等于 OS 沙箱');
await confirm.getByRole('button', { name: '确认启动并探测', exact: true }).click();
await expect(confirm).toBeHidden(); await expect(row).toContainText('握手已核验'); await expect(row).toContainText('发现 3 项工具');
await expect(row.locator('.mcp-tool-option')).toHaveCount(3);
const echo = row.locator('.mcp-tool-option').filter({ hasText: 'echo' });
await echo.locator('summary').click(); await expect(echo.locator('pre')).toContainText('additionalProperties');
await echo.locator('input[type="checkbox"]').check();
await row.getByRole('button', { name: '保存并启用所选工具', exact: true }).click();
await expect(row).toContainText('已启用'); await expect(dialog.locator('.mcp-history-row')).toContainText('完成');
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
assert.equal(registration.enabled, true); assert.deepEqual(registration.allowedTools, ['echo']);
await dialog.locator('.mcp-heading').scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(directory, 'desktop-mcp-registry.png') });
mark('actual stdio tools/list schema is visible; only the reviewed echo tool is enabled');
fs.appendFileSync(entry, '\n// Owned changed entry for disable acceptance.\n');
await dialog.getByRole('button', { name: '刷新', exact: true }).click(); await expect(row).toContainText('来源或契约异常');
await row.getByRole('button', { name: '立即停用', exact: true }).click(); await expect(row).toContainText('已停用');
assert.equal((await host.api('/projects/' + f.project.id + '/mcp')).registrations[0].enabled, false);
fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
await dialog.getByRole('button', { name: '刷新', exact: true }).click();
mark('an enabled program whose source changes can be disabled immediately without executing the changed entry');
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
await host.api('/projects/' + f.project.id + '/mcp/configure', { id: registration.id, configurationRevision: registration.configurationRevision, enabled: true, allowedTools: ['hold'] });
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'owned-mcp-ui', apiKey: 'ui-mcp-fixture-key', contextLimit: 24000 });
await host.api('/sessions/' + f.session.id + '/messages', { content: 'Use the enabled external tool and preserve its real execution record.' });
await until(async () => (await host.api('/sessions/' + f.session.id)).status === 'awaiting_approval');
await page.keyboard.press('Escape');
await page.getByRole('button', { name: '确认执行', exact: true }).click();
await until(async () => (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).some(value => value.operation === 'call' && value.status === 'calling'));
const job = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.operation === 'call' && value.status === 'calling');
await until(() => fs.existsSync(path.join(job.directory, 'owned-call-dispatched.txt')));
await page.getByRole('button', { name: '停止', exact: true }).click();
await until(async () => (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).some(value => value.id === job.id && value.status === 'unknown'));
const unknown = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === job.id);
assert.equal(unknown.requiresVerification, true); assert.ok(unknown.reportArtifactId);
mark('approved external call interrupted after dispatch remains unknown and does not replay');
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
const history = page.getByLabel('MCP运行记录', { exact: true }); await expect(history).toContainText('结果待核对');
await history.locator('.mcp-history-row.state-unknown').getByRole('button', { name: '查看原报告', exact: true }).click();
await expect(history.getByLabel('MCP原报告')).toContainText('"status": "unknown"');
await history.getByLabel('MCP原报告').locator('summary').click();
await expect(history.getByLabel('MCP原报告')).toContainText('effect-can-be-unknown');
await history.getByRole('button', { name: '人工核对结果', exact: true }).click();
const review = page.getByRole('dialog', { name: '核对未知的外部结果' }); await expect(review).toContainText('不会把未知结果改成成功');
await review.getByRole('button', { name: '我已查看原报告', exact: true }).click();
await expect(review).toBeHidden();
const acknowledged = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === job.id);
assert.equal(acknowledged.status, 'unknown'); assert.equal(acknowledged.requiresVerification, false);
assert.deepEqual((await host.api('/sessions/' + f.session.id)).execution.progress.requiresVerification, []);
mark('TaskPanel exposes Task/Run source, original report, and human acknowledgement without marking unknown as success');
await page.setViewportSize({ width: 390, height: 844 });
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
const mobileDialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); await expect(mobileDialog).toBeVisible();
await expect(mobileDialog.locator('.mcp-registration')).toHaveCount(1);
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
await mobileDialog.locator('.mcp-heading').scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(directory, 'mobile-mcp-registry.png') });
mark('390px MCP registry, long executable paths, controls, and operation history have no horizontal overflow');
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, assertions, viewportWidths: [1440, 390], pageErrors, requestFailures,
localFixtureModelRequests: model.requests.length, realModelRequests: 0, unknownJobId: job.id,
screenshots: ['desktop-mcp-registry.png', 'mobile-mcp-registry.png'] }, null, 2));
console.log('EVIDENCE ' + directory);
} catch (error) {
await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {});
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.message, assertions, pageErrors, requestFailures }, null, 2));
console.error('EVIDENCE ' + directory); throw error;
} finally { await browser.close(); await host.close(); for (const dispose of disposers.reverse()) await dispose(); }