Files
Agent-Workbench/scripts/verify-mcp-schema-ui.mjs
Agent Workbench Dev e306f40cae
Official contracts and Windows source CI / windows-source (push) Has been cancelled
feat(memory): release Mnemo project bridge and schema2 recovery
2026-11-11 03:03:43 +08:00

104 lines
13 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import net from 'node:net';
import assert from 'node:assert/strict';
import { randomUUID, createHash } from 'node:crypto';
import { chromium, _electron } from 'playwright';
import { expect } from 'playwright/test';
import { fixture, launchWorkbench, compatibleFixture, completion, call, until } from '../tests/helpers.mjs';
import { HostController } from '../src/core/desktop/host-controller.mjs';
import { desktopLaunchCommand } from './desktop-launch.mjs';
import { projectEnvironment } from '../src/core/process-environment.mjs';
import { advancedArguments, advancedSchema, schemaFixtureTools } from '../tests/fixtures/schema-mcp-server.mjs';
import { openWorkbenchTool } from './ui-navigation.mjs';
const desktop = process.argv.includes('--desktop'), appRoot = fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT);
const root = path.resolve('.qa/m1-mcp-schema-2026-11-07'), directory = path.join(root, (desktop ? 'electron-' : 'web-') + randomUUID());
fs.mkdirSync(directory); console.log('EVIDENCE ' + directory);
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true }); f.store.saveConfig({ approvalMode: 'full' }); f.store.close();
const entry = path.join(f.root, 'schema-server.mjs'); fs.copyFileSync('tests/fixtures/schema-mcp-server.mjs', entry);
let registration, selectedTool = 'advanced', selectedArguments = advancedArguments(), phaseRequests = 0;
const model = await compatibleFixture(t, () => ++phaseRequests === 1 ? completion([call('owned-ui-schema-' + selectedTool, 'call_registered_mcp', {
registration_id: registration.id, configuration_revision: registration.configurationRevision, contract_sha256: registration.contract.sha256, tool: selectedTool, arguments: selectedArguments
})]) : completion([], '自有服务器已保留本轮实际结果。'));
let host, controller, browser, application, page, window, result; const errors = [], checks = [], screenshots = [];
const mark = value => { checks.push(value); console.log('PASS ' + value); };
async function resize(width) {
if (window) { await window.evaluate((win, width) => win.setContentSize(width, 960), width); await until(() => page.evaluate(width => innerWidth === width, width)); }
else await page.setViewportSize({ width, height: 960 });
}
async function shot(name) {
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); const file = path.join(directory, name + '.png');
await page.screenshot({ path: file, scale: 'css' }); screenshots.push({ file: path.basename(file), sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') });
fs.writeFileSync(path.join(directory, name + '.txt'), await page.locator('body').innerText());
}
async function closeDialog() { await page.getByRole('dialog').getByRole('button', { name: '关闭窗口', exact: true }).click(); }
async function callSelected() {
phaseRequests = 0;
await host.api('/sessions/' + f.session.id + '/messages', { content: '使用已启用工具 ' + selectedTool + ' 一次;保留原参数与服务器结果,不重试。' });
await until(async () => !['running', 'awaiting_approval', 'stopping'].includes((await host.api('/sessions/' + f.session.id)).status), 45000);
const jobs = await host.api('/sessions/' + f.session.id + '/mcp/jobs'); const job = jobs.find(value => value.operation === 'call' && value.input.call.tool === selectedTool);
assert.ok(job, 'Actual owned MCP call must exist'); return { job, evidence: await host.api('/sessions/' + f.session.id + '/mcp/jobs/' + job.id + '/evidence') };
}
try {
if (desktop) {
const reservation = net.createServer(); await new Promise(resolve => reservation.listen(0, '127.0.0.1', resolve)); const port = reservation.address().port; await new Promise(resolve => reservation.close(resolve)); assert.notEqual(port, 4318);
controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start();
host = { url: controller.url, api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) };
application = await _electron.launch({ ...desktopLaunchCommand(appRoot, ['--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window']), cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
page = await application.firstWindow(); window = await application.browserWindow(page); await window.evaluate(win => win.showInactive());
} else { host = await launchWorkbench(t, f.data, {}, { appRoot }); browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage(); }
assert.notEqual(new URL(host.url).port, '4318'); page.on('pageerror', error => errors.push(error.message));
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
await resize(1280); await page.goto(host.url); await openWorkbenchTool(page, '项目MCP工具');
let panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }), form = panel.locator('.mcp-register');
await form.getByLabel('显示名称', { exact: true }).fill('原始 Schema 工程工具'); await form.getByLabel('可执行文件绝对路径', { exact: true }).fill(process.execPath);
await form.getByLabel('Node 脚本入口(可选)', { exact: true }).fill(entry); await form.getByRole('button', { name: '登记后保持停用', exact: true }).click();
let row = panel.locator('.mcp-registration'); await expect(row).toContainText('尚未探测'); await row.getByRole('button', { name: '实际探测', exact: true }).click();
await page.getByRole('dialog', { name: '启动 MCP 实际探测', exact: true }).getByRole('button', { name: '确认启动并探测', exact: true }).click(); await expect(row).toContainText('握手已核验');
await expect(row).toContainText('参数由工具服务器校验'); await expect(row.locator('.mcp-tool-option')).toHaveCount(schemaFixtureTools().length);
let advanced = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^advanced$/ }) });
const invalid = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^invalid_root$/ }) }); await expect(invalid.locator('input')).toBeDisabled();
await advanced.locator('summary').click(); assert.deepEqual(JSON.parse(await advanced.locator('pre').innerText()), advancedSchema());
await advanced.locator('input').check(); for (let index = 0; index < 13; index++) await row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: new RegExp('^extra_' + index + '$') }) }).locator('input').check();
await row.getByRole('button', { name: '保存并启用所选工具', exact: true }).click(); await expect(row).toContainText('已选择 14');
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0]; assert.equal(registration.allowedTools.length, 14); await advanced.scrollIntoViewIfNeeded(); await shot('schema-1280');
mark('Real stdio probe exposes the complete union/ref/deep/80-property schema, permits 14 tools and keeps invalid object envelopes unavailable');
await page.reload(); await openWorkbenchTool(page, '项目MCP工具'); panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); row = panel.locator('.mcp-registration');
await expect(row).toContainText('已启用'); await expect(row).toContainText('已选择 14'); await resize(390);
advanced = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^advanced$/ }) }); await advanced.locator('summary').click(); await advanced.scrollIntoViewIfNeeded();
assert.deepEqual(JSON.parse(await advanced.locator('pre').innerText()), advancedSchema()); await shot('schema-390'); mark('Saved selection and original large schema survive reload and remain readable at 390px');
await closeDialog(); await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'owned-ui-schema-model', apiKey: 'owned-ui-schema-fixture-key', approvalMode: 'full' });
const accepted = await callSelected(); assert.equal(accepted.job.status, 'completed'); assert.deepEqual(accepted.evidence.report.result._meta.received, advancedArguments());
assert.equal(accepted.evidence.report.trace.filter(value => value.method === 'tools/call').length, 1); fs.writeFileSync(path.join(directory, 'accepted-evidence.json'), JSON.stringify(accepted.evidence, null, 2));
mark('Compiled host forwards all 2100 argument records once, receives original server result and requires no manual approval');
await openWorkbenchTool(page, '项目MCP工具'); panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); row = panel.locator('.mcp-registration');
const errorTool = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^tool_error$/ }) }); await errorTool.locator('input').check(); await row.getByRole('button', { name: '保存并启用所选工具', exact: true }).click();
await expect(row).toContainText('已选择 15'); registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0]; await closeDialog();
selectedTool = 'tool_error'; selectedArguments = { fail: true, original: '服务器决定是否符合约束' }; const rejected = await callSelected(); assert.equal(rejected.job.status, 'failed'); assert.equal(rejected.evidence.report.result.isError, true);
fs.writeFileSync(path.join(directory, 'rejected-evidence.json'), JSON.stringify(rejected.evidence, null, 2));
await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); const history = page.getByLabel('MCP运行记录', { exact: true }); await expect(history).toContainText('tool_error');
await history.locator('.mcp-history-row').filter({ hasText: 'tool_error' }).getByRole('button', { name: '查看原报告', exact: true }).click(); await expect(history.locator('.mcp-report')).toContainText('原服务器校验失败');
await history.locator('.mcp-report').scrollIntoViewIfNeeded();
const revealServerError = () => history.locator('.mcp-report > pre').evaluate(element => {
const text = element.textContent, offset = text.indexOf('原服务器校验失败');
if (offset < 0 || !element.firstChild) throw new Error('Original server failure must exist in the displayed report');
const range = document.createRange(); range.setStart(element.firstChild, offset); range.collapse(true);
element.scrollTop += range.getBoundingClientRect().top - element.getBoundingClientRect().top - 50;
});
await revealServerError(); await shot('server-error-390'); await resize(1280); await revealServerError(); await shot('server-error-1280');
mark('Original CallToolResult validation failure is visible in the task report, remains failed and is not retried');
await page.reload(); await openWorkbenchTool(page, '项目MCP工具'); panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); row = panel.locator('.mcp-registration');
fs.appendFileSync(entry, '\n// Owned source drift for disable verification\n'); await panel.getByRole('button', { name: '刷新', exact: true }).click(); await expect(row).toContainText('来源或契约异常');
await row.getByRole('button', { name: '立即停用', exact: true }).click(); await expect(row).toContainText('已停用');
await expect(row).toContainText('原契约已保留,当前程序待重新核验'); await expect(row).not.toContainText('实际服务没有声明可发现工具。');
await row.scrollIntoViewIfNeeded(); await shot('source-drift-1280');
mark('Changed executable entry still reports drift and can be disabled without running the changed program');
assert.deepEqual(errors, []); result = { success: true, actualElectron: desktop, appRoot, checks, screenshots, pageErrors: errors, viewportWidths: [1280, 390], localModelFixtureRequests: model.requests.length, actualMcpCalls: 2, realModelRequests: 0, productionWrites: 0 };
} catch (error) { if (page) await shot('failure').catch(() => {}); result = { success: false, actualElectron: desktop, checks, screenshots, pageErrors: errors, error: error.stack, realModelRequests: 0 }; }
finally {
if (browser) await browser.close(); if (application) await application.close(); if (controller && await controller.health()) await controller.stop(); await host?.close?.();
for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
}
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;