104 lines
13 KiB
JavaScript
104 lines
13 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import net from 'node:net';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, _electron } from 'playwright';
|
|
import { expect } from 'playwright/test';
|
|
import { fixture, launchWorkbench, compatibleFixture, completion, call, until } from '../tests/helpers.mjs';
|
|
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
|
import { desktopLaunchCommand } from './desktop-launch.mjs';
|
|
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
|
import { advancedArguments, advancedSchema, schemaFixtureTools } from '../tests/fixtures/schema-mcp-server.mjs';
|
|
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
|
|
const desktop = process.argv.includes('--desktop'), appRoot = fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT);
|
|
const root = path.resolve('.qa/m1-mcp-schema-2026-11-07'), directory = path.join(root, (desktop ? 'electron-' : 'web-') + randomUUID());
|
|
fs.mkdirSync(directory); console.log('EVIDENCE ' + directory);
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true }); f.store.saveConfig({ approvalMode: 'full' }); f.store.close();
|
|
const entry = path.join(f.root, 'schema-server.mjs'); fs.copyFileSync('tests/fixtures/schema-mcp-server.mjs', entry);
|
|
let registration, selectedTool = 'advanced', selectedArguments = advancedArguments(), phaseRequests = 0;
|
|
const model = await compatibleFixture(t, () => ++phaseRequests === 1 ? completion([call('owned-ui-schema-' + selectedTool, 'call_registered_mcp', {
|
|
registration_id: registration.id, configuration_revision: registration.configurationRevision, contract_sha256: registration.contract.sha256, tool: selectedTool, arguments: selectedArguments
|
|
})]) : completion([], '自有服务器已保留本轮实际结果。'));
|
|
let host, controller, browser, application, page, window, result; const errors = [], checks = [], screenshots = [];
|
|
const mark = value => { checks.push(value); console.log('PASS ' + value); };
|
|
async function resize(width) {
|
|
if (window) { await window.evaluate((win, width) => win.setContentSize(width, 960), width); await until(() => page.evaluate(width => innerWidth === width, width)); }
|
|
else await page.setViewportSize({ width, height: 960 });
|
|
}
|
|
async function shot(name) {
|
|
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); const file = path.join(directory, name + '.png');
|
|
await page.screenshot({ path: file, scale: 'css' }); screenshots.push({ file: path.basename(file), sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') });
|
|
fs.writeFileSync(path.join(directory, name + '.txt'), await page.locator('body').innerText());
|
|
}
|
|
async function closeDialog() { await page.getByRole('dialog').getByRole('button', { name: '关闭窗口', exact: true }).click(); }
|
|
async function callSelected() {
|
|
phaseRequests = 0;
|
|
await host.api('/sessions/' + f.session.id + '/messages', { content: '使用已启用工具 ' + selectedTool + ' 一次;保留原参数与服务器结果,不重试。' });
|
|
await until(async () => !['running', 'awaiting_approval', 'stopping'].includes((await host.api('/sessions/' + f.session.id)).status), 45000);
|
|
const jobs = await host.api('/sessions/' + f.session.id + '/mcp/jobs'); const job = jobs.find(value => value.operation === 'call' && value.input.call.tool === selectedTool);
|
|
assert.ok(job, 'Actual owned MCP call must exist'); return { job, evidence: await host.api('/sessions/' + f.session.id + '/mcp/jobs/' + job.id + '/evidence') };
|
|
}
|
|
try {
|
|
if (desktop) {
|
|
const reservation = net.createServer(); await new Promise(resolve => reservation.listen(0, '127.0.0.1', resolve)); const port = reservation.address().port; await new Promise(resolve => reservation.close(resolve)); assert.notEqual(port, 4318);
|
|
controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start();
|
|
host = { url: controller.url, api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) };
|
|
application = await _electron.launch({ ...desktopLaunchCommand(appRoot, ['--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window']), cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
|
page = await application.firstWindow(); window = await application.browserWindow(page); await window.evaluate(win => win.showInactive());
|
|
} else { host = await launchWorkbench(t, f.data, {}, { appRoot }); browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage(); }
|
|
assert.notEqual(new URL(host.url).port, '4318'); page.on('pageerror', error => errors.push(error.message));
|
|
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
|
await resize(1280); await page.goto(host.url); await openWorkbenchTool(page, '项目MCP工具');
|
|
let panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }), form = panel.locator('.mcp-register');
|
|
await form.getByLabel('显示名称', { exact: true }).fill('原始 Schema 工程工具'); await form.getByLabel('可执行文件绝对路径', { exact: true }).fill(process.execPath);
|
|
await form.getByLabel('Node 脚本入口(可选)', { exact: true }).fill(entry); await form.getByRole('button', { name: '登记后保持停用', exact: true }).click();
|
|
let row = panel.locator('.mcp-registration'); await expect(row).toContainText('尚未探测'); await row.getByRole('button', { name: '实际探测', exact: true }).click();
|
|
await page.getByRole('dialog', { name: '启动 MCP 实际探测', exact: true }).getByRole('button', { name: '确认启动并探测', exact: true }).click(); await expect(row).toContainText('握手已核验');
|
|
await expect(row).toContainText('参数由工具服务器校验'); await expect(row.locator('.mcp-tool-option')).toHaveCount(schemaFixtureTools().length);
|
|
let advanced = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^advanced$/ }) });
|
|
const invalid = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^invalid_root$/ }) }); await expect(invalid.locator('input')).toBeDisabled();
|
|
await advanced.locator('summary').click(); assert.deepEqual(JSON.parse(await advanced.locator('pre').innerText()), advancedSchema());
|
|
await advanced.locator('input').check(); for (let index = 0; index < 13; index++) await row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: new RegExp('^extra_' + index + '$') }) }).locator('input').check();
|
|
await row.getByRole('button', { name: '保存并启用所选工具', exact: true }).click(); await expect(row).toContainText('已选择 14');
|
|
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0]; assert.equal(registration.allowedTools.length, 14); await advanced.scrollIntoViewIfNeeded(); await shot('schema-1280');
|
|
mark('Real stdio probe exposes the complete union/ref/deep/80-property schema, permits 14 tools and keeps invalid object envelopes unavailable');
|
|
await page.reload(); await openWorkbenchTool(page, '项目MCP工具'); panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); row = panel.locator('.mcp-registration');
|
|
await expect(row).toContainText('已启用'); await expect(row).toContainText('已选择 14'); await resize(390);
|
|
advanced = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^advanced$/ }) }); await advanced.locator('summary').click(); await advanced.scrollIntoViewIfNeeded();
|
|
assert.deepEqual(JSON.parse(await advanced.locator('pre').innerText()), advancedSchema()); await shot('schema-390'); mark('Saved selection and original large schema survive reload and remain readable at 390px');
|
|
await closeDialog(); await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'owned-ui-schema-model', apiKey: 'owned-ui-schema-fixture-key', approvalMode: 'full' });
|
|
const accepted = await callSelected(); assert.equal(accepted.job.status, 'completed'); assert.deepEqual(accepted.evidence.report.result._meta.received, advancedArguments());
|
|
assert.equal(accepted.evidence.report.trace.filter(value => value.method === 'tools/call').length, 1); fs.writeFileSync(path.join(directory, 'accepted-evidence.json'), JSON.stringify(accepted.evidence, null, 2));
|
|
mark('Compiled host forwards all 2100 argument records once, receives original server result and requires no manual approval');
|
|
await openWorkbenchTool(page, '项目MCP工具'); panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); row = panel.locator('.mcp-registration');
|
|
const errorTool = row.locator('.mcp-tool-option').filter({ has: page.locator('strong', { hasText: /^tool_error$/ }) }); await errorTool.locator('input').check(); await row.getByRole('button', { name: '保存并启用所选工具', exact: true }).click();
|
|
await expect(row).toContainText('已选择 15'); registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0]; await closeDialog();
|
|
selectedTool = 'tool_error'; selectedArguments = { fail: true, original: '服务器决定是否符合约束' }; const rejected = await callSelected(); assert.equal(rejected.job.status, 'failed'); assert.equal(rejected.evidence.report.result.isError, true);
|
|
fs.writeFileSync(path.join(directory, 'rejected-evidence.json'), JSON.stringify(rejected.evidence, null, 2));
|
|
await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); const history = page.getByLabel('MCP运行记录', { exact: true }); await expect(history).toContainText('tool_error');
|
|
await history.locator('.mcp-history-row').filter({ hasText: 'tool_error' }).getByRole('button', { name: '查看原报告', exact: true }).click(); await expect(history.locator('.mcp-report')).toContainText('原服务器校验失败');
|
|
await history.locator('.mcp-report').scrollIntoViewIfNeeded();
|
|
const revealServerError = () => history.locator('.mcp-report > pre').evaluate(element => {
|
|
const text = element.textContent, offset = text.indexOf('原服务器校验失败');
|
|
if (offset < 0 || !element.firstChild) throw new Error('Original server failure must exist in the displayed report');
|
|
const range = document.createRange(); range.setStart(element.firstChild, offset); range.collapse(true);
|
|
element.scrollTop += range.getBoundingClientRect().top - element.getBoundingClientRect().top - 50;
|
|
});
|
|
await revealServerError(); await shot('server-error-390'); await resize(1280); await revealServerError(); await shot('server-error-1280');
|
|
mark('Original CallToolResult validation failure is visible in the task report, remains failed and is not retried');
|
|
await page.reload(); await openWorkbenchTool(page, '项目MCP工具'); panel = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); row = panel.locator('.mcp-registration');
|
|
fs.appendFileSync(entry, '\n// Owned source drift for disable verification\n'); await panel.getByRole('button', { name: '刷新', exact: true }).click(); await expect(row).toContainText('来源或契约异常');
|
|
await row.getByRole('button', { name: '立即停用', exact: true }).click(); await expect(row).toContainText('已停用');
|
|
await expect(row).toContainText('原契约已保留,当前程序待重新核验'); await expect(row).not.toContainText('实际服务没有声明可发现工具。');
|
|
await row.scrollIntoViewIfNeeded(); await shot('source-drift-1280');
|
|
mark('Changed executable entry still reports drift and can be disabled without running the changed program');
|
|
assert.deepEqual(errors, []); result = { success: true, actualElectron: desktop, appRoot, checks, screenshots, pageErrors: errors, viewportWidths: [1280, 390], localModelFixtureRequests: model.requests.length, actualMcpCalls: 2, realModelRequests: 0, productionWrites: 0 };
|
|
} catch (error) { if (page) await shot('failure').catch(() => {}); result = { success: false, actualElectron: desktop, checks, screenshots, pageErrors: errors, error: error.stack, realModelRequests: 0 }; }
|
|
finally {
|
|
if (browser) await browser.close(); if (application) await application.close(); if (controller && await controller.health()) await controller.stop(); await host?.close?.();
|
|
for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
|
}
|
|
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;
|