Files
Agent-Workbench/scripts/verify-model-protocols-ui.mjs
2026-11-06 14:08:53 +08:00

161 lines
18 KiB
JavaScript

import { openWorkbenchTool } from './ui-navigation.mjs';
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import https from 'node:https';
import assert from 'node:assert/strict';
import { createHash, randomUUID, X509Certificate } from 'node:crypto';
import { chromium, expect } from 'playwright/test';
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
if (process.argv.includes('--help')) { console.log('node scripts/verify-model-protocols-ui.mjs [--app-root <absolute-package>] — owned HTTP Responses/SSE, connection persistence, scoped detection, pool rejection, original stream download and desktop/mobile stop. Explicit package uses contained Node/server. No real model calls or OS trust changes.'); process.exit(0); }
const args = process.argv.slice(2); assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
const appRoot = fs.realpathSync.native(args.length ? args[1] : process.cwd()), nodePath = args.length ? path.join(appRoot, 'runtime/node.exe') : process.execPath;
const directory = path.resolve('.qa', 'model-protocols-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
const disposers = [], t = { after: action => disposers.push(action) }, f = fixture(t); f.store.close();
const requests = [], sockets = new Set(), held = new Set(); let mode = 'success', closedStreams = 0;
const item = text => ({ type: 'message', id: 'msg_owned', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text }] });
const responseBody = output => ({ id: 'resp_owned', status: 'completed', output, usage: { input_tokens: 60, output_tokens: 20, total_tokens: 80 } });
const modelServer = http.createServer(async (request, response) => {
try {
const parts = []; for await (const part of request) parts.push(part);
const body = JSON.parse(Buffer.concat(parts)); requests.push({ route: request.url, body });
assert.equal(request.url, '/v1/responses');
const prompt = body.input.findLast(value => value.role === 'user')?.content ?? '';
const probe = prompt === 'Reply OK.' || prompt.startsWith('Call workbench_capability_echo');
const nonce = prompt.match(/nonce ([a-f0-9-]+)/)?.[1];
const tool = { type: 'function_call', id: 'fc_owned', call_id: 'owned_call', name: nonce ? 'workbench_capability_echo' : 'list_files', arguments: JSON.stringify(nonce ? { nonce } : { path: '', depth: 1 }), status: 'completed' };
const output = probe ? nonce ? [tool] : [item('OK')] : body.input.some(value => value.type === 'function_call_output') ? [item('项目读取完成。')] : [item('正在核对当前项目。'), tool];
if (!body.stream) { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify(responseBody(output))); return; }
response.writeHead(200, { 'Content-Type': 'text/event-stream' }); let sequence = 0;
const send = (type, data) => response.write('data: ' + JSON.stringify({ type, sequence_number: sequence++, ...data }) + '\r\n\r\n');
send('response.created', { response: { id: 'resp_owned' } });
if (!probe && mode !== 'success') {
send('response.output_text.delta', { delta: '正在核对当前项目,尚未执行工具。' });
send('response.output_item.added', { item: { ...tool, arguments: '', status: 'in_progress' }, output_index: 1 });
send('response.function_call_arguments.delta', { item_id: tool.id, delta: '{"path":' });
if (mode === 'truncate') response.end();
else { held.add(response); response.on('close', () => { held.delete(response); closedStreams++; }); }
return;
}
for (const value of output) {
if (value.type === 'message') send('response.output_text.delta', { delta: value.content[0].text });
else { send('response.output_item.added', { item: { ...value, arguments: '', status: 'in_progress' }, output_index: 1 }); send('response.function_call_arguments.delta', { item_id: value.id, delta: value.arguments }); send('response.function_call_arguments.done', { item_id: value.id, arguments: value.arguments }); }
}
send('response.completed', { response: responseBody(output) }); response.end();
} catch { response.destroy(); }
});
modelServer.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
await new Promise(resolve => modelServer.listen(0, '127.0.0.1', resolve));
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath }), browser = await chromium.launch({ channel: 'msedge', headless: true });
assert.equal(fs.realpathSync.native(host.bootstrap.runtimeRoot), appRoot); assert.notEqual(new URL(host.url).port, '4318');
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }), errors = [], failures = [], passed = [], screenshots = [];
page.on('pageerror', error => errors.push(error.message)); page.on('requestfailed', request => failures.push(request.url() + ': ' + request.failure()?.errorText));
const mark = text => { passed.push(text); console.log('PASS ' + text); };
async function shot(target, name) { const file = path.join(directory, name + '.png'); await target.screenshot({ path: file, fullPage: false }); screenshots.push(file); assert.ok(await target.locator('body').evaluate(node => node.scrollWidth <= innerWidth), name + ' page overflow'); }
const getSession = () => host.api('/sessions/' + f.session.id);
let mobile, relay, cookie = '', gateway;
try {
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
await page.goto(host.url); await page.waitForLoadState('networkidle');
await openWorkbenchTool(page, '打开模型中心');
const dialog = page.getByRole('dialog', { name: '模型中心', exact: true });
await dialog.getByRole('button', { name: '手动填写(兼容)', exact: true }).click();
const form = dialog.getByRole('form', { name: '编辑模型连接', exact: true });
await form.getByLabel('连接名称', { exact: true }).fill('验收 Responses');
await form.getByLabel('接口协议', { exact: true }).selectOption('responses');
await expect(form.getByLabel('使用流式输出', { exact: true })).toBeDisabled();
await form.getByLabel('流式能力', { exact: true }).selectOption('supported'); await form.getByLabel('使用流式输出', { exact: true }).check();
await form.getByLabel('接口根地址', { exact: true }).fill('http://127.0.0.1:' + modelServer.address().port + '/v1');
await form.getByLabel('请求模型名称', { exact: true }).fill('owned-responses'); await form.getByLabel('服务需要 API Key', { exact: true }).uncheck();
await shot(page, 'desktop-editor'); await page.setViewportSize({ width: 390, height: 844 }); await form.scrollIntoViewIfNeeded(); await shot(page, 'narrow-editor');
await form.getByRole('button', { name: '保存模型连接', exact: true }).click(); await expect(form).toHaveCount(0);
await page.reload(); await openWorkbenchTool(page, '打开模型中心');
const card = dialog.locator('.mc-connection').filter({ hasText: '验收 Responses' }); await expect(card).toContainText('Responses · 流式开启');
let center = await host.api('/model-center'), connection = center.connections.find(value => value.name === '验收 Responses');
assert.equal(connection.protocol, 'responses'); assert.equal(connection.streaming, true);
mark('actual form saves Responses and declared stream, refresh preserves both and narrow editor remains operable');
await page.setViewportSize({ width: 1440, height: 1000 });
for (const [scope, label] of [['text', '文本'], ['tools', '工具'], ['streaming', '流式']]) {
await card.getByLabel('验收 Responses检测范围', { exact: true }).selectOption(scope);
await card.getByRole('button', { name: '验证响应(调用模型)', exact: true }).click();
await expect(dialog.locator('.mc-notice')).toContainText(label + '检测通过');
await expect(card.locator('.mc-capabilities')).toContainText(label + ':已验证');
if (scope === 'text') await expect(card.locator('.mc-capabilities')).toContainText('工具:声明支持');
}
await shot(page, 'desktop-capabilities'); await page.setViewportSize({ width: 390, height: 844 }); await shot(page, 'narrow-capabilities'); await page.setViewportSize({ width: 1440, height: 1000 });
mark('separate text, nonce tool and actual SSE probes bind only their requested capability and own attempts');
const text = await host.api('/model-center/connections', { name: '只支持文本', model: 'text-only', baseUrl: connection.baseUrl, requiresKey: false, capabilityDeclarations: { tools: 'unsupported' } });
const textId = text.connection?.id ?? (await host.api('/model-center')).connections.find(value => value.name === '只支持文本').id;
await page.reload(); await openWorkbenchTool(page, '打开模型中心');
await dialog.getByRole('button', { name: '模型池', exact: true }).click(); await dialog.getByRole('button', { name: '新建模型池', exact: true }).click();
const poolForm = dialog.getByRole('form', { name: '编辑模型池', exact: true }); await poolForm.getByLabel('模型池名称', { exact: true }).fill('按能力选择的验收池');
for (const name of ['验收 Responses', '只支持文本']) await poolForm.locator('.mc-member-choice').filter({ hasText: name }).locator('input[type=checkbox]').check();
await poolForm.getByRole('button', { name: '保存模型池', exact: true }).click(); await dialog.locator('.mc-pool').filter({ hasText: '按能力选择的验收池' }).getByRole('button', { name: '使用此模型池', exact: true }).click();
await page.keyboard.press('Escape');
await expect(page.locator('.model-button')).toContainText('按能力选择的验收池');
await page.reload(); await expect(page.locator('.model-button')).toContainText('按能力选择的验收池');
assert.equal((await host.api('/bootstrap')).config.model, '', 'The unselected legacy model remains untouched.');
await page.getByLabel('任务内容', { exact: true }).fill('受控完整流程:读取项目后结束。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
await until(async () => (await getSession()).status === 'completed', 15000);
const completed = await getSession(); assert.equal(completed.events.filter(value => value.tool === 'list_files' && value.phase === 'completed').length, 1);
center = await host.api('/model-center'); const attempt = center.attempts.find(value => value.runId === completed.runId);
assert.equal(attempt.protocol, 'responses'); assert.ok(attempt.rejections.some(value => value.connectionId === textId && value.capability === 'tools')); assert.ok(requests.every(value => value.body.model !== 'text-only'));
await openWorkbenchTool(page, '打开模型中心'); await dialog.getByRole('button', { name: '请求账本', exact: true }).click();
await expect(dialog.locator('.mc-route-rejections').first()).toContainText('只支持文本');
const downloadEvent = page.waitForEvent('download'); await dialog.getByRole('button', { name: '下载已保存的响应', exact: true }).first().click();
const download = await downloadEvent, saved = path.join(directory, 'original-stream.sse'); await download.saveAs(saved);
assert.equal(createHash('sha256').update(fs.readFileSync(saved)).digest('hex'), attempt.responseSha256); await shot(page, 'desktop-ledger');
mark('actual pooled task excludes text-only before dispatch, executes original tool once and downloads SHA-matched original SSE');
await page.keyboard.press('Escape');
const initial = await host.api('/companion'); assert.ok(initial.addresses.length);
const reservation = http.createServer(); await new Promise(resolve => reservation.listen(0, initial.addresses[0].address, resolve)); const port = reservation.address().port; await new Promise(resolve => reservation.close(resolve));
await host.api('/companion/start', { address: initial.addresses[0].address, port }); gateway = await host.api('/companion');
const cert = await fetch(host.url + '/api/companion/certificate', { headers: { 'X-Workbench-Token': host.bootstrap.token } }); assert.ok(cert.ok);
const ca = new X509Certificate(Buffer.from(await cert.arrayBuffer())).toString(), target = new URL(gateway.origin);
const ticket = await host.api('/companion/pair', { projectIds: [f.project.id], permissions: { observe: true, approve: false, stop: true }, expiresInHours: 1 });
relay = http.createServer((request, response) => {
const forwarded = https.request(new URL(request.url, target), { method: request.method, ca, servername: '', headers: { Host: target.host, Origin: target.origin, Connection: 'close', ...(cookie ? { Cookie: cookie } : {}), ...(request.headers['content-type'] ? { 'Content-Type': request.headers['content-type'] } : {}) } }, incoming => {
if (incoming.headers['set-cookie']?.length) cookie = incoming.headers['set-cookie'][0].split(';')[0]; const headers = { ...incoming.headers }; delete headers['set-cookie']; response.writeHead(incoming.statusCode, headers); incoming.pipe(response);
}); forwarded.on('error', () => response.destroy()); request.pipe(forwarded);
});
await new Promise(resolve => relay.listen(0, '127.0.0.1', resolve));
mobile = await browser.newPage({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true }); mobile.on('pageerror', error => errors.push(error.message));
await mobile.goto('http://127.0.0.1:' + relay.address().port + '/#pair=' + new URLSearchParams(new URL(ticket.url).hash.slice(1)).get('pair'));
await mobile.getByLabel('设备名称', { exact: true }).fill('协议验收观察窗口'); await mobile.getByRole('button', { name: '连接这台电脑', exact: true }).click(); await expect(mobile.getByRole('heading', { name: '项目任务', exact: true })).toBeVisible();
mode = 'hold'; const count = requests.length;
await page.getByLabel('任务内容', { exact: true }).fill('受控停止:保留片段。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
await expect(page.getByLabel('模型输出片段', { exact: true }).last()).toContainText('正在核对当前项目');
const active = await getSession(); assert.equal(active.events.filter(value => value.kind === 'tool' && value.runId === active.runId).length, 0);
await shot(page, 'desktop-streaming'); await page.setViewportSize({ width: 390, height: 844 }); await shot(page, 'narrow-streaming');
await mobile.getByRole('button', { name: '刷新任务', exact: true }).click(); await mobile.locator('.mobile-task-row').first().click();
await expect(mobile.locator('.mobile-stream')).toContainText('正在核对当前项目'); await mobile.locator('.mobile-stream').scrollIntoViewIfNeeded(); await shot(mobile, 'mobile-streaming');
await mobile.getByRole('button', { name: '停止原任务', exact: true }).click(); await mobile.getByRole('dialog').getByRole('button', { name: '确认', exact: true }).click();
await until(async () => (await getSession()).status === 'paused' && closedStreams === 1, 15000);
await expect(mobile.locator('.mobile-stream')).toContainText('输出未完成');
assert.equal(requests.length, count + 1); await mobile.locator('.mobile-stream').scrollIntoViewIfNeeded(); await shot(mobile, 'mobile-stopped');
await expect(page.getByLabel('模型输出片段', { exact: true }).last()).toContainText('输出未完成'); await shot(page, 'narrow-stopped');
mark('desktop and actual paired gateway show same bounded stream; mobile stops original HTTP with zero fragment tool dispatch or replay');
await page.setViewportSize({ width: 1440, height: 1000 }); mode = 'truncate';
await page.getByLabel('任务内容', { exact: true }).fill('受控断流:半截参数不得执行。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
await until(async () => (await getSession()).status === 'failed', 15000);
const broken = await getSession(), latest = (await host.api('/model-center')).attempts.find(value => value.runId === broken.runId);
assert.equal(latest.status, 'unknown'); assert.equal(latest.stream.terminal, false); assert.equal(requests.length, count + 2);
await openWorkbenchTool(page, '打开模型中心'); await dialog.getByRole('button', { name: '请求账本', exact: true }).click();
await expect(dialog.locator('.mc-attempts article').first()).toContainText('结果待核对'); await expect(dialog.locator('.mc-attempts article').first()).toContainText('响应未完整'); await shot(page, 'desktop-incomplete');
await page.setViewportSize({ width: 390, height: 844 }); await shot(page, 'narrow-incomplete');
mark('truncated output keeps explicit unknown ledger, partial evidence and no automatic failover or incomplete tool execution');
assert.deepEqual(errors, []); assert.deepEqual(failures, []);
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, version: host.bootstrap.version, appRoot, nodePath, containedRuntime: args.length > 0, nodeSha256: createHash('sha256').update(fs.readFileSync(nodePath)).digest('hex'), passed, screenshots, pageErrors: errors, requestFailures: failures, fixtureModelRequests: requests.length, realModelRequests: 0, osTrustChanged: false, nativeIPhone: 'not exercised; paired Edge viewport uses CA-verifying owned relay', taskId: completed.taskId, runIds: [completed.runId, active.runId, broken.runId] }, null, 2));
} catch (error) {
await shot(page, 'failure-desktop').catch(() => {}); if (mobile) await shot(mobile, 'failure-mobile').catch(() => {});
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.stack, passed, screenshots, pageErrors: errors, requestFailures: failures }, null, 2)); throw error;
} finally {
console.log('EVIDENCE ' + directory); await browser.close();
if (relay) await new Promise(resolve => { relay.closeAllConnections(); relay.close(resolve); });
if (gateway) await host.api('/companion/close', {});
for (const response of held) response.destroy(); for (const socket of sockets) socket.destroy(); await new Promise(resolve => modelServer.close(resolve));
for (const dispose of disposers.reverse()) await dispose();
}