124 lines
15 KiB
JavaScript
124 lines
15 KiB
JavaScript
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import http from 'node:http';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, expect } from 'playwright/test';
|
|
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
|
|
|
if (process.argv.includes('--help')) { console.log('node scripts/verify-anthropic-messages-ui.mjs [--app-root ABS] — isolated Messages editor, actual JSON/SSE, tool roundtrip, identity mismatch, URLs, raw errors, reload and 390px; synthetic credentials only.'); process.exit(0); }
|
|
const args = process.argv.slice(2); assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
|
const appRoot = fs.realpathSync.native(args[1] ?? process.cwd());
|
|
const directory = path.resolve('.qa', 'messages-ui-m-' + randomUUID()); fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory);
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t); f.store.close();
|
|
const requests = [], sockets = new Set(), checks = [], screenshots = [], errors = [];
|
|
let mode = 'ok';
|
|
const msg = (content, stop_reason) => ({ id: 'msg_owned_ui', type: 'message', role: 'assistant', model: mode === 'mismatch' ? 'gpt-owned-other' : 'claude-owned-ui', content, stop_reason,
|
|
usage: { input_tokens: 40, cache_creation_input_tokens: 10, cache_read_input_tokens: 20, output_tokens: 8 } });
|
|
const modelServer = http.createServer(async (req, res) => {
|
|
try {
|
|
const parts = []; for await (const chunk of req) parts.push(chunk);
|
|
const body = JSON.parse(Buffer.concat(parts)), raw = Buffer.concat(parts);
|
|
requests.push({ route: req.url, body, auth: req.headers.authorization === 'Bearer owned-ui-key', keyHeaderPresent: req.headers['x-api-key'] !== undefined, version: req.headers['anthropic-version'], sha256: createHash('sha256').update(raw).digest('hex') });
|
|
assert.equal(req.url, '/v1/messages'); assert.equal(req.headers.authorization, 'Bearer owned-ui-key'); assert.equal(req.headers['x-api-key'], undefined); assert.equal(req.headers['anthropic-version'], '2023-06-01');
|
|
if (mode === 'error') { res.writeHead(429, { 'Content-Type': 'application/json', 'request-id': 'req_owned_limit', 'Retry-After': '1' }); res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'owned Messages limit' } })); return; }
|
|
const prompt = body.messages.filter(m => m.role === 'user').flatMap(m => m.content).filter(b => b.type === 'text').map(b => b.text).join('\n');
|
|
const nonce = prompt.match(/nonce ([a-f0-9-]+)/)?.[1], probe = prompt === 'Reply OK.' || !!nonce;
|
|
const completedTool = body.messages.some(m => m.content.some(b => b.type === 'tool_result'));
|
|
const tool = { type: 'tool_use', id: 'call_owned_ui', name: nonce ? 'workbench_capability_echo' : 'list_files', input: nonce ? { nonce } : { path: '', depth: 1 } };
|
|
const content = mode === 'mismatch' || probe && !nonce || completedTool ? [{ type: 'text', text: '已核对。' }] : [tool];
|
|
const final = msg(content, content[0].type === 'tool_use' ? 'tool_use' : 'end_turn');
|
|
if (!body.stream) { res.writeHead(200, { 'Content-Type': 'application/json', 'request-id': 'req_owned_native' }); res.end(JSON.stringify(final)); return; }
|
|
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'request-id': 'req_owned_native' });
|
|
const send = (type, fields = {}) => res.write('event: ' + type + '\ndata: ' + JSON.stringify({ type, ...fields }) + '\n\n');
|
|
send('message_start', { message: { ...final, content: [], stop_reason: null, usage: { ...final.usage, output_tokens: 1 } } });
|
|
content.forEach((block, index) => {
|
|
send('content_block_start', { index, content_block: block.type === 'text' ? { ...block, text: '' } : { ...block, input: {} } });
|
|
send('content_block_delta', { index, delta: block.type === 'text' ? { type: 'text_delta', text: block.text } : { type: 'input_json_delta', partial_json: JSON.stringify(block.input) } });
|
|
send('content_block_stop', { index });
|
|
});
|
|
send('message_delta', { delta: { stop_reason: final.stop_reason, stop_sequence: null }, usage: { output_tokens: final.usage.output_tokens } }); send('message_stop'); res.end();
|
|
} catch (cause) { errors.push('owned upstream: ' + cause.message); res.destroy(); }
|
|
});
|
|
modelServer.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
|
|
await new Promise(resolve => modelServer.listen(0, '127.0.0.1', resolve));
|
|
const baseUrl = 'http://127.0.0.1:' + modelServer.address().port;
|
|
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath: args.length ? path.join(appRoot, 'runtime/node.exe') : process.execPath });
|
|
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
|
page.on('pageerror', cause => errors.push(cause.message));
|
|
const mark = message => { checks.push(message); console.log('PASS ' + message); };
|
|
const dialog = () => page.getByRole('dialog', { name: '模型中心', exact: true });
|
|
async function open() { if (!await dialog().isVisible()) await openWorkbenchTool(page, '打开模型中心'); return dialog(); }
|
|
async function shot(name) {
|
|
const file = path.join(directory, name + '.png'); await page.screenshot({ path: file }); screenshots.push(file);
|
|
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= innerWidth), 'page overflow: ' + name);
|
|
assert.ok(await dialog().evaluate(node => node.scrollWidth <= node.clientWidth + 1), 'dialog overflow: ' + name);
|
|
}
|
|
let result;
|
|
try {
|
|
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
|
await page.goto(host.url); await page.waitForLoadState('networkidle'); await open();
|
|
await dialog().getByRole('button', { name: '手动填写(兼容)', exact: true }).click(); const form = dialog().getByRole('form', { name: '编辑模型连接', exact: true });
|
|
await form.getByLabel('连接名称', { exact: true }).fill('本机 Messages 验收'); await form.getByLabel('接口协议', { exact: true }).selectOption('anthropic-messages');
|
|
await form.getByLabel('接口根地址', { exact: true }).fill(baseUrl); await form.getByLabel('请求模型名称', { exact: true }).fill('claude-owned-ui');
|
|
await form.getByLabel('临时 API Key', { exact: true }).fill('owned-ui-key'); await form.getByLabel('Messages 鉴权', { exact: true }).selectOption('bearer');
|
|
await form.getByLabel('型号核对', { exact: true }).selectOption('require-match'); await form.getByLabel('流式能力', { exact: true }).selectOption('supported'); await form.getByLabel('使用流式输出', { exact: true }).check();
|
|
await expect(form.getByLabel('请求地址预览', { exact: true })).toContainText(baseUrl + '/v1/messages'); await shot('desktop-editor');
|
|
await page.setViewportSize({ width: 390, height: 844 }); await form.scrollIntoViewIfNeeded(); await shot('narrow-editor');
|
|
await form.getByLabel('接口根地址', { exact: true }).fill(baseUrl + '/v1/messages'); await form.getByRole('button', { name: '保存模型连接', exact: true }).click();
|
|
await expect(dialog().locator('.mc-notice')).toContainText('根地址'); await expect(form.getByLabel('连接名称', { exact: true })).toHaveValue('本机 Messages 验收');
|
|
await form.getByLabel('接口根地址', { exact: true }).fill(baseUrl + '/v1'); await form.getByRole('button', { name: '保存模型连接', exact: true }).click(); await expect(form).toHaveCount(0);
|
|
assert.equal(requests.length, 0); mark('root and /v1 preview agree, full endpoint rejects with input retained; save makes no model request');
|
|
await page.reload(); await open(); const card = dialog().locator('.mc-connection').filter({ hasText: '本机 Messages 验收' });
|
|
await expect(card).toContainText('Anthropic Messages · 流式开启'); await expect(card).toContainText('Bearer'); await expect(card).toContainText(baseUrl + '/v1/messages');
|
|
let center = await host.api('/model-center'), connection = center.connections.find(c => c.name === '本机 Messages 验收'); assert.equal(connection.messagesAuth, 'bearer');
|
|
await card.getByRole('button', { name: '用于新任务', exact: true }).click(); assert.equal(requests.length, 0); await page.reload(); await open();
|
|
await expect(dialog().getByLabel('新任务默认模型', { exact: true })).toHaveValue('connection:' + connection.id); await shot('narrow-saved');
|
|
mark('saved protocol, auth and selected connection survive reload and remain operable at 390px');
|
|
await page.setViewportSize({ width: 1440, height: 1000 }); await card.getByRole('button', { name: '推理与能力', exact: true }).click();
|
|
const native = dialog().getByRole('form', { name: '原生请求参数', exact: true }); await expect(native.getByLabel('输出上限字段')).toHaveValue('max_tokens');
|
|
await expect(native.getByLabel('输出上限字段').locator('option')).toHaveCount(1); await native.getByLabel('原生参数 JSON').fill('{"reasoning_effort":"high"}');
|
|
await native.getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(native.getByRole('alert')).toContainText('尚未接入');
|
|
await native.getByLabel('原生参数 JSON').fill('{"temperature":0.3,"top_k":12}'); await native.getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(native).toHaveCount(0);
|
|
assert.equal(requests.length, 0); mark('native settings reject other-protocol fields and keep a single hard max_tokens output field');
|
|
for (const [scope, title] of [['text', '文本'], ['tools', '工具'], ['streaming', '流式']]) {
|
|
await card.getByLabel('本机 Messages 验收检测范围').selectOption(scope); await card.getByRole('button', { name: '验证响应(调用模型)', exact: true }).click();
|
|
await expect(dialog().locator('.mc-notice')).toContainText(title + '检测通过');
|
|
}
|
|
assert.equal(requests.length, 3); assert.ok(requests.every(r => r.body.top_k === 12 && r.body.temperature === 0.3)); mark('actual local JSON text/tool and native SSE capability probes use the saved protocol and auth');
|
|
await dialog().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
|
await host.api('/sessions/' + f.session.id + '/messages', { content: '读取本项目文件一次后结束。' });
|
|
await until(async () => !['running', 'queued'].includes((await host.api('/sessions/' + f.session.id)).status), 20000);
|
|
const session = await host.api('/sessions/' + f.session.id); assert.equal(session.status, 'completed');
|
|
assert.equal(requests.length, 5); assert.ok(requests[4].body.messages.some(m => m.content.some(b => b.type === 'tool_result' && b.tool_use_id === 'call_owned_ui')));
|
|
await open(); await dialog().getByRole('button', { name: '请求账本', exact: true }).click();
|
|
await expect(dialog().locator('.mc-attempts')).toContainText('请求与返回型号一致'); await expect(dialog().locator('.mc-attempts')).toContainText(baseUrl + '/v1/messages');
|
|
await expect(dialog().locator('.mc-attempts')).toContainText('70 输入 / 8 输出 tokens'); await dialog().locator('.mc-attempts article').first().scrollIntoViewIfNeeded(); await shot('desktop-ledger');
|
|
await page.setViewportSize({ width: 390, height: 844 }); await dialog().locator('.mc-attempts article').first().scrollIntoViewIfNeeded(); await shot('narrow-ledger'); mark('compiled native tool roundtrip completes with original call ID, cache-inclusive usage and observed request/response URLs');
|
|
await dialog().getByRole('button', { name: '关闭窗口', exact: true }).click(); mode = 'mismatch';
|
|
const second = await host.api('/sessions', { projectId: f.project.id }); await host.api('/sessions/' + second.id + '/messages', { content: '有界身份异常夹具。' });
|
|
await until(async () => !['running', 'queued'].includes((await host.api('/sessions/' + second.id)).status), 20000);
|
|
center = await host.api('/model-center'); assert.equal(center.attempts[0].modelIdentity.status, 'mismatch'); assert.equal(requests.length, 6);
|
|
await open(); await dialog().getByRole('button', { name: '请求账本', exact: true }).click(); await expect(dialog().locator('.mc-attempts')).toContainText('请求与返回型号不一致'); await shot('narrow-mismatch');
|
|
mark('strict mismatch is visible with original GPT identifier and one request, no retry or tool dispatch');
|
|
// A separate owned connection avoids bypassing the intentionally blocked identity route.
|
|
mode = 'error'; const original = center.connections.find(c => c.id === connection.id);
|
|
connection = await host.api('/model-center/connections', { ...original, id: undefined, name: '本机 Messages 错误验收', apiKey: 'owned-ui-key' });
|
|
// The API returns the authoritative center; resolve by its explicit owned name.
|
|
center = await host.api('/model-center'); connection = center.connections.find(c => c.name === '本机 Messages 错误验收');
|
|
await expect(dialog().getByRole('button', { name: '刷新模型中心', exact: true })).toBeEnabled(); await dialog().getByRole('button', { name: '刷新模型中心', exact: true }).click();
|
|
await dialog().getByRole('button', { name: '模型连接', exact: true }).click(); const errorCard = dialog().locator('.mc-connection').filter({ hasText: '本机 Messages 错误验收' });
|
|
await errorCard.getByRole('button', { name: '验证响应(调用模型)', exact: true }).click(); await expect(dialog().locator('.mc-notice')).toContainText('模型服务请求达到限额');
|
|
await dialog().locator('.mc-notice summary').click(); await expect(dialog().getByLabel('上游错误原文')).toContainText('owned Messages limit');
|
|
await expect(dialog().locator('.mc-notice')).toContainText('req_owned_limit'); await expect(dialog().locator('.mc-notice')).toContainText(baseUrl + '/v1/messages'); await shot('narrow-error');
|
|
assert.equal(requests.length, 7); mark('429 preserves native type, raw message, request ID, URL and inputs without automatic replay');
|
|
assert.deepEqual(errors, []);
|
|
result = { success: true, appRoot, testedVersion: (await host.api('/health')).version, directory, checks, screenshots, errors, requests, realModelRequests: 0,
|
|
scope: 'Compiled UI and actual owned localhost API; no configured-model capability or model-weight claim.' };
|
|
} catch (cause) { result = { success: false, directory, checks, screenshots, errors, error: cause.stack, requests, realModelRequests: 0 }; await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {}); process.exitCode = 1; }
|
|
finally {
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify({ success: result.success, checks: checks.length, directory, error: result.error }));
|
|
await browser.close(); for (const dispose of disposers.reverse()) await dispose(); for (const socket of sockets) socket.destroy(); await new Promise(resolve => modelServer.close(resolve));
|
|
}
|