152 lines
18 KiB
JavaScript
152 lines
18 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import net from 'node:net';
|
|
import http from 'node:http';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, _electron } from 'playwright';
|
|
import { expect } from 'playwright/test';
|
|
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
|
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
|
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
|
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
|
|
|
const desktop = process.argv.includes('--desktop'), appRoot = path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT || '.');
|
|
assert.ok(process.argv.slice(2).every(value => value === '--desktop'));
|
|
const directory = path.resolve('.qa/reasoning-controls', (desktop ? 'catalog-electron-' : 'catalog-web-') + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
|
console.log('EVIDENCE ' + directory);
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t); f.store.close();
|
|
const requests = [], checks = [], screenshots = [], errors = []; let scenario = 'normal', release;
|
|
const models = Array.from({ length: 132 }, (_, i) => ({ id: 'owned-model-' + String(i).padStart(3, '0'), owned_by: '自有目录夹具',
|
|
context_length: 64000, max_output_tokens: 8192, capabilities: { text: true, tools: i !== 1, streaming: true, image_input: { supported: i === 2 },
|
|
effort: { supported: true, low: { supported: true }, high: { supported: true } }, thinking: { supported: true, types: { adaptive: { supported: true }, disabled: { supported: true }, enabled: { supported: false } } } } }));
|
|
const server = http.createServer(async (req, res) => {
|
|
const parts = []; for await (const part of req) parts.push(part);
|
|
const body = parts.length ? JSON.parse(Buffer.concat(parts)) : null;
|
|
requests.push({ method: req.method, route: req.url, body, authorizationPresent: !!req.headers.authorization, apiKeyHeaderPresent: !!req.headers['x-api-key'], anthropicVersion: req.headers['anthropic-version'] });
|
|
if (scenario === 'wait') await new Promise(resolve => { release = resolve; });
|
|
if (scenario === 'error') { res.writeHead(503, { 'Content-Type': 'application/json', 'x-request-id': 'owned-catalog-503' }); res.end(JSON.stringify({ error: { message: '自有目录暂不可用', code: 'owned_unavailable' } })); return; }
|
|
const input = new URL(req.url, 'http://owned.test');
|
|
const value = scenario === 'empty' ? { data: [] } : input.searchParams.has('after_id') ? { data: models.slice(100), has_more: false }
|
|
: { data: models.slice(0, 100), has_more: true, last_id: models[99].id };
|
|
res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(value));
|
|
});
|
|
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
|
let browser, application, controller, host, page, window, result;
|
|
const mark = name => { checks.push(name); console.log('PASS ' + name); };
|
|
const dialog = () => page.getByRole('dialog', { name: '模型中心', exact: true });
|
|
const catalog = () => dialog().getByRole('region', { name: '上游模型目录', exact: true });
|
|
async function resize(width, height) {
|
|
if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); }
|
|
else await page.setViewportSize({ width, height });
|
|
}
|
|
async function shot(name) {
|
|
const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' });
|
|
screenshots.push({ path: file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') });
|
|
fs.writeFileSync(path.join(directory, name + '.txt'), await page.locator('body').innerText());
|
|
}
|
|
async function open() { if (!await dialog().count()) await page.getByRole('button', { name: '打开模型中心', exact: true }).click(); }
|
|
async function add() { await open(); await dialog().getByRole('button', { name: '从上游添加模型', exact: true }).click(); await catalog().waitFor(); }
|
|
async function fetchModels() { await catalog().getByRole('button', { name: /获取上游模型|重新获取模型目录/ }).click(); await expect(catalog().getByLabel('上游模型列表', { exact: true })).toBeVisible(); }
|
|
try {
|
|
if (desktop) {
|
|
const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve));
|
|
assert.notEqual(port, 4318); controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start();
|
|
host = { api: (route, input) => controller.api(route, input ? { method: 'POST', input } : {}), bootstrap: await controller.api('/bootstrap') };
|
|
const executable = desktopExecutableIdentity(appRoot).path;
|
|
application = await _electron.launch({ executablePath: executable, args: [...(executable.toLowerCase().endsWith('agentworkbench.exe') ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]),
|
|
'--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port)], env: projectEnvironment({ AGENT_WORKBENCH_QA: '1', AGENT_WORKBENCH_NODE: process.execPath }) });
|
|
page = await application.firstWindow(); window = await application.browserWindow(page); await page.waitForLoadState(); await resize(1440, 1080);
|
|
} else {
|
|
host = await launchWorkbench(t, f.data, {}, { appRoot }); browser = await chromium.launch({ channel: 'msedge', headless: true });
|
|
page = await browser.newPage({ viewport: { width: 1440, height: 1080 } }); await page.goto(host.url);
|
|
}
|
|
page.on('pageerror', error => errors.push(error.message)); await page.locator('.connection.connected').waitFor();
|
|
await add(); await expect(catalog().getByRole('textbox', { name: '请求模型名称', exact: true })).toHaveCount(0);
|
|
await catalog().getByLabel('上游名称', { exact: true }).fill('自有多模型上游');
|
|
await catalog().getByLabel('目录接口根地址', { exact: true }).fill('http://127.0.0.1:' + server.address().port + '/v1');
|
|
await catalog().getByLabel('目录 API Key', { exact: true }).fill('owned-catalog-ui-key'); await fetchModels();
|
|
assert.equal(requests.length, 2); assert.ok(requests.every(value => value.method === 'GET' && value.body === null));
|
|
await expect(catalog().getByLabel('上游模型列表', { exact: true }).getByRole('checkbox')).toHaveCount(132);
|
|
await catalog().getByLabel('搜索上游模型', { exact: true }).fill('owned-model-00');
|
|
await catalog().getByLabel('选择模型 owned-model-000', { exact: true }).check(); await catalog().getByLabel('选择模型 owned-model-001', { exact: true }).check(); await catalog().getByLabel('选择模型 owned-model-002', { exact: true }).check();
|
|
await catalog().getByLabel('模型使用方式', { exact: true }).selectOption('auto'); await shot('catalog-multiselect-1440');
|
|
await catalog().getByRole('button', { name: '保存所选 3 个模型', exact: true }).click(); await expect(catalog().getByRole('status')).toContainText('已新增 3 个模型');
|
|
let center = await host.api('/model-center'); const members = center.connections.filter(value => value.upstreamModel);
|
|
assert.equal(members.length, 3); assert.equal(members.filter(value => value.credentialSource).length, 2); assert.equal(center.selected.type, 'pool');
|
|
assert.equal(center.pools.find(value => value.id === center.selected.id).members.length, 3); assert.equal(requests.length, 2);
|
|
mark('new upstream needs no manual model; 132 paginated IDs are searchable and selectable; saving creates shared-source members and the existing automatic pool without inference');
|
|
await page.reload(); await open(); await expect(dialog().getByLabel('新任务默认模型', { exact: true })).toHaveValue('pool:' + center.selected.id);
|
|
const child = members.find(value => value.model === 'owned-model-002'); await dialog().getByLabel('新任务默认模型', { exact: true }).selectOption('connection:' + child.id);
|
|
assert.equal((await host.api('/model-center')).selected.id, child.id);
|
|
const card = () => dialog().locator('.mc-connection').filter({ has: page.getByRole('heading', { name: child.name, exact: true }) });
|
|
await expect(card()).toContainText('共用'); await card().getByRole('button', { name: '推理与能力', exact: true }).click();
|
|
const params = () => dialog().getByRole('form', { name: '原生请求参数', exact: true });
|
|
await expect(params().getByLabel('推理强度', { exact: true }).locator('option')).toHaveCount(3);
|
|
await params().getByRole('button', { name: '取消', exact: true }).click();
|
|
mark('reload retains the automatic pool; users can manually choose a member, see its credential source and receive only upstream-declared reasoning strengths');
|
|
await card().getByRole('button', { name: '获取同源模型', exact: true }).click(); await expect(catalog().getByLabel('目录 API Key', { exact: true })).toHaveCount(0); await fetchModels();
|
|
await catalog().getByLabel('搜索上游模型', { exact: true }).fill('owned-model-003'); await catalog().getByLabel('选择模型 owned-model-003', { exact: true }).check();
|
|
await catalog().getByLabel('模型使用方式', { exact: true }).selectOption('manual'); await catalog().getByLabel('目录默认模型', { exact: true }).selectOption('owned-model-003');
|
|
await catalog().getByRole('button', { name: '保存所选 1 个模型', exact: true }).click(); await expect(catalog().getByRole('status')).toContainText('已新增 1 个模型');
|
|
center = await host.api('/model-center'); assert.equal(center.connections.find(value => value.id === center.selected.id).model, 'owned-model-003');
|
|
assert.equal(center.connections.filter(value => value.upstreamModel).length, 4); assert.equal(requests.length, 4);
|
|
mark('an existing member opens its original upstream without exposing or re-entering a key and adds another model with an explicit manual default');
|
|
await catalog().getByRole('button', { name: '关闭目录', exact: true }).click(); await card().getByRole('button', { name: '获取同源模型', exact: true }).click();
|
|
scenario = 'error'; await catalog().getByRole('button', { name: '获取上游模型', exact: true }).click(); await expect(catalog().getByRole('alert')).toContainText('自有目录暂不可用');
|
|
await catalog().getByText('查看上游原文与请求信息', { exact: true }).click(); await expect(catalog()).toContainText('owned-catalog-503'); await shot('catalog-upstream-error');
|
|
assert.equal(requests.length, 5); scenario = 'empty'; await fetchModels(); await expect(catalog()).toContainText('上游返回了空目录');
|
|
assert.equal((await host.api('/model-center')).connections.length, 5);
|
|
scenario = 'wait'; await catalog().getByRole('button', { name: '重新获取模型目录', exact: true }).click(); await until(() => !!release);
|
|
await catalog().getByRole('button', { name: '取消获取', exact: true }).click(); release(); await expect(catalog().getByRole('status')).toContainText('已取消获取');
|
|
await expect(catalog().getByLabel('上游模型列表', { exact: true })).toHaveCount(0); scenario = 'normal'; await fetchModels();
|
|
await expect(catalog().getByLabel('上游模型列表', { exact: true }).getByRole('checkbox')).toHaveCount(132);
|
|
mark('upstream error retains request ID and original reason with one GET; empty directory and cancellation preserve members, and a later explicit refresh recovers');
|
|
await resize(390, 844); await catalog().getByLabel('搜索上游模型', { exact: true }).fill('owned-model-13');
|
|
await catalog().getByRole('button', { name: '选中当前筛选', exact: true }).click(); await catalog().getByLabel('模型使用方式', { exact: true }).selectOption('keep');
|
|
await catalog().getByLabel('上游模型列表', { exact: true }).scrollIntoViewIfNeeded(); await shot('catalog-list-390');
|
|
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), true);
|
|
const save = catalog().getByRole('button', { name: '保存所选 2 个模型', exact: true }); await save.scrollIntoViewIfNeeded(); const bounds = await save.boundingBox(); assert.ok(bounds && bounds.x >= 0 && bounds.x + bounds.width <= 391);
|
|
await save.click(); await expect(catalog().getByRole('status')).toContainText('已新增 2 个模型'); await shot('catalog-saved-390');
|
|
assert.equal((await host.api('/model-center')).connections.filter(value => value.upstreamModel).length, 6);
|
|
mark('390px search, filtered bulk selection, source feedback and save action fit and complete the same real API flow');
|
|
await resize(1440, 1080); await catalog().getByRole('button', { name: '关闭目录', exact: true }).click(); await add();
|
|
await catalog().getByLabel('目录服务品牌', { exact: true }).selectOption('Anthropic');
|
|
await expect(catalog().getByLabel('目录接口协议', { exact: true })).toHaveValue('anthropic-messages');
|
|
await expect(catalog().getByLabel('目录接口根地址', { exact: true })).toHaveValue('https://api.anthropic.com');
|
|
await catalog().getByLabel('上游名称', { exact: true }).fill('自有 Claude 上游');
|
|
await catalog().getByLabel('目录接口根地址', { exact: true }).fill('http://127.0.0.1:' + server.address().port);
|
|
await catalog().getByLabel('目录 API Key', { exact: true }).fill('owned-messages-catalog'); await fetchModels();
|
|
assert.equal(requests.at(-1).route, '/v1/models?after_id=owned-model-099&limit=100'); assert.equal(requests.at(-1).apiKeyHeaderPresent, true); assert.equal(requests.at(-1).authorizationPresent, false);
|
|
await catalog().getByLabel('搜索上游模型', { exact: true }).fill('owned-model-003'); await catalog().getByLabel('选择模型 owned-model-003', { exact: true }).check();
|
|
await catalog().getByRole('button', { name: '保存所选 1 个模型', exact: true }).click(); await expect(catalog().getByRole('status')).toContainText('已新增 1 个模型');
|
|
const claude = (await host.api('/model-center')).connections.find(value => value.name === '自有 Claude 上游 · owned-model-003');
|
|
await catalog().getByRole('button', { name: '关闭目录', exact: true }).click(); await dialog().locator('.mc-connection').filter({ has: page.getByRole('heading', { name: claude.name, exact: true }) }).getByRole('button', { name: '推理与能力', exact: true }).click();
|
|
await expect(params().getByLabel('推理模式', { exact: true }).locator('option[value="enabled"]')).toHaveJSProperty('disabled', true);
|
|
await expect(params().getByLabel('推理模式', { exact: true }).locator('option[value="adaptive"]')).toHaveJSProperty('disabled', false);
|
|
await params().getByRole('button', { name: '取消', exact: true }).click();
|
|
mark('Anthropic brand pairs its endpoint, protocol and x-api-key pagination; upstream thinking modes restrict the actual controls');
|
|
await add(); await catalog().getByLabel('目录服务品牌', { exact: true }).selectOption('DeepSeek');
|
|
await expect(catalog().getByLabel('目录接口协议', { exact: true })).toHaveValue('chat-completions');
|
|
await expect(catalog().getByLabel('目录接口根地址', { exact: true })).toHaveValue('https://api.deepseek.com');
|
|
await catalog().getByLabel('上游名称', { exact: true }).fill('自有 DeepSeek 上游');
|
|
await catalog().getByLabel('目录接口根地址', { exact: true }).fill('http://127.0.0.1:' + server.address().port + '/v1');
|
|
await catalog().getByLabel('目录 API Key', { exact: true }).fill('owned-deepseek-catalog'); await fetchModels();
|
|
assert.equal(requests.at(-1).authorizationPresent, true); assert.equal(requests.at(-1).apiKeyHeaderPresent, false);
|
|
await catalog().getByLabel('搜索上游模型', { exact: true }).fill('owned-model-005'); await catalog().getByLabel('选择模型 owned-model-005', { exact: true }).check();
|
|
await catalog().getByRole('button', { name: '保存所选 1 个模型', exact: true }).click(); await expect(catalog().getByRole('status')).toContainText('已新增 1 个模型');
|
|
mark('DeepSeek brand pairs Chat and bearer discovery with a separate explicitly entered source key and no inference');
|
|
await catalog().getByRole('button', { name: '关闭目录', exact: true }).click(); await dialog().getByRole('button', { name: '手动填写(兼容)', exact: true }).click();
|
|
await expect(dialog().getByRole('form', { name: '编辑模型连接', exact: true }).getByLabel('请求模型名称', { exact: true })).toBeVisible();
|
|
assert.deepEqual(errors, []); assert.ok(requests.every(value => value.method === 'GET')); assert.equal((await host.api('/model-center')).attempts.length, 0);
|
|
mark('manual compatibility entry remains usable; every upstream call was a directory GET with zero model attempts');
|
|
result = { success: true, version: host.bootstrap.version, appRoot, actualElectron: desktop, checks, screenshots, errors, directoryGets: requests.length, realModelRequests: 0, officialServiceTouched: false };
|
|
} catch (error) {
|
|
if (page) await shot('failure').catch(() => {}); result = { success: false, appRoot, actualElectron: desktop, checks, screenshots, errors, error: error.stack, realModelRequests: 0 };
|
|
} finally {
|
|
release?.(); if (application) await application.close(); if (browser) await browser.close(); if (controller && await controller.health()) await controller.stop();
|
|
await new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }); for (const dispose of disposers.reverse()) await dispose();
|
|
fs.writeFileSync(path.join(directory, 'requests.json'), JSON.stringify(requests, null, 2)); fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
|
}
|
|
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;
|