148 lines
16 KiB
JavaScript
148 lines
16 KiB
JavaScript
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import net from 'node:net';
|
|
import http from 'node:http';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, _electron } from 'playwright';
|
|
import { expect } from 'playwright/test';
|
|
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
|
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
|
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
|
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
|
|
|
const desktop = process.argv.includes('--desktop'), appRoot = path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT || '.');
|
|
assert.ok(process.argv.slice(2).every(value => value === '--desktop'));
|
|
const directory = path.resolve('.qa/reasoning-controls', (desktop ? 'electron-' : 'web-') + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
|
console.log('EVIDENCE ' + directory);
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t); f.store.close();
|
|
const requests = [], checks = [], screenshots = [], errors = []; let empty = false;
|
|
const server = http.createServer(async (req, res) => {
|
|
const parts = []; for await (const part of req) parts.push(part); const body = JSON.parse(Buffer.concat(parts)); requests.push({ route: req.url, body });
|
|
const usage = { input_tokens: 338, cache_creation_input_tokens: 331, cache_read_input_tokens: 6023, output_tokens: 0 };
|
|
if (req.url.endsWith('/messages')) {
|
|
const value = { id: 'msg_owned', type: 'message', role: 'assistant', model: body.model, content: empty ? [] : [{ type: 'text', text: 'OK' }], stop_reason: 'end_turn', usage: empty ? usage : { input_tokens: 20, output_tokens: 1 } };
|
|
if (body.stream) {
|
|
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
|
const event = (type, fields = {}) => 'event: ' + type + '\ndata: ' + JSON.stringify({ type, ...fields }) + '\n\n';
|
|
res.end(event('message_start', { message: { ...value, content: [], stop_reason: null } }) + event('message_delta', { delta: { stop_reason: 'end_turn' }, usage }) + event('message_stop'));
|
|
} else { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(value)); }
|
|
} else if (req.url.endsWith('/responses')) {
|
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ id: 'resp_owned', model: body.model, status: 'completed', output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'OK' }] }], usage: { input_tokens: 20, output_tokens: 1 } }));
|
|
} else {
|
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ model: body.model, choices: [{ message: { role: 'assistant', content: 'OK' }, finish_reason: 'stop' }], usage: { prompt_tokens: 20, completion_tokens: 1 } }));
|
|
}
|
|
});
|
|
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
|
let browser, application, controller, host, page, window, result;
|
|
const mark = name => { checks.push(name); console.log('PASS ' + name); };
|
|
const dialog = () => page.getByRole('dialog', { name: '模型中心', exact: true });
|
|
const form = () => page.getByRole('form', { name: '原生请求参数', exact: true });
|
|
async function resize(width, height) {
|
|
if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); }
|
|
else await page.setViewportSize({ width, height });
|
|
}
|
|
async function shot(name) {
|
|
const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' });
|
|
screenshots.push({ path: file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') });
|
|
fs.writeFileSync(path.join(directory, name + '.txt'), await page.locator('body').innerText());
|
|
}
|
|
async function edit(name) {
|
|
if (!await dialog().count()) await openWorkbenchTool(page, '打开模型中心');
|
|
await dialog().locator('.mc-connection').filter({ has: page.getByRole('heading', { name, exact: true }) }).getByRole('button', { name: '推理与能力', exact: true }).click();
|
|
await form().waitFor();
|
|
}
|
|
async function save() {
|
|
await form().getByRole('button', { name: '保存原生参数', exact: true }).click();
|
|
await expect(form()).toHaveCount(0); await expect(dialog().getByRole('status')).toContainText('原生参数已保存');
|
|
}
|
|
async function current(id) { return (await host.api('/model-center')).connections.find(c => c.id === id); }
|
|
async function probe(member) { await host.api('/model-center/probe', { id: member.id, scope: 'text' }); return requests.at(-1).body; }
|
|
try {
|
|
if (desktop) {
|
|
const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve));
|
|
assert.notEqual(port, 4318); controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start();
|
|
host = { api: (route, input) => controller.api(route, input ? { method: 'POST', input } : {}), bootstrap: await controller.api('/bootstrap') };
|
|
const executable = desktopExecutableIdentity(appRoot).path;
|
|
application = await _electron.launch({ executablePath: executable, args: [...(executable.toLowerCase().endsWith('agentworkbench.exe') ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]),
|
|
'--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window'], cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
|
page = await application.firstWindow(); window = await application.browserWindow(page); await window.evaluate(win => win.showInactive()); await resize(1440, 1080);
|
|
} else {
|
|
host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318');
|
|
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ viewport: { width: 1440, height: 1080 } }); await page.goto(host.url);
|
|
}
|
|
page.on('pageerror', error => errors.push(error.message));
|
|
const baseUrl = 'http://127.0.0.1:' + server.address().port + '/v1';
|
|
const members = [];
|
|
for (const [name, protocol, model] of [['自定义兼容线路', 'chat-completions', 'owned-custom'], ['Responses 线路', 'responses', 'owned-responses'], ['Claude 线路', 'anthropic-messages', 'owned-claude']]) {
|
|
members.push(await host.api('/model-center/connections', { name, protocol, model, baseUrl, brand: '自定义', requiresKey: false }));
|
|
}
|
|
await page.reload(); await page.locator('.connection.connected').waitFor();
|
|
await edit(members[0].name); await expect(form().getByLabel('推理模式', { exact: true })).toHaveValue('default');
|
|
await form().getByLabel('原生参数 JSON', { exact: true }).fill('{"temperature":0.4,"seed":3}');
|
|
await form().getByLabel('参数配置档', { exact: true }).selectOption('deepseek'); await form().getByLabel('推理模式', { exact: true }).selectOption('enabled');
|
|
await form().getByLabel('推理强度', { exact: true }).selectOption('low'); await save();
|
|
assert.equal(requests.length, 0); await page.reload(); await edit(members[0].name);
|
|
await expect(form().getByLabel('参数配置档', { exact: true })).toHaveValue('deepseek'); await expect(form().getByLabel('推理模式', { exact: true })).toHaveValue('enabled');
|
|
await expect(form().getByLabel('推理强度', { exact: true })).toHaveValue('low');
|
|
await form().getByText('推理设置', { exact: true }).scrollIntoViewIfNeeded(); await shot('deepseek-desktop');
|
|
assert.deepEqual((await current(members[0].id)).nativeParameters, { temperature: 0.4, seed: 3, thinking: { type: 'enabled' }, reasoning_effort: 'low' });
|
|
mark('custom-compatible profile and enabled/low controls save and reload without requests or losing unrelated parameters');
|
|
await form().getByLabel('推理模式', { exact: true }).selectOption('disabled'); await expect(form().getByLabel('推理强度', { exact: true })).toBeDisabled(); await save();
|
|
let wire = await probe(members[0]); assert.deepEqual(wire.thinking, { type: 'disabled' }); assert.equal(wire.reasoning_effort, undefined);
|
|
await edit(members[0].name); await form().getByLabel('推理模式', { exact: true }).selectOption('enabled'); await form().getByLabel('推理强度', { exact: true }).selectOption('max'); await save();
|
|
wire = await probe(members[0]); assert.deepEqual(wire.thinking, { type: 'enabled' }); assert.equal(wire.reasoning_effort, 'max');
|
|
await edit(members[0].name); await form().getByLabel('推理模式', { exact: true }).selectOption('default'); await save();
|
|
wire = await probe(members[0]); assert.equal(wire.thinking, undefined); assert.equal(wire.reasoning_effort, undefined); assert.equal(wire.seed, 3);
|
|
mark('DeepSeek off/on/max/default reach actual HTTP with no stale fields and preserve the original output reservation');
|
|
await edit(members[0].name); await form().getByLabel('原生参数 JSON', { exact: true }).fill('{"reasoning_effort":"high","seed":3}');
|
|
await expect(form().getByLabel('推理强度', { exact: true })).toHaveValue('high');
|
|
await form().getByLabel('原生参数 JSON', { exact: true }).fill('{'); await expect(form().getByLabel('推理模式', { exact: true })).toBeDisabled();
|
|
await form().getByLabel('原生参数 JSON', { exact: true }).fill('{"tools":[]}'); await form().getByRole('button', { name: '保存原生参数', exact: true }).click();
|
|
await expect(form().getByRole('alert')).toContainText('尚未接入原生参数:tools'); assert.equal(requests.length, 3); await shot('invalid-json');
|
|
await form().getByLabel('原生参数 JSON', { exact: true }).fill('{"seed":3}'); await form().getByLabel('参数配置档', { exact: true }).selectOption('openai');
|
|
await form().getByLabel('推理模式', { exact: true }).selectOption('enabled'); await form().getByLabel('推理强度', { exact: true }).selectOption('medium'); await save();
|
|
wire = await probe(members[0]); assert.equal(wire.reasoning_effort, 'medium'); assert.equal(wire.thinking, undefined);
|
|
mark('JSON and controls edit one value; malformed JSON and protected fields fail visibly; changing custom dialect uses the matching wire format');
|
|
await edit(members[1].name); await form().getByLabel('推理强度', { exact: true }).selectOption('max'); await save(); wire = await probe(members[1]);
|
|
assert.deepEqual(wire.reasoning, { effort: 'max' }); assert.equal(wire.reasoning_effort, undefined);
|
|
await edit(members[1].name); await form().getByLabel('推理模式', { exact: true }).selectOption('disabled'); await save(); wire = await probe(members[1]); assert.deepEqual(wire.reasoning, { effort: 'none' });
|
|
mark('Responses strength and disable controls send reasoning.effort using their own protocol');
|
|
await edit(members[2].name); await form().getByLabel('推理模式', { exact: true }).selectOption('adaptive'); await form().getByLabel('推理强度', { exact: true }).selectOption('high'); await save();
|
|
wire = await probe(members[2]); assert.deepEqual(wire.thinking, { type: 'adaptive' }); assert.deepEqual(wire.output_config, { effort: 'high' });
|
|
await edit(members[2].name); await form().getByLabel('推理模式', { exact: true }).selectOption('enabled'); await form().getByLabel('推理预算 tokens', { exact: true }).fill('1024'); await save();
|
|
const manual = await current(members[2].id); assert.deepEqual(manual.nativeParameters.thinking, { type: 'enabled', budget_tokens: 1024 });
|
|
wire = await probe(members[2]); assert.deepEqual(wire.thinking, { type: 'enabled', budget_tokens: 1024 }); assert.equal(wire.max_tokens, 2048);
|
|
await page.reload(); await edit(members[2].name); await expect(form().getByLabel('推理预算 tokens', { exact: true })).toHaveValue('1024');
|
|
await resize(390, 844); await form().getByLabel('推理强度', { exact: true }).scrollIntoViewIfNeeded(); await shot('messages-390');
|
|
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), true);
|
|
await form().getByText('查看接口能力与接入范围', { exact: true }).click(); await form().locator('table').scrollIntoViewIfNeeded(); await shot('features-390');
|
|
await expect(form().locator('table')).toContainText('尚未接入'); await expect(form().locator('table')).toContainText('待确认');
|
|
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), true);
|
|
await form().getByRole('button', { name: '保存原生参数', exact: true }).scrollIntoViewIfNeeded(); const bounds = await form().getByRole('button', { name: '保存原生参数', exact: true }).boundingBox();
|
|
assert.ok(bounds && bounds.x >= 0 && bounds.x + bounds.width <= 391); await save();
|
|
mark('Messages adaptive/effort and manual budget persist; 390px controls, capability scope and save action remain readable and usable');
|
|
await resize(1440, 1080); await page.keyboard.press('Escape');
|
|
await host.api('/model-center/native-parameters', { id: manual.id, revision: (await current(manual.id)).revision, parameterProfile: 'anthropic', nativeParameters: { thinking: { type: 'disabled' } } });
|
|
const selected = await current(manual.id); await host.api('/model-center/connections', { ...selected, streaming: true, capabilityDeclarations: { ...selected.capabilityDeclarations, streaming: 'supported' } });
|
|
await host.api('/model-center/select', { type: 'connection', id: selected.id }); empty = true; await page.reload();
|
|
await page.getByLabel('当前项目').selectOption(f.project.id); await page.getByLabel('任务内容').fill('检查当前工程,保留实际失败。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
|
let failed; await until(async () => { const rows = await host.api('/sessions'); const row = rows.find(s => s.title === '检查当前工程,保留实际失败。'); failed = row && await host.api('/sessions/' + row.id); return failed?.status === 'failed'; }, 15000);
|
|
assert.equal(failed.failure.kind, 'protocol'); assert.match(failed.failure.message, /返回为空/); assert.equal(failed.execution.run.usage.totalTokens, 6692);
|
|
await expect(page.locator('.failure-card').last()).toContainText('返回为空'); await shot('empty-response-desktop'); await resize(390, 844); await page.locator('.failure-card').last().scrollIntoViewIfNeeded(); await shot('empty-response-390');
|
|
assert.equal(failed.execution.run.usage.requests, 1); assert.equal(failed.changes.length, 0); assert.equal(requests.length, 9);
|
|
mark('actual Messages empty SSE is diagnosed explicitly in the task at both widths with 6692/0 use, zero tools and one request');
|
|
assert.deepEqual(errors, []); fs.writeFileSync(path.join(directory, 'requests.json'), JSON.stringify(requests, null, 2));
|
|
result = { success: true, version: host.bootstrap.version, appRoot, actualElectron: desktop, checks, screenshots, errors, syntheticRequests: requests.length, realModelRequests: 0, officialServiceTouched: false };
|
|
} catch (error) {
|
|
if (page) await shot('failure').catch(() => {}); result = { success: false, appRoot, actualElectron: desktop, checks, screenshots, errors, error: error.stack, realModelRequests: 0 };
|
|
} finally {
|
|
if (application) await application.close(); if (browser) await browser.close(); if (controller && await controller.health()) await controller.stop();
|
|
await new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }); for (const dispose of disposers.reverse()) await dispose();
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
|
}
|
|
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;
|