205 lines
18 KiB
JavaScript
205 lines
18 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import http from 'node:http';
|
|
import net from 'node:net';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, _electron } from 'playwright';
|
|
import { expect } from 'playwright/test';
|
|
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
|
import { desktopLaunchCommand } from './desktop-launch.mjs';
|
|
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
|
import { catalogModel } from '../src/shared/contracts/model-catalog.mjs';
|
|
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
|
|
const desktop = process.argv.includes('--desktop');
|
|
assert.ok(process.argv.slice(2).every(value => value === '--desktop'));
|
|
assert.ok(process.env.AGENT_WORKBENCH_QA_APP_ROOT, 'Use an independent candidate build');
|
|
const appRoot = fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT);
|
|
const directory = path.resolve('.qa/m1-model-parameters-2026-11-06', (desktop ? 'electron-' : 'web-') + randomUUID());
|
|
fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory);
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true });
|
|
const requests = [], checks = [], screenshots = [], errors = [], externalBlocked = [];
|
|
const format = { type: 'json_schema', schema: { type: 'object', properties: { result: { type: 'string' } } } };
|
|
const model = http.createServer(async (req, res) => {
|
|
const chunks = []; for await (const chunk of req) chunks.push(chunk);
|
|
const body = JSON.parse(Buffer.concat(chunks)); requests.push({ route: req.url, body });
|
|
if (req.url.endsWith('/messages')) {
|
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ type: 'message', id: 'owned-msg', role: 'assistant', model: body.model,
|
|
content: [{ type: 'text', text: 'OK' }], stop_reason: 'end_turn', usage: { input_tokens: 11, output_tokens: 1 } }));
|
|
} else if (req.url.endsWith('/responses')) {
|
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
res.end(JSON.stringify({ id: 'owned-resp', model: body.model, status: 'completed',
|
|
output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'OK' }] }], usage: { input_tokens: 11, output_tokens: 1 } }));
|
|
} else {
|
|
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
|
res.end('data: ' + JSON.stringify({ model: body.model, choices: [{ index: 0, delta: { content: 'OK' }, finish_reason: 'stop' }] }) + '\n\ndata: [DONE]\n\n');
|
|
}
|
|
});
|
|
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
|
const provider = Object.create(Provider.prototype); provider.apiKey = '';
|
|
const hub = provider.modelHub(f.store), members = {};
|
|
for (const [key, protocol, name] of [
|
|
['opus', 'anthropic-messages', 'claude-opus-5-5'], ['fable', 'anthropic-messages', 'claude-fable-5-1'],
|
|
['sonnet', 'anthropic-messages', 'claude-sonnet-5-5'], ['haiku', 'anthropic-messages', 'claude-haiku-5-5'],
|
|
['responses', 'responses', 'owned-responses'], ['chat', 'chat-completions', 'owned-chat'],
|
|
['negative', 'anthropic-messages', 'owned-negative'], ['unknown', 'anthropic-messages', 'owned-unknown'],
|
|
]) members[key] = hub.upsert({ name: '自有验证 · ' + key, model: name, protocol, requiresKey: false,
|
|
baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', streaming: key === 'chat',
|
|
capabilityDeclarations: { streaming: key === 'chat' ? 'supported' : 'unknown' },
|
|
nativeParameters: protocol === 'anthropic-messages' ? { thinking: { type: 'adaptive', display: 'summarized' }, output_config: { effort: 'high', format } }
|
|
: protocol === 'responses' ? { reasoning: { effort: 'high', summary: 'auto' }, text: { format } }
|
|
: { thinking: { type: 'enabled', custom: { untouched: true } }, reasoning_effort: 'high', stream_options: { include_usage: false, include_obfuscation: false } } });
|
|
hub.connection(members.negative.id).upstreamModel = catalogModel({ id: members.negative.model, capabilities: {
|
|
thinking: { supported: true, types: Object.fromEntries(['disabled', 'enabled', 'adaptive', 'between_tools'].map(value => [value, false])) },
|
|
effort: { supported: true, low: false, medium: false, high: false, xhigh: false, max: false },
|
|
} });
|
|
hub.save(); f.store.close();
|
|
let host, browser, application, controller, page, window, result, desktopStdout = '', desktopStderr = '';
|
|
const form = () => page.getByRole('form', { name: '原生请求参数', exact: true });
|
|
const field = label => form().getByLabel(label, { exact: true });
|
|
const json = () => field('原生参数 JSON');
|
|
const parameters = async () => JSON.parse(await json().inputValue());
|
|
const current = async key => (await host.api('/model-center')).connections.find(row => row.id === members[key].id);
|
|
const mark = name => { checks.push(name); console.log('PASS ' + name); };
|
|
async function resize(width, height = 960) {
|
|
if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); }
|
|
else await page.setViewportSize({ width, height });
|
|
}
|
|
async function edit(key) {
|
|
if (!await page.getByRole('dialog', { name: '模型中心', exact: true }).count()) await openWorkbenchTool(page, '打开模型中心');
|
|
await page.locator('.mc-connection[data-connection-id="' + members[key].id + '"]').getByRole('button', { name: '推理与能力', exact: true }).click();
|
|
await expect(form()).toBeVisible();
|
|
}
|
|
async function save(key, expected) {
|
|
await form().getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(form()).toHaveCount(0);
|
|
assert.deepEqual((await current(key)).nativeParameters, expected);
|
|
}
|
|
async function probe(key) {
|
|
const before = requests.length; await host.api('/model-center/probe', { id: members[key].id, scope: key === 'chat' ? 'streaming' : 'text' });
|
|
assert.equal(requests.length, before + 1); return requests.at(-1).body;
|
|
}
|
|
async function options(label) { return field(label).locator('option').evaluateAll(nodes => nodes.map(node => ({ value: node.value, disabled: node.disabled, label: node.textContent }))); }
|
|
async function shot(name, anchor = '推理模式') {
|
|
await field(anchor).scrollIntoViewIfNeeded();
|
|
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), 'No page overflow');
|
|
const filename = path.join(directory, name + '.png'); await page.screenshot({ path: filename, scale: 'css' });
|
|
screenshots.push({ file: path.basename(filename), sha256: createHash('sha256').update(fs.readFileSync(filename)).digest('hex') });
|
|
fs.writeFileSync(path.join(directory, name + '.txt'), await form().innerText());
|
|
}
|
|
try {
|
|
if (desktop) {
|
|
const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve));
|
|
assert.notEqual(port, 4318); controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start();
|
|
host = { url: controller.url, api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) };
|
|
application = await _electron.launch({ ...desktopLaunchCommand(appRoot, ['--app-root', appRoot,
|
|
'--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window']), cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
|
application.process().stdout?.on('data', bytes => { desktopStdout += bytes.toString('utf8'); });
|
|
application.process().stderr?.on('data', bytes => { desktopStderr += bytes.toString('utf8'); });
|
|
page = await application.firstWindow(); window = await application.browserWindow(page); await window.evaluate(win => win.showInactive());
|
|
} else {
|
|
host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318');
|
|
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage();
|
|
}
|
|
page.on('pageerror', cause => errors.push(cause.message));
|
|
await page.route('**/*', route => {
|
|
const url = new URL(route.request().url());
|
|
if (url.origin === new URL(host.url).origin || ['data:', 'blob:'].includes(url.protocol)) return route.continue();
|
|
externalBlocked.push(url.origin); return route.abort();
|
|
});
|
|
await resize(1280); await page.goto(host.url);
|
|
await edit('opus');
|
|
let modes = await options('推理模式');
|
|
for (const value of ['disabled', 'enabled', 'between_tools']) assert.equal(modes.find(row => row.value === value).disabled, true);
|
|
assert.equal(modes.find(row => row.value === 'adaptive').disabled, false);
|
|
await expect(form()).toContainText('Anthropic 官方推理说明');
|
|
await field('推理强度').selectOption('max');
|
|
let expected = await parameters(); assert.deepEqual(expected.output_config, { effort: 'max', format }); assert.equal(expected.thinking.display, 'summarized');
|
|
await field('推理模式').selectOption('default'); assert.deepEqual((await parameters()).thinking, { type: 'adaptive', display: 'summarized' });
|
|
await shot('opus-1280'); await save('opus', expected);
|
|
let wire = await probe('opus'); assert.deepEqual(wire.output_config, expected.output_config); assert.deepEqual(wire.thinking, expected.thinking); assert.equal(wire.max_tokens, 128000);
|
|
await page.reload(); await edit('opus'); assert.deepEqual(await parameters(), expected);
|
|
await field('推理强度').selectOption(''); expected = await parameters(); assert.deepEqual(expected.output_config, { format });
|
|
await save('opus', expected); await edit('opus'); assert.deepEqual(await parameters(), expected);
|
|
await field('参数配置档').selectOption('anthropic'); assert.deepEqual(await parameters(), expected);
|
|
await field('推理模式').selectOption('enabled'); expected = await parameters(); assert.deepEqual(expected.thinking, { type: 'enabled', display: 'summarized', budget_tokens: 1024 });
|
|
assert.equal((await options('推理模式')).find(row => row.value === 'default').disabled, true);
|
|
await save('opus', expected); wire = await probe('opus'); assert.deepEqual(wire.thinking, expected.thinking);
|
|
await edit('opus'); await field('参数配置档').selectOption('auto'); assert.deepEqual(await parameters(), expected);
|
|
await expect(form()).toContainText('原值仍完整保留');
|
|
await shot('opus-custom-preserved-1280'); await save('opus', expected);
|
|
mark('Opus official choices, sibling fields, valid default reset, explicit compatibility override and save/reopen/local wire');
|
|
|
|
await edit('fable'); modes = await options('推理模式');
|
|
for (const value of ['disabled', 'enabled', 'between_tools']) assert.equal(modes.find(row => row.value === value).disabled, true);
|
|
await field('推理强度').selectOption('xhigh'); expected = await parameters(); assert.deepEqual(expected.output_config.format, format);
|
|
await save('fable', expected); wire = await probe('fable'); assert.deepEqual(wire.output_config, expected.output_config);
|
|
mark('Fable only offers adaptive mode and preserves format with xhigh');
|
|
|
|
await edit('sonnet'); await field('推理模式').selectOption('between_tools');
|
|
assert.equal((await options('推理强度')).find(row => row.value === 'xhigh').disabled, true);
|
|
await field('推理强度').selectOption('high'); expected = await parameters(); assert.deepEqual(expected.thinking, { type: 'between_tools', display: 'summarized' });
|
|
await shot('sonnet-between-tools-1280'); await save('sonnet', expected); wire = await probe('sonnet'); assert.deepEqual(wire.thinking, expected.thinking);
|
|
await edit('sonnet'); await field('推理模式').selectOption('adaptive'); await field('推理强度').selectOption('xhigh');
|
|
assert.equal((await options('推理模式')).find(row => row.value === 'between_tools').disabled, true);
|
|
await field('推理强度').selectOption('high'); await field('推理模式').selectOption('between_tools');
|
|
await resize(390, 844); await shot('sonnet-controls-390');
|
|
await json().scrollIntoViewIfNeeded(); await shot('sonnet-json-390', '原生参数 JSON');
|
|
const button = form().getByRole('button', { name: '保存原生参数', exact: true }); await button.scrollIntoViewIfNeeded();
|
|
const box = await button.boundingBox(); assert.ok(box && box.x >= 0 && box.x + box.width <= 391); await save('sonnet', expected);
|
|
await page.reload(); await edit('sonnet'); assert.deepEqual(await parameters(), expected);
|
|
await form().getByRole('button', { name: '取消', exact: true }).click(); await resize(1280);
|
|
mark('Sonnet between_tools/high and adaptive/xhigh conditions, 390px controls, JSON, save and reload');
|
|
|
|
await edit('haiku'); await field('推理模式').selectOption('disabled'); await expect(field('推理强度')).toBeEnabled();
|
|
assert.equal((await options('推理强度')).find(row => row.value === 'max').disabled, true);
|
|
await field('推理强度').selectOption('low'); expected = await parameters(); await save('haiku', expected);
|
|
wire = await probe('haiku'); assert.deepEqual(wire.thinking, expected.thinking); assert.equal(wire.output_config.effort, 'low');
|
|
mark('Haiku disabled mode retains selectable low/medium/high and independent display/format');
|
|
|
|
await edit('responses'); await field('推理强度').selectOption('max'); expected = await parameters(); assert.deepEqual(expected.reasoning, { effort: 'max', summary: 'auto' });
|
|
await save('responses', expected); wire = await probe('responses'); assert.deepEqual(wire.reasoning, expected.reasoning); assert.equal(wire.max_output_tokens, undefined);
|
|
await edit('responses'); await field('推理模式').selectOption('default'); expected = await parameters(); assert.deepEqual(expected.reasoning, { summary: 'auto' }); assert.deepEqual(expected.text, { format });
|
|
await save('responses', expected);
|
|
mark('Responses effort edit/default preserve reasoning.summary and text.format, with no optional output cap');
|
|
|
|
await edit('chat'); expected = await parameters(); await field('参数配置档').selectOption('openai'); assert.deepEqual(await parameters(), expected);
|
|
await field('推理强度').selectOption('max'); expected = await parameters(); assert.deepEqual(expected.thinking, { type: 'enabled', custom: { untouched: true } });
|
|
await save('chat', expected); wire = await probe('chat'); assert.deepEqual(wire.stream_options, expected.stream_options); assert.deepEqual(wire.thinking, expected.thinking); assert.equal(wire.max_tokens, undefined);
|
|
await page.reload(); await edit('chat'); assert.deepEqual(await parameters(), expected); await form().getByRole('button', { name: '取消', exact: true }).click();
|
|
mark('Profile switch and Chat controls preserve opaque thinking and explicit streaming options through persistence and actual SSE wire');
|
|
|
|
await edit('negative'); modes = await options('推理模式'); assert.ok(modes.filter(row => row.value !== 'default').every(row => row.disabled));
|
|
assert.equal((await options('推理强度')).filter(row => row.value !== '').every(row => row.disabled), true);
|
|
expected = await parameters(); await expect(form()).toContainText('原值仍完整保留');
|
|
await shot('explicit-negative-1280'); await save('negative', expected);
|
|
await edit('unknown'); await expect(form()).toContainText('未取得的型号能力待确认');
|
|
assert.equal((await options('推理模式')).find(row => row.value === 'default').disabled, true);
|
|
expected = await parameters(); expected.thinking.type = 'owned-custom-mode'; expected.output_config.effort = 'owned-custom-effort';
|
|
await json().fill(JSON.stringify(expected, null, 2)); await expect(field('推理模式')).toHaveValue('owned-custom-mode');
|
|
await expect(field('推理强度')).toHaveValue('owned-custom-effort'); await expect(form()).toContainText('原值仍完整保留');
|
|
await resize(390, 844); await shot('custom-values-390'); await save('unknown', expected);
|
|
await page.reload(); await edit('unknown'); assert.deepEqual(await parameters(), expected);
|
|
mark('Explicit all-negative declarations stay negative, missing metadata stays unconfirmed, custom values stay visible and persisted');
|
|
|
|
const state = await host.api('/model-center'); assert.equal(state.attempts.length, requests.length);
|
|
assert.equal(requests.length, 7); assert.deepEqual(errors, []); assert.deepEqual(externalBlocked, []);
|
|
result = { success: true, appRoot, actualElectron: desktop, checks, screenshots, errors, externalBlocked, hostUrl: host.url,
|
|
syntheticRequests: requests.length, realModelRequests: 0, realCredentialsRead: 0, productionWrites: 0, viewportWidths: [1280, 390] };
|
|
} catch (cause) {
|
|
if (page) { await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {}); fs.writeFileSync(path.join(directory, 'failure.txt'), await page.locator('body').innerText().catch(() => '')); }
|
|
result = { success: false, appRoot, actualElectron: desktop, checks, screenshots, errors, externalBlocked, error: cause.stack, syntheticRequests: requests.length, realModelRequests: 0 };
|
|
} finally {
|
|
fs.writeFileSync(path.join(directory, 'requests.json'), JSON.stringify(requests, null, 2));
|
|
if (application) await application.close(); if (browser) await browser.close();
|
|
if (controller && await controller.health()) await controller.stop();
|
|
await host?.close?.(); await new Promise(resolve => { model.closeAllConnections(); model.close(resolve); });
|
|
for (const dispose of disposers.reverse()) await dispose();
|
|
if (desktop) { fs.writeFileSync(path.join(directory, 'desktop-stdout.txt'), desktopStdout); fs.writeFileSync(path.join(directory, 'desktop-stderr.txt'), desktopStderr); }
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
|
}
|
|
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;
|