From 68603f9c76ac6219feddbf28a41e8d388a71c52e Mon Sep 17 00:00:00 2001 From: Agent Workbench Dev Date: Fri, 6 Nov 2026 23:07:49 +0800 Subject: [PATCH] fix: preserve native model parameters and align official Claude reasoning --- .../verify-model-parameter-fidelity-ui.mjs | 205 ++++++++++++++++++ src/client/NativeParameters.tsx | 14 +- src/client/ReasoningSettings.tsx | 33 ++- src/shared/contracts/model-catalog.d.mts | 1 + src/shared/contracts/model-catalog.mjs | 8 +- .../contracts/model-parameter-profiles.d.mts | 6 +- .../contracts/model-parameter-profiles.mjs | 71 ++++-- src/shared/contracts/model-request.mjs | 2 +- .../contracts/official-model-parameters.mjs | 20 ++ tests/model-parameter-fidelity.test.mjs | 194 +++++++++++++++++ 10 files changed, 508 insertions(+), 46 deletions(-) create mode 100644 scripts/verify-model-parameter-fidelity-ui.mjs create mode 100644 src/shared/contracts/official-model-parameters.mjs create mode 100644 tests/model-parameter-fidelity.test.mjs diff --git a/scripts/verify-model-parameter-fidelity-ui.mjs b/scripts/verify-model-parameter-fidelity-ui.mjs new file mode 100644 index 00000000..e90cfda4 --- /dev/null +++ b/scripts/verify-model-parameter-fidelity-ui.mjs @@ -0,0 +1,205 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import http from 'node:http'; +import net from 'node:net'; +import assert from 'node:assert/strict'; +import { randomUUID, createHash } from 'node:crypto'; +import { chromium, _electron } from 'playwright'; +import { expect } from 'playwright/test'; +import { fixture, launchWorkbench, until } from '../tests/helpers.mjs'; +import { Provider } from '../src/server/provider.mjs'; +import { HostController } from '../src/core/desktop/host-controller.mjs'; +import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs'; +import { projectEnvironment } from '../src/core/process-environment.mjs'; +import { catalogModel } from '../src/shared/contracts/model-catalog.mjs'; +import { openWorkbenchTool } from './ui-navigation.mjs'; + +const desktop = process.argv.includes('--desktop'); +assert.ok(process.argv.slice(2).every(value => value === '--desktop')); +assert.ok(process.env.AGENT_WORKBENCH_QA_APP_ROOT, 'Use an independent candidate build'); +const appRoot = fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT); +const directory = path.resolve('.qa/m1-model-parameters-2026-11-06', (desktop ? 'electron-' : 'web-') + randomUUID()); +fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory); +const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true }); +const requests = [], checks = [], screenshots = [], errors = [], externalBlocked = []; +const format = { type: 'json_schema', schema: { type: 'object', properties: { result: { type: 'string' } } } }; +const model = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const body = JSON.parse(Buffer.concat(chunks)); requests.push({ route: req.url, body }); + if (req.url.endsWith('/messages')) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ type: 'message', id: 'owned-msg', role: 'assistant', model: body.model, + content: [{ type: 'text', text: 'OK' }], stop_reason: 'end_turn', usage: { input_tokens: 11, output_tokens: 1 } })); + } else if (req.url.endsWith('/responses')) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ id: 'owned-resp', model: body.model, status: 'completed', + output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'OK' }] }], usage: { input_tokens: 11, output_tokens: 1 } })); + } else { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + res.end('data: ' + JSON.stringify({ model: body.model, choices: [{ index: 0, delta: { content: 'OK' }, finish_reason: 'stop' }] }) + '\n\ndata: [DONE]\n\n'); + } +}); +await new Promise(resolve => model.listen(0, '127.0.0.1', resolve)); +const provider = Object.create(Provider.prototype); provider.apiKey = ''; +const hub = provider.modelHub(f.store), members = {}; +for (const [key, protocol, name] of [ + ['opus', 'anthropic-messages', 'claude-opus-5-5'], ['fable', 'anthropic-messages', 'claude-fable-5-1'], + ['sonnet', 'anthropic-messages', 'claude-sonnet-5-5'], ['haiku', 'anthropic-messages', 'claude-haiku-5-5'], + ['responses', 'responses', 'owned-responses'], ['chat', 'chat-completions', 'owned-chat'], + ['negative', 'anthropic-messages', 'owned-negative'], ['unknown', 'anthropic-messages', 'owned-unknown'], +]) members[key] = hub.upsert({ name: '自有验证 · ' + key, model: name, protocol, requiresKey: false, + baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', streaming: key === 'chat', + capabilityDeclarations: { streaming: key === 'chat' ? 'supported' : 'unknown' }, + nativeParameters: protocol === 'anthropic-messages' ? { thinking: { type: 'adaptive', display: 'summarized' }, output_config: { effort: 'high', format } } + : protocol === 'responses' ? { reasoning: { effort: 'high', summary: 'auto' }, text: { format } } + : { thinking: { type: 'enabled', custom: { untouched: true } }, reasoning_effort: 'high', stream_options: { include_usage: false, include_obfuscation: false } } }); +hub.connection(members.negative.id).upstreamModel = catalogModel({ id: members.negative.model, capabilities: { + thinking: { supported: true, types: Object.fromEntries(['disabled', 'enabled', 'adaptive', 'between_tools'].map(value => [value, false])) }, + effort: { supported: true, low: false, medium: false, high: false, xhigh: false, max: false }, +} }); +hub.save(); f.store.close(); +let host, browser, application, controller, page, window, result, desktopStdout = '', desktopStderr = ''; +const form = () => page.getByRole('form', { name: '原生请求参数', exact: true }); +const field = label => form().getByLabel(label, { exact: true }); +const json = () => field('原生参数 JSON'); +const parameters = async () => JSON.parse(await json().inputValue()); +const current = async key => (await host.api('/model-center')).connections.find(row => row.id === members[key].id); +const mark = name => { checks.push(name); console.log('PASS ' + name); }; +async function resize(width, height = 960) { + if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); } + else await page.setViewportSize({ width, height }); +} +async function edit(key) { + if (!await page.getByRole('dialog', { name: '模型中心', exact: true }).count()) await openWorkbenchTool(page, '打开模型中心'); + await page.locator('.mc-connection[data-connection-id="' + members[key].id + '"]').getByRole('button', { name: '推理与能力', exact: true }).click(); + await expect(form()).toBeVisible(); +} +async function save(key, expected) { + await form().getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(form()).toHaveCount(0); + assert.deepEqual((await current(key)).nativeParameters, expected); +} +async function probe(key) { + const before = requests.length; await host.api('/model-center/probe', { id: members[key].id, scope: key === 'chat' ? 'streaming' : 'text' }); + assert.equal(requests.length, before + 1); return requests.at(-1).body; +} +async function options(label) { return field(label).locator('option').evaluateAll(nodes => nodes.map(node => ({ value: node.value, disabled: node.disabled, label: node.textContent }))); } +async function shot(name, anchor = '推理模式') { + await field(anchor).scrollIntoViewIfNeeded(); + assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), 'No page overflow'); + const filename = path.join(directory, name + '.png'); await page.screenshot({ path: filename, scale: 'css' }); + screenshots.push({ file: path.basename(filename), sha256: createHash('sha256').update(fs.readFileSync(filename)).digest('hex') }); + fs.writeFileSync(path.join(directory, name + '.txt'), await form().innerText()); +} +try { + if (desktop) { + const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve)); + assert.notEqual(port, 4318); controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start(); + host = { url: controller.url, api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) }; + const executable = desktopExecutableIdentity(appRoot).path; + application = await _electron.launch({ executablePath: executable, args: [path.join(appRoot, 'dist-desktop/main.cjs'), '--app-root', appRoot, + '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window'], cwd: appRoot, env: projectEnvironment(), timeout: 60000 }); + application.process().stdout?.on('data', bytes => { desktopStdout += bytes.toString('utf8'); }); + application.process().stderr?.on('data', bytes => { desktopStderr += bytes.toString('utf8'); }); + page = await application.firstWindow(); window = await application.browserWindow(page); await window.evaluate(win => win.showInactive()); + } else { + host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318'); + browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage(); + } + page.on('pageerror', cause => errors.push(cause.message)); + await page.route('**/*', route => { + const url = new URL(route.request().url()); + if (url.origin === new URL(host.url).origin || ['data:', 'blob:'].includes(url.protocol)) return route.continue(); + externalBlocked.push(url.origin); return route.abort(); + }); + await resize(1280); await page.goto(host.url); + await edit('opus'); + let modes = await options('推理模式'); + for (const value of ['disabled', 'enabled', 'between_tools']) assert.equal(modes.find(row => row.value === value).disabled, true); + assert.equal(modes.find(row => row.value === 'adaptive').disabled, false); + await expect(form()).toContainText('Anthropic 官方推理说明'); + await field('推理强度').selectOption('max'); + let expected = await parameters(); assert.deepEqual(expected.output_config, { effort: 'max', format }); assert.equal(expected.thinking.display, 'summarized'); + await field('推理模式').selectOption('default'); assert.deepEqual((await parameters()).thinking, { type: 'adaptive', display: 'summarized' }); + await shot('opus-1280'); await save('opus', expected); + let wire = await probe('opus'); assert.deepEqual(wire.output_config, expected.output_config); assert.deepEqual(wire.thinking, expected.thinking); assert.equal(wire.max_tokens, 128000); + await page.reload(); await edit('opus'); assert.deepEqual(await parameters(), expected); + await field('推理强度').selectOption(''); expected = await parameters(); assert.deepEqual(expected.output_config, { format }); + await save('opus', expected); await edit('opus'); assert.deepEqual(await parameters(), expected); + await field('参数配置档').selectOption('anthropic'); assert.deepEqual(await parameters(), expected); + await field('推理模式').selectOption('enabled'); expected = await parameters(); assert.deepEqual(expected.thinking, { type: 'enabled', display: 'summarized', budget_tokens: 1024 }); + assert.equal((await options('推理模式')).find(row => row.value === 'default').disabled, true); + await save('opus', expected); wire = await probe('opus'); assert.deepEqual(wire.thinking, expected.thinking); + await edit('opus'); await field('参数配置档').selectOption('auto'); assert.deepEqual(await parameters(), expected); + await expect(form()).toContainText('原值仍完整保留'); + await shot('opus-custom-preserved-1280'); await save('opus', expected); + mark('Opus official choices, sibling fields, valid default reset, explicit compatibility override and save/reopen/local wire'); + + await edit('fable'); modes = await options('推理模式'); + for (const value of ['disabled', 'enabled', 'between_tools']) assert.equal(modes.find(row => row.value === value).disabled, true); + await field('推理强度').selectOption('xhigh'); expected = await parameters(); assert.deepEqual(expected.output_config.format, format); + await save('fable', expected); wire = await probe('fable'); assert.deepEqual(wire.output_config, expected.output_config); + mark('Fable only offers adaptive mode and preserves format with xhigh'); + + await edit('sonnet'); await field('推理模式').selectOption('between_tools'); + assert.equal((await options('推理强度')).find(row => row.value === 'xhigh').disabled, true); + await field('推理强度').selectOption('high'); expected = await parameters(); assert.deepEqual(expected.thinking, { type: 'between_tools', display: 'summarized' }); + await shot('sonnet-between-tools-1280'); await save('sonnet', expected); wire = await probe('sonnet'); assert.deepEqual(wire.thinking, expected.thinking); + await edit('sonnet'); await field('推理模式').selectOption('adaptive'); await field('推理强度').selectOption('xhigh'); + assert.equal((await options('推理模式')).find(row => row.value === 'between_tools').disabled, true); + await field('推理强度').selectOption('high'); await field('推理模式').selectOption('between_tools'); + await resize(390, 844); await shot('sonnet-controls-390'); + await json().scrollIntoViewIfNeeded(); await shot('sonnet-json-390', '原生参数 JSON'); + const button = form().getByRole('button', { name: '保存原生参数', exact: true }); await button.scrollIntoViewIfNeeded(); + const box = await button.boundingBox(); assert.ok(box && box.x >= 0 && box.x + box.width <= 391); await save('sonnet', expected); + await page.reload(); await edit('sonnet'); assert.deepEqual(await parameters(), expected); + await form().getByRole('button', { name: '取消', exact: true }).click(); await resize(1280); + mark('Sonnet between_tools/high and adaptive/xhigh conditions, 390px controls, JSON, save and reload'); + + await edit('haiku'); await field('推理模式').selectOption('disabled'); await expect(field('推理强度')).toBeEnabled(); + assert.equal((await options('推理强度')).find(row => row.value === 'max').disabled, true); + await field('推理强度').selectOption('low'); expected = await parameters(); await save('haiku', expected); + wire = await probe('haiku'); assert.deepEqual(wire.thinking, expected.thinking); assert.equal(wire.output_config.effort, 'low'); + mark('Haiku disabled mode retains selectable low/medium/high and independent display/format'); + + await edit('responses'); await field('推理强度').selectOption('max'); expected = await parameters(); assert.deepEqual(expected.reasoning, { effort: 'max', summary: 'auto' }); + await save('responses', expected); wire = await probe('responses'); assert.deepEqual(wire.reasoning, expected.reasoning); assert.equal(wire.max_output_tokens, undefined); + await edit('responses'); await field('推理模式').selectOption('default'); expected = await parameters(); assert.deepEqual(expected.reasoning, { summary: 'auto' }); assert.deepEqual(expected.text, { format }); + await save('responses', expected); + mark('Responses effort edit/default preserve reasoning.summary and text.format, with no optional output cap'); + + await edit('chat'); expected = await parameters(); await field('参数配置档').selectOption('openai'); assert.deepEqual(await parameters(), expected); + await field('推理强度').selectOption('max'); expected = await parameters(); assert.deepEqual(expected.thinking, { type: 'enabled', custom: { untouched: true } }); + await save('chat', expected); wire = await probe('chat'); assert.deepEqual(wire.stream_options, expected.stream_options); assert.deepEqual(wire.thinking, expected.thinking); assert.equal(wire.max_tokens, undefined); + await page.reload(); await edit('chat'); assert.deepEqual(await parameters(), expected); await form().getByRole('button', { name: '取消', exact: true }).click(); + mark('Profile switch and Chat controls preserve opaque thinking and explicit streaming options through persistence and actual SSE wire'); + + await edit('negative'); modes = await options('推理模式'); assert.ok(modes.filter(row => row.value !== 'default').every(row => row.disabled)); + assert.equal((await options('推理强度')).filter(row => row.value !== '').every(row => row.disabled), true); + expected = await parameters(); await expect(form()).toContainText('原值仍完整保留'); + await shot('explicit-negative-1280'); await save('negative', expected); + await edit('unknown'); await expect(form()).toContainText('未取得的型号能力待确认'); + assert.equal((await options('推理模式')).find(row => row.value === 'default').disabled, true); + expected = await parameters(); expected.thinking.type = 'owned-custom-mode'; expected.output_config.effort = 'owned-custom-effort'; + await json().fill(JSON.stringify(expected, null, 2)); await expect(field('推理模式')).toHaveValue('owned-custom-mode'); + await expect(field('推理强度')).toHaveValue('owned-custom-effort'); await expect(form()).toContainText('原值仍完整保留'); + await resize(390, 844); await shot('custom-values-390'); await save('unknown', expected); + await page.reload(); await edit('unknown'); assert.deepEqual(await parameters(), expected); + mark('Explicit all-negative declarations stay negative, missing metadata stays unconfirmed, custom values stay visible and persisted'); + + const state = await host.api('/model-center'); assert.equal(state.attempts.length, requests.length); + assert.equal(requests.length, 7); assert.deepEqual(errors, []); assert.deepEqual(externalBlocked, []); + result = { success: true, appRoot, actualElectron: desktop, checks, screenshots, errors, externalBlocked, hostUrl: host.url, + syntheticRequests: requests.length, realModelRequests: 0, realCredentialsRead: 0, productionWrites: 0, viewportWidths: [1280, 390] }; +} catch (cause) { + if (page) { await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {}); fs.writeFileSync(path.join(directory, 'failure.txt'), await page.locator('body').innerText().catch(() => '')); } + result = { success: false, appRoot, actualElectron: desktop, checks, screenshots, errors, externalBlocked, error: cause.stack, syntheticRequests: requests.length, realModelRequests: 0 }; +} finally { + fs.writeFileSync(path.join(directory, 'requests.json'), JSON.stringify(requests, null, 2)); + if (application) await application.close(); if (browser) await browser.close(); + if (controller && await controller.health()) await controller.stop(); + await host?.close?.(); await new Promise(resolve => { model.closeAllConnections(); model.close(resolve); }); + for (const dispose of disposers.reverse()) await dispose(); + if (desktop) { fs.writeFileSync(path.join(directory, 'desktop-stdout.txt'), desktopStdout); fs.writeFileSync(path.join(directory, 'desktop-stderr.txt'), desktopStderr); } + fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); +} +console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1; diff --git a/src/client/NativeParameters.tsx b/src/client/NativeParameters.tsx index 7501956b..67173627 100644 --- a/src/client/NativeParameters.tsx +++ b/src/client/NativeParameters.tsx @@ -1,7 +1,6 @@ import { messagesOutputSetting } from '../shared/contracts/model-capacity.mjs'; import { useEffect, useRef, useState } from 'react'; import { nativeParameterSettings } from '../shared/contracts/native-parameters.mjs'; -import { changeReasoningSetting, modelParameterProfile, reasoningSetting } from '../shared/contracts/model-parameter-profiles.mjs'; import { ReasoningSettings } from './ReasoningSettings'; import type { CatalogModel } from '../shared/contracts/model-catalog.mjs'; @@ -24,19 +23,12 @@ export function NativeParameters({ connection, busy, onCancel, onSave, execution }}>

{connection.name} · 推理与原生参数

参数按所选协议原字段发送,并封存在实际请求中。具体模型是否接受这些选项,以服务响应为准;未填写的选项采用上游默认值。

- { - if (parameters) { - const before = { ...connection, parameterProfile: profile }, after = { ...connection, parameterProfile: value }; - if (modelParameterProfile(before).thinking !== modelParameterProfile(after).thinking) { - updateParameters(changeReasoningSetting(after, parameters, 'mode', reasoningSetting(before, parameters).mode)); - } - } - setProfile(value); - }} /> +

{connection.protocol === 'anthropic-messages' ? 'Messages 协议必须带 max_tokens;当前 ' + String(messagesOutputSetting(connection).value) + '(' + ({ explicit_native: '显式原生参数', upstream_catalog: '上游目录最大输出', official_model_spec: 'Anthropic 官方同步输出规格', connection_declaration: '连接声明,非已验证最大值' }[messagesOutputSetting(connection).source] ?? '') + ')。可在 JSON 中显式设置;本地不裁剪输出。' : executionMode === 'unrestricted' ? '默认不附加输出上限;可在 JSON 中显式填写上游参数,服务是否接受以原始响应为准。' : '数值沿用当前任务和连接的输出预算。'}消息、工具、接口和凭据由各自设置管理。