111 lines
12 KiB
JavaScript
111 lines
12 KiB
JavaScript
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import http from 'node:http';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { chromium, expect } from 'playwright/test';
|
|
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
|
|
const args = process.argv.slice(2);
|
|
assert.ok(args.length === 0 || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
|
const appRoot = args[1] ?? process.cwd(), directory = path.resolve('.qa', 'model-fidelity-ui-' + randomUUID());
|
|
fs.mkdirSync(directory, { recursive: true });
|
|
const disposers = [], t = { after: action => disposers.push(action) }, f = fixture(t), requests = [], screenshots = [], checks = [], errors = [];
|
|
let mode = 'quota', browser, page;
|
|
const model = http.createServer(async (request, response) => {
|
|
const parts = []; for await (const part of request) parts.push(part); requests.push({ route: request.url, body: JSON.parse(Buffer.concat(parts)) });
|
|
if (mode === 'quota') {
|
|
response.writeHead(429, { 'Content-Type': 'application/json', 'X-Request-ID': 'owned-upstream-quota-ui', 'Retry-After': '180' });
|
|
response.end(JSON.stringify({ error: { code: 'insufficient_quota', type: 'billing_error', param: 'account', message: 'Owned quota error; owned-ui-private-key' } }));
|
|
} else {
|
|
response.writeHead(200, { 'Content-Type': 'text/event-stream', 'X-Request-ID': 'owned-upstream-stream-ui' });
|
|
const event = value => 'data: ' + JSON.stringify(value) + '\n\n';
|
|
response.end(event({ type: 'response.created', sequence_number: 1, response: { id: 'resp-owned-ui' } }) +
|
|
event({ type: 'response.output_item.added', sequence_number: 2, item: { id: 'fc-owned-ui', call_id: 'owned-ui-tool', type: 'function_call', name: 'propose_file', arguments: '' } }) +
|
|
event({ type: 'response.function_call_arguments.delta', sequence_number: 3, item_id: 'fc-owned-ui', delta: '{"path":' }) +
|
|
event({ type: 'response.failed', sequence_number: 4, response: { id: 'resp-owned-ui', status: 'failed', error: { code: 'server_error', type: 'api_error', message: 'Owned stream failed' }, usage: { input_tokens: 20, output_tokens: 4 } } }));
|
|
}
|
|
});
|
|
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
|
const provider = new Provider(), hub = provider.modelHub(f.store);
|
|
let member = hub.upsert({ name: 'Owned UI 模型', model: 'owned-ui-model', baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', apiKey: 'owned-ui-private-key', requiresKey: true });
|
|
hub.select({ type: 'connection', id: member.id }); f.store.close();
|
|
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath: fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath });
|
|
// The test credential is intentionally volatile; set it in this new test host.
|
|
member = await host.api('/model-center/connections', { ...member, apiKey: 'owned-ui-private-key' });
|
|
const mark = text => { checks.push(text); console.log('PASS ' + text); };
|
|
async function capture(name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file }); screenshots.push({ path: file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') }); }
|
|
async function openCenter() { await openWorkbenchTool(page, '打开模型中心'); return page.getByRole('dialog', { name: '模型中心', exact: true }); }
|
|
async function run(prompt) {
|
|
await page.getByLabel('任务内容').fill(prompt); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
|
let id; await until(async () => { id = (await host.api('/sessions')).find(value => value.title === prompt)?.id; return id && (await host.api('/sessions/' + id)).status === 'failed'; }, 15000);
|
|
return host.api('/sessions/' + id);
|
|
}
|
|
let result;
|
|
try {
|
|
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
|
page.on('pageerror', error => errors.push(error.message)); await page.goto(host.url); await page.getByLabel('当前项目').selectOption(f.project.id);
|
|
let dialog = await openCenter(); const card = dialog.locator('.mc-connection').filter({ has: page.getByRole('heading', { name: 'Owned UI 模型', exact: true }) });
|
|
await card.getByRole('button', { name: '推理与能力', exact: true }).click();
|
|
await page.getByLabel('原生参数 JSON', { exact: true }).fill('{"reasoning_effort":"high","temperature":0.4,"parallel_tool_calls":false}');
|
|
await page.getByLabel('输出上限字段', { exact: true }).selectOption('max_completion_tokens');
|
|
await page.getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(dialog.getByRole('status')).toContainText('原生参数已保存');
|
|
assert.equal(requests.length, 0); await page.reload(); dialog = await openCenter();
|
|
await dialog.locator('.mc-connection').filter({ has: page.getByRole('heading', { name: 'Owned UI 模型', exact: true }) }).getByRole('button', { name: '推理与能力', exact: true }).click();
|
|
await expect(page.getByLabel('原生参数 JSON', { exact: true })).toContainText('reasoning_effort');
|
|
await capture('native-desktop'); await page.setViewportSize({ width: 390, height: 844 });
|
|
await page.getByRole('button', { name: '保存原生参数', exact: true }).scrollIntoViewIfNeeded();
|
|
const bounds = await page.getByRole('button', { name: '保存原生参数', exact: true }).boundingBox(); assert.ok(bounds && bounds.x >= 0 && bounds.x + bounds.width <= 390);
|
|
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); await capture('native-390'); mark('native parameters persist after reload and the narrow form remains usable without calling a model');
|
|
await page.getByLabel('原生参数 JSON', { exact: true }).fill('{"tools":[]}'); await page.getByRole('button', { name: '保存原生参数', exact: true }).click();
|
|
await expect(page.getByRole('form', { name: '原生请求参数' }).getByRole('alert')).toContainText('尚未接入原生参数:tools');
|
|
assert.equal(requests.length, 0); await page.keyboard.press('Escape'); await page.setViewportSize({ width: 1440, height: 1000 }); mark('unsupported field is rejected visibly before any dispatch');
|
|
const first = await run('读取当前项目,HTTP错误原文验收'); assert.equal(requests.length, 1); assert.equal(first.changes.length, 0);
|
|
assert.equal(requests[0].body.reasoning_effort, 'high'); assert.equal(requests[0].body.temperature, 0.4); assert.equal(requests[0].body.parallel_tool_calls, false);
|
|
assert.ok(requests[0].body.max_completion_tokens > 0); assert.equal(requests[0].body.max_tokens, undefined);
|
|
const failureCard = page.locator('.failure-card').last(); await failureCard.getByText('查看上游原文与请求信息', { exact: true }).click();
|
|
await expect(failureCard).toContainText('owned-upstream-quota-ui'); await expect(failureCard).toContainText('insufficient_quota'); await expect(failureCard).toContainText('180 秒');
|
|
assert.ok(!(await failureCard.innerText()).includes('owned-ui-private-key')); await capture('http-error-desktop');
|
|
await failureCard.getByLabel('上游错误原文').scrollIntoViewIfNeeded(); await capture('http-raw-desktop');
|
|
const downloadEvent = page.waitForEvent('download'); await failureCard.getByRole('button', { name: '下载已保存的原文', exact: true }).click();
|
|
const download = await downloadEvent, downloaded = path.join(directory, 'owned-http-error.txt'); await download.saveAs(downloaded);
|
|
assert.equal(createHash('sha256').update(fs.readFileSync(downloaded)).digest('hex'), first.failure.upstream.artifactSha256);
|
|
await page.setViewportSize({ width: 390, height: 844 }); await failureCard.scrollIntoViewIfNeeded(); await capture('http-error-390');
|
|
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); mark('actual HTTP error shows raw fields and original wait, keeps secrets hidden and downloads SHA-verified original at both widths');
|
|
dialog = await openCenter(); await dialog.getByRole('button', { name: '请求账本', exact: true }).click();
|
|
const attempt = dialog.locator('.mc-attempts article').first(); await attempt.getByText('查看上游原文与请求信息', { exact: true }).click();
|
|
await expect(attempt).toContainText('owned-upstream-quota-ui'); await expect(attempt.getByRole('button', { name: '下载实际请求', exact: true })).toBeVisible();
|
|
await capture('ledger-390'); await page.keyboard.press('Escape'); mark('physical request ledger preserves the same error origin and offers the sealed actual request');
|
|
mode = 'stream'; let center = await host.api('/model-center'), current = center.connections.find(value => value.id === member.id);
|
|
member = await host.api('/model-center/connections', { ...current, protocol: 'responses', streaming: true, capabilityDeclarations: { ...current.capabilityDeclarations, streaming: 'supported' },
|
|
nativeParameters: { reasoning: { effort: 'medium' }, text: { verbosity: 'high' } }, outputTokenField: 'max_output_tokens' });
|
|
const secondSession = await host.api('/sessions', { projectId: f.project.id });
|
|
await page.evaluate(value => { localStorage.setItem('awb-project', value.projectId); localStorage.setItem('awb-session', value.id); }, secondSession);
|
|
await page.reload(); await page.setViewportSize({ width: 1440, height: 1000 });
|
|
const second = await run('读取当前项目,Responses流失败验收');
|
|
assert.equal(requests.length, 2); assert.equal(requests[1].route, '/v1/responses'); assert.deepEqual(requests[1].body.reasoning, { effort: 'medium' });
|
|
assert.equal(second.execution.run.usage.totalTokens, 24); assert.equal(second.changes.length, 0); assert.equal(second.execution.run.usage.requests, 1);
|
|
const streamCard = page.locator('.failure-card').last(); await streamCard.getByText('查看上游原文与请求信息', { exact: true }).click();
|
|
await expect(streamCard).toContainText('response.failed'); await expect(streamCard).toContainText('resp-owned-ui'); await expect(streamCard).toContainText('owned-upstream-stream-ui');
|
|
await expect(streamCard).toContainText('流式响应已开始'); await capture('sse-error-desktop');
|
|
await page.setViewportSize({ width: 390, height: 844 }); await streamCard.scrollIntoViewIfNeeded(); await capture('sse-error-390');
|
|
await streamCard.getByLabel('上游错误原文').scrollIntoViewIfNeeded(); await capture('sse-raw-390');
|
|
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); assert.deepEqual(errors, []);
|
|
mark('Responses failed event retains original identity and reported use at both widths; partial tool is never dispatched or retried');
|
|
mode = 'quota';
|
|
await until(async () => (await host.api('/model-center')).connections.find(value => value.id === member.id).status === 'configured');
|
|
dialog = await openCenter();
|
|
await dialog.locator('.mc-connection').filter({ has: page.getByRole('heading', { name: 'Owned UI 模型', exact: true }) }).getByRole('button', { name: '验证响应(调用模型)', exact: true }).click();
|
|
await expect(dialog.locator('.mc-notice')).toContainText('额度或消费限制');
|
|
await dialog.locator('.mc-notice').getByText('查看上游原文与请求信息', { exact: true }).click();
|
|
await expect(dialog.locator('.mc-notice')).toContainText('owned-upstream-quota-ui');
|
|
await expect(dialog.locator('.mc-notice')).toContainText('仅保留以上诊断预览');
|
|
assert.equal(requests.length, 3); assert.deepEqual(errors, []); await capture('probe-error-390');
|
|
mark('explicit synthetic probe also shows the API diagnostic and labels its preview-only retention without inventing a Task artifact');
|
|
result = { success: true, appRoot, version: (await host.api('/health')).version, checks, screenshots, errors, modelRequests: requests.length, realModelRequests: 0 };
|
|
} catch (error) { if (page) await capture('failure').catch(() => {}); result = { success: false, appRoot, error: error.stack, checks, screenshots, errors, realModelRequests: 0 }; throw error; }
|
|
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); console.log(JSON.stringify({ success: result.success, directory, checks: checks.length, screenshots: screenshots.length }));
|
|
await browser?.close(); await new Promise(resolve => { model.closeAllConnections(); model.close(resolve); }); for (const action of disposers.reverse()) await action(); }
|