183 lines
17 KiB
JavaScript
183 lines
17 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import net from 'node:net';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, _electron } from 'playwright';
|
|
import { expect } from 'playwright/test';
|
|
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
|
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
|
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
|
import { usageSamples, usageModelId, cacheUsageFixture } from '../tests/fixtures/cache-usage-model.mjs';
|
|
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
|
|
assert.ok(process.argv.slice(2).every(value => value === '--desktop'));
|
|
assert.ok(process.env.AGENT_WORKBENCH_QA_APP_ROOT, 'Use a frozen independent candidate');
|
|
const desktop = process.argv.includes('--desktop'), appRoot = fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT);
|
|
const directory = path.resolve('.qa/m2-cache-usage-2026-11-07', (desktop ? 'electron-' : 'web-') + randomUUID());
|
|
fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory);
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true });
|
|
const hub = Object.assign(Object.create(Provider.prototype), { apiKey: '' }).modelHub(f.store);
|
|
const historical = { id: randomUUID(), connectionId: 'owned-legacy-connection', runId: 'owned-legacy-run', ordinal: 1,
|
|
model: 'owned-historical-cache', protocol: 'chat-completions', status: 'completed', startedAt: '2026-11-01T00:00:00Z', usageStatus: 'reported',
|
|
usage: { prompt_tokens: 1000, completion_tokens: 100, total_tokens: 1100, protocol: 'chat-completions', raw: structuredClone(usageSamples.chat.raw) } };
|
|
hub.attempts.push(historical); hub.saveAttempts();
|
|
const conflict = structuredClone(usageSamples.responses); conflict.raw.total_tokens = 1;
|
|
const scenes = [
|
|
{ name: 'chat', ...usageSamples.chat }, { name: 'responses', ...usageSamples.responses, streaming: true },
|
|
{ name: 'messages', ...usageSamples.messages, streaming: true }, { name: 'missing', protocol: 'chat-completions' },
|
|
{ name: 'conflict', ...conflict }, { name: 'upstream-error', ...usageSamples.chat, status: 400 },
|
|
];
|
|
for (const scene of scenes) {
|
|
scene.model = await cacheUsageFixture(t, scene);
|
|
scene.connection = hub.upsert({ name: '自有用量验证 · ' + scene.name, model: usageModelId, protocol: scene.protocol, streaming: !!scene.streaming,
|
|
baseUrl: scene.model.baseUrl, requiresKey: false, modelIdentityPolicy: 'require-match' });
|
|
}
|
|
f.store.close();
|
|
const checks = [], screenshots = [], errors = [], externalBlocked = [], downloads = [];
|
|
let host, controller, browser, app, page, window, result, desktopStdout = '', desktopStderr = '';
|
|
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
const mark = value => { checks.push(value); console.log('PASS ' + value); };
|
|
const center = () => page.getByRole('dialog', { name: '模型中心', exact: true });
|
|
const panel = () => page.getByLabel('请求上下文', { exact: true });
|
|
const metric = (scope, name) => scope.locator(':scope > dl > div').filter({ has: page.locator('dt').filter({ hasText: name }) }).locator('dd');
|
|
async function closeDialog() {
|
|
if (await page.getByRole('dialog').count()) await page.getByRole('dialog').getByRole('button', { name: '关闭窗口', exact: true }).click();
|
|
else await page.getByRole('button', { name: '关闭检视区', exact: true }).click();
|
|
}
|
|
async function resize(width, height = 960) {
|
|
if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); }
|
|
else await page.setViewportSize({ width, height });
|
|
}
|
|
async function shot(name, scope) {
|
|
await scope.scrollIntoViewIfNeeded();
|
|
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), 'No unintended page overflow');
|
|
const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' });
|
|
screenshots.push({ file: path.basename(file), sha256: sha(fs.readFileSync(file)) });
|
|
fs.writeFileSync(path.join(directory, name + '.txt'), await scope.innerText());
|
|
}
|
|
async function ledger() { await openWorkbenchTool(page, '打开模型中心'); await center().getByRole('button', { name: '请求账本', exact: true }).click(); }
|
|
function rowFor(scene) { return center().locator('.mc-attempts article').filter({ has: page.locator('code').filter({ hasText: 'Run ' + scene.session.runId }) }); }
|
|
async function originalDownload(scene, row) {
|
|
const file = path.join(directory, scene.name + '-original-response.sse'); let native = null;
|
|
if (app) {
|
|
const id = randomUUID();
|
|
await app.evaluate(({ session }, { id, file }) => {
|
|
globalThis.workbenchQaDownloads ??= {}; globalThis.workbenchQaDownloads[id] = { state: 'waiting' };
|
|
session.defaultSession.once('will-download', (_event, item) => {
|
|
const value = globalThis.workbenchQaDownloads[id]; value.suggestedFilename = item.getFilename(); value.state = 'started';
|
|
item.setSavePath(file);
|
|
item.once('done', (_event, state) => { value.state = state; value.bytes = item.getReceivedBytes(); value.totalBytes = item.getTotalBytes(); });
|
|
});
|
|
}, { id, file });
|
|
await row.getByRole('button', { name: '下载已保存的响应', exact: true }).click();
|
|
await expect.poll(() => app.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id].state, id), { timeout: 30000 }).toBe('completed');
|
|
native = await app.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id], id);
|
|
assert.equal(native.bytes, native.totalBytes); assert.equal(native.bytes, fs.statSync(file).size);
|
|
} else {
|
|
const downloadEvent = page.waitForEvent('download');
|
|
await row.getByRole('button', { name: '下载已保存的响应', exact: true }).click();
|
|
const download = await downloadEvent; await download.saveAs(file);
|
|
}
|
|
assert.equal(sha(fs.readFileSync(file)), sha(scene.model.responses[0])); downloads.push({ file: path.basename(file), sha256: sha(fs.readFileSync(file)),
|
|
...(native ? { native, scope: 'Actual Electron DownloadItem; QA selects owned path, not the Windows save dialog' } : { scope: 'Actual browser download' }) });
|
|
}
|
|
async function viewContext(scene) {
|
|
await page.evaluate(id => localStorage.setItem('awb-session', id), scene.session.id); await page.reload(); await page.waitForLoadState('networkidle');
|
|
await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await panel().waitFor({ state: 'visible' });
|
|
}
|
|
try {
|
|
if (desktop) {
|
|
const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve));
|
|
assert.notEqual(port, 4318); controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start();
|
|
host = { url: controller.url, api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) };
|
|
app = await _electron.launch({ executablePath: desktopExecutableIdentity(appRoot).path,
|
|
args: [path.join(appRoot, 'dist-desktop/main.cjs'), '--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window'],
|
|
cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
|
app.process().stdout?.on('data', bytes => { desktopStdout += bytes.toString('utf8'); }); app.process().stderr?.on('data', bytes => { desktopStderr += bytes.toString('utf8'); });
|
|
page = await app.firstWindow(); window = await app.browserWindow(page); await window.evaluate(win => win.showInactive());
|
|
} else {
|
|
host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318');
|
|
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ acceptDownloads: true });
|
|
}
|
|
page.on('pageerror', error => errors.push(error.message));
|
|
await page.route('**/*', route => {
|
|
const url = new URL(route.request().url());
|
|
if (url.origin === new URL(host.url).origin || ['data:', 'blob:'].includes(url.protocol)) return route.continue();
|
|
externalBlocked.push(url.origin); return route.abort();
|
|
});
|
|
await page.addInitScript(id => localStorage.setItem('awb-project', id), f.project.id);
|
|
for (const scene of scenes) {
|
|
await host.api('/model-center/select', { type: 'connection', id: scene.connection.id });
|
|
scene.session = await host.api('/sessions', { projectId: f.project.id });
|
|
await host.api('/sessions/' + scene.session.id + '/messages', { content: '自有用量样本 ' + scene.name + ':只说明收到完整回复,不修改文件或调用工具。' });
|
|
await until(async () => ['completed', 'failed'].includes((await host.api('/sessions/' + scene.session.id)).status), 30000);
|
|
scene.session = await host.api('/sessions/' + scene.session.id);
|
|
scene.plan = (await host.api('/sessions/' + scene.session.id + '/context-plans'))[0];
|
|
scene.attempt = (await host.api('/model-center')).attempts.find(value => value.sessionId === scene.session.id);
|
|
assert.equal(scene.model.requests.length, 1); assert.equal(scene.session.execution.progress.totalTools, 0);
|
|
assert.equal(scene.session.status, scene.status === 400 ? 'failed' : 'completed');
|
|
assert.deepEqual(scene.plan.actualUsage?.raw, scene.raw);
|
|
fs.writeFileSync(path.join(directory, scene.name + '-receipt.json'), JSON.stringify({ session: scene.session, plan: scene.plan, attempt: scene.attempt }, null, 2));
|
|
fs.writeFileSync(path.join(directory, scene.name + '-original-response.bin'), scene.model.responses[0]);
|
|
}
|
|
await resize(1280); await page.goto(host.url); await page.waitForLoadState('networkidle'); await ledger();
|
|
const chat = rowFor(scenes[0]); await chat.locator('.model-usage-details > summary').click(); const details = chat.locator('.model-usage-details');
|
|
await expect(metric(details, '缓存读取')).toHaveText('600 tokens'); await expect(metric(details, '缓存写入')).toHaveText('300 tokens');
|
|
await expect(metric(details, '未缓存输入')).toHaveText('100 tokens'); await expect(metric(details, '推理输出')).toHaveText('40 tokens');
|
|
await details.locator('.model-usage-sources > summary').click(); await expect(details).toContainText('usage.prompt_tokens_details.cache_write_tokens');
|
|
await shot('chat-source-1280', details); mark('Real local Chat receipt displays exclusive categories and original field sources');
|
|
|
|
const responses = rowFor(scenes[1]); await responses.locator('.model-usage-details > summary').click(); const responsesDetails = responses.locator('.model-usage-details');
|
|
await expect(metric(responsesDetails, '缓存写入')).toHaveText('7,000 tokens'); await expect(metric(responsesDetails, '未缓存输入')).toHaveText('1,000 tokens');
|
|
await expect(responsesDetails).toContainText('按上游用量计算'); await expect(metric(responsesDetails, '30 分钟写入')).toHaveText('3,000 tokens');
|
|
await expect(metric(responsesDetails, '12 小时写入')).toHaveText('4,000 tokens'); await originalDownload(scenes[1], responses);
|
|
await resize(390, 844); await shot('responses-ttl-390', responsesDetails); mark('Responses SSE derived ordinary input and two lifetime categories fit 390px; actual downloaded bytes match the original');
|
|
|
|
const messages = rowFor(scenes[2]); await messages.locator('.model-usage-details > summary').click(); const messagesDetails = messages.locator('.model-usage-details');
|
|
await expect(metric(messagesDetails, '5 分钟写入')).toHaveText('148 tokens'); await expect(metric(messagesDetails, '1 小时写入')).toHaveText('100 tokens');
|
|
await expect(metric(messagesDetails, '推理输出')).toHaveText('上游未报告'); await expect(metric(messagesDetails, '未缓存输入')).toHaveText('50 tokens');
|
|
await originalDownload(scenes[2], messages); await resize(1280); await shot('messages-ttl-1280', messagesDetails);
|
|
mark('Messages SSE separates uncached/read/write and 5m/1h; absent reasoning stays unknown and original download remains exact');
|
|
|
|
const missing = rowFor(scenes[3]); await expect(missing).toContainText('用量未知'); assert.equal(await missing.locator('.model-usage-details').count(), 0);
|
|
const bad = rowFor(scenes[4]); await bad.locator('.model-usage-details > summary').click(); await bad.locator('.model-usage-sources > summary').click();
|
|
await expect(bad).toContainText('上游用量字段存在不一致'); await expect(bad).toContainText('输入输出合计不一致');
|
|
const error = rowFor(scenes[5]); await error.locator('.model-usage-details > summary').click(); await expect(error).toContainText('自有上游原错误');
|
|
await expect(metric(error.locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens');
|
|
await resize(390, 844); await shot('upstream-error-390', error.locator('.model-usage-details'));
|
|
const legacy = center().locator('.mc-attempts article').filter({ hasText: 'owned-legacy-run' });
|
|
await legacy.locator('.model-usage-details > summary').focus(); await page.keyboard.press('Enter'); await expect(metric(legacy.locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens');
|
|
assert.deepEqual((await host.api('/model-center')).attempts.find(value => value.id === historical.id), historical);
|
|
mark('Unknown, conflicting, failed and legacy receipts remain distinct; keyboard expansion works without rewriting legacy records or retrying');
|
|
await closeDialog(); await resize(1280); await viewContext(scenes[2]);
|
|
await expect(metric(panel().locator('.model-usage-details'), '缓存读取')).toHaveText('100 tokens');
|
|
await panel().locator('.model-usage-sources > summary').click(); await expect(panel()).toContainText('usage.cache_creation.ephemeral_1h_input_tokens');
|
|
await shot('context-messages-source-1280', panel().locator('.model-usage-details')); await closeDialog();
|
|
await resize(390, 844); await viewContext(scenes[5]);
|
|
await expect(panel()).toContainText('请求失败'); await expect(metric(panel().locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens');
|
|
await shot('context-error-390', panel().locator('.model-usage-details')); await closeDialog();
|
|
await viewContext(scenes[3]); await expect(panel()).toContainText('服务未返回'); assert.equal(await panel().locator('.model-usage-details').count(), 0); await closeDialog();
|
|
await ledger(); assert.deepEqual((await host.api('/model-center')).attempts.find(value => value.id === historical.id), historical);
|
|
for (const scene of scenes) assert.equal(scene.model.requests.length, 1);
|
|
assert.deepEqual(errors, []); assert.deepEqual(externalBlocked, []);
|
|
mark('Existing task record and reload show accurate success/error/unknown context usage at 1280/390 with no additional model dispatch');
|
|
const config = (await host.api('/bootstrap')).config; assert.equal(config.executionMode, 'unrestricted'); assert.equal(config.capacityMode, 'passthrough'); assert.equal(config.historyPolicy, 'preserve');
|
|
fs.writeFileSync(path.join(directory, 'final-model-center.json'), JSON.stringify(await host.api('/model-center'), null, 2));
|
|
result = { success: true, appRoot, actualElectron: desktop, checks, screenshots, downloads, errors, externalBlocked, hostUrl: host.url,
|
|
syntheticRequests: scenes.reduce((sum, scene) => sum + scene.model.requests.length, 0), realModelRequests: 0, realCredentialsRead: 0, productionWrites: 0, viewportWidths: [1280, 390] };
|
|
} catch (cause) {
|
|
if (page) { await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {}); fs.writeFileSync(path.join(directory, 'failure.txt'), await page.locator('body').innerText().catch(() => '')); }
|
|
result = { success: false, appRoot, actualElectron: desktop, checks, screenshots, errors, externalBlocked, error: cause.stack,
|
|
syntheticRequests: scenes.reduce((sum, scene) => sum + scene.model.requests.length, 0), realModelRequests: 0 };
|
|
} finally {
|
|
fs.writeFileSync(path.join(directory, 'requests.json'), JSON.stringify(scenes.map(scene => ({ name: scene.name, requests: scene.model.requests })), null, 2));
|
|
await app?.close(); await browser?.close(); if (controller && await controller.health()) await controller.stop(); await host?.close?.();
|
|
for (const dispose of disposers.reverse()) await dispose();
|
|
if (desktop) { fs.writeFileSync(path.join(directory, 'desktop-stdout.txt'), desktopStdout); fs.writeFileSync(path.join(directory, 'desktop-stderr.txt'), desktopStderr); }
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
|
}
|
|
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;
|