import fs from 'node:fs'; import path from 'node:path'; import net from 'node:net'; import assert from 'node:assert/strict'; import { randomUUID, createHash } from 'node:crypto'; import { chromium, _electron } from 'playwright'; import { expect } from 'playwright/test'; import { fixture, launchWorkbench, until } from '../tests/helpers.mjs'; import { Provider } from '../src/server/provider.mjs'; import { HostController } from '../src/core/desktop/host-controller.mjs'; import { desktopLaunchCommand } from './desktop-launch.mjs'; import { projectEnvironment } from '../src/core/process-environment.mjs'; import { usageSamples, usageModelId, cacheUsageFixture } from '../tests/fixtures/cache-usage-model.mjs'; import { openWorkbenchTool } from './ui-navigation.mjs'; assert.ok(process.argv.slice(2).every(value => value === '--desktop')); assert.ok(process.env.AGENT_WORKBENCH_QA_APP_ROOT, 'Use a frozen independent candidate'); const desktop = process.argv.includes('--desktop'), appRoot = fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT); const directory = path.resolve('.qa/m2-cache-usage-2026-11-07', (desktop ? 'electron-' : 'web-') + randomUUID()); fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory); const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true }); const hub = Object.assign(Object.create(Provider.prototype), { apiKey: '' }).modelHub(f.store); const historical = { id: randomUUID(), connectionId: 'owned-legacy-connection', runId: 'owned-legacy-run', ordinal: 1, model: 'owned-historical-cache', protocol: 'chat-completions', status: 'completed', startedAt: '2026-11-01T00:00:00Z', usageStatus: 'reported', usage: { prompt_tokens: 1000, completion_tokens: 100, total_tokens: 1100, protocol: 'chat-completions', raw: structuredClone(usageSamples.chat.raw) } }; hub.attempts.push(historical); hub.saveAttempts(); const conflict = structuredClone(usageSamples.responses); conflict.raw.total_tokens = 1; const scenes = [ { name: 'chat', ...usageSamples.chat }, { name: 'responses', ...usageSamples.responses, streaming: true }, { name: 'messages', ...usageSamples.messages, streaming: true }, { name: 'missing', protocol: 'chat-completions' }, { name: 'conflict', ...conflict }, { name: 'upstream-error', ...usageSamples.chat, status: 400 }, ]; for (const scene of scenes) { scene.model = await cacheUsageFixture(t, scene); scene.connection = hub.upsert({ name: '自有用量验证 · ' + scene.name, model: usageModelId, protocol: scene.protocol, streaming: !!scene.streaming, baseUrl: scene.model.baseUrl, requiresKey: false, modelIdentityPolicy: 'require-match' }); } f.store.close(); const checks = [], screenshots = [], errors = [], externalBlocked = [], downloads = []; let host, controller, browser, app, page, window, result, desktopStdout = '', desktopStderr = ''; const sha = bytes => createHash('sha256').update(bytes).digest('hex'); const mark = value => { checks.push(value); console.log('PASS ' + value); }; const center = () => page.getByRole('dialog', { name: '模型中心', exact: true }); const panel = () => page.getByLabel('请求上下文', { exact: true }); const metric = (scope, name) => scope.locator(':scope > dl > div').filter({ has: page.locator('dt').filter({ hasText: name }) }).locator('dd'); async function closeDialog() { if (await page.getByRole('dialog').count()) await page.getByRole('dialog').getByRole('button', { name: '关闭窗口', exact: true }).click(); else await page.getByRole('button', { name: '关闭检视区', exact: true }).click(); } async function resize(width, height = 960) { if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); } else await page.setViewportSize({ width, height }); } async function shot(name, scope) { await scope.scrollIntoViewIfNeeded(); assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), 'No unintended page overflow'); const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push({ file: path.basename(file), sha256: sha(fs.readFileSync(file)) }); fs.writeFileSync(path.join(directory, name + '.txt'), await scope.innerText()); } async function ledger() { await openWorkbenchTool(page, '打开模型中心'); await center().getByRole('button', { name: '请求账本', exact: true }).click(); } function rowFor(scene) { return center().locator('.mc-attempts article').filter({ has: page.locator('code').filter({ hasText: 'Run ' + scene.session.runId }) }); } async function originalDownload(scene, row) { const file = path.join(directory, scene.name + '-original-response.sse'); let native = null; if (app) { const id = randomUUID(); await app.evaluate(({ session }, { id, file }) => { globalThis.workbenchQaDownloads ??= {}; globalThis.workbenchQaDownloads[id] = { state: 'waiting' }; session.defaultSession.once('will-download', (_event, item) => { const value = globalThis.workbenchQaDownloads[id]; value.suggestedFilename = item.getFilename(); value.state = 'started'; item.setSavePath(file); item.once('done', (_event, state) => { value.state = state; value.bytes = item.getReceivedBytes(); value.totalBytes = item.getTotalBytes(); }); }); }, { id, file }); await row.getByRole('button', { name: '下载已保存的响应', exact: true }).click(); await expect.poll(() => app.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id].state, id), { timeout: 30000 }).toBe('completed'); native = await app.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id], id); assert.equal(native.bytes, native.totalBytes); assert.equal(native.bytes, fs.statSync(file).size); } else { const downloadEvent = page.waitForEvent('download'); await row.getByRole('button', { name: '下载已保存的响应', exact: true }).click(); const download = await downloadEvent; await download.saveAs(file); } assert.equal(sha(fs.readFileSync(file)), sha(scene.model.responses[0])); downloads.push({ file: path.basename(file), sha256: sha(fs.readFileSync(file)), ...(native ? { native, scope: 'Actual Electron DownloadItem; QA selects owned path, not the Windows save dialog' } : { scope: 'Actual browser download' }) }); } async function viewContext(scene) { await page.evaluate(id => localStorage.setItem('awb-session', id), scene.session.id); await page.reload(); await page.waitForLoadState('networkidle'); await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await panel().waitFor({ state: 'visible' }); } try { if (desktop) { const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve)); assert.notEqual(port, 4318); controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start(); host = { url: controller.url, api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) }; app = await _electron.launch({ ...desktopLaunchCommand(appRoot, ['--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window']), cwd: appRoot, env: projectEnvironment(), timeout: 60000 }); app.process().stdout?.on('data', bytes => { desktopStdout += bytes.toString('utf8'); }); app.process().stderr?.on('data', bytes => { desktopStderr += bytes.toString('utf8'); }); page = await app.firstWindow(); window = await app.browserWindow(page); await window.evaluate(win => win.showInactive()); } else { host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318'); browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ acceptDownloads: true }); } page.on('pageerror', error => errors.push(error.message)); await page.route('**/*', route => { const url = new URL(route.request().url()); if (url.origin === new URL(host.url).origin || ['data:', 'blob:'].includes(url.protocol)) return route.continue(); externalBlocked.push(url.origin); return route.abort(); }); await page.addInitScript(id => localStorage.setItem('awb-project', id), f.project.id); for (const scene of scenes) { await host.api('/model-center/select', { type: 'connection', id: scene.connection.id }); scene.session = await host.api('/sessions', { projectId: f.project.id }); await host.api('/sessions/' + scene.session.id + '/messages', { content: '自有用量样本 ' + scene.name + ':只说明收到完整回复,不修改文件或调用工具。' }); await until(async () => ['completed', 'failed'].includes((await host.api('/sessions/' + scene.session.id)).status), 30000); scene.session = await host.api('/sessions/' + scene.session.id); scene.plan = (await host.api('/sessions/' + scene.session.id + '/context-plans'))[0]; scene.attempt = (await host.api('/model-center')).attempts.find(value => value.sessionId === scene.session.id); assert.equal(scene.model.requests.length, 1); assert.equal(scene.session.execution.progress.totalTools, 0); assert.equal(scene.session.status, scene.status === 400 ? 'failed' : 'completed'); assert.deepEqual(scene.plan.actualUsage?.raw, scene.raw); fs.writeFileSync(path.join(directory, scene.name + '-receipt.json'), JSON.stringify({ session: scene.session, plan: scene.plan, attempt: scene.attempt }, null, 2)); fs.writeFileSync(path.join(directory, scene.name + '-original-response.bin'), scene.model.responses[0]); } await resize(1280); await page.goto(host.url); await page.waitForLoadState('networkidle'); await ledger(); const chat = rowFor(scenes[0]); await chat.locator('.model-usage-details > summary').click(); const details = chat.locator('.model-usage-details'); await expect(metric(details, '缓存读取')).toHaveText('600 tokens'); await expect(metric(details, '缓存写入')).toHaveText('300 tokens'); await expect(metric(details, '未缓存输入')).toHaveText('100 tokens'); await expect(metric(details, '推理输出')).toHaveText('40 tokens'); await details.locator('.model-usage-sources > summary').click(); await expect(details).toContainText('usage.prompt_tokens_details.cache_write_tokens'); await shot('chat-source-1280', details); mark('Real local Chat receipt displays exclusive categories and original field sources'); const responses = rowFor(scenes[1]); await responses.locator('.model-usage-details > summary').click(); const responsesDetails = responses.locator('.model-usage-details'); await expect(metric(responsesDetails, '缓存写入')).toHaveText('7,000 tokens'); await expect(metric(responsesDetails, '未缓存输入')).toHaveText('1,000 tokens'); await expect(responsesDetails).toContainText('按上游用量计算'); await expect(metric(responsesDetails, '30 分钟写入')).toHaveText('3,000 tokens'); await expect(metric(responsesDetails, '12 小时写入')).toHaveText('4,000 tokens'); await originalDownload(scenes[1], responses); await resize(390, 844); await shot('responses-ttl-390', responsesDetails); mark('Responses SSE derived ordinary input and two lifetime categories fit 390px; actual downloaded bytes match the original'); const messages = rowFor(scenes[2]); await messages.locator('.model-usage-details > summary').click(); const messagesDetails = messages.locator('.model-usage-details'); await expect(metric(messagesDetails, '5 分钟写入')).toHaveText('148 tokens'); await expect(metric(messagesDetails, '1 小时写入')).toHaveText('100 tokens'); await expect(metric(messagesDetails, '推理输出')).toHaveText('上游未报告'); await expect(metric(messagesDetails, '未缓存输入')).toHaveText('50 tokens'); await originalDownload(scenes[2], messages); await resize(1280); await shot('messages-ttl-1280', messagesDetails); mark('Messages SSE separates uncached/read/write and 5m/1h; absent reasoning stays unknown and original download remains exact'); const missing = rowFor(scenes[3]); await expect(missing).toContainText('用量未知'); assert.equal(await missing.locator('.model-usage-details').count(), 0); const bad = rowFor(scenes[4]); await bad.locator('.model-usage-details > summary').click(); await bad.locator('.model-usage-sources > summary').click(); await expect(bad).toContainText('上游用量字段存在不一致'); await expect(bad).toContainText('输入输出合计不一致'); const error = rowFor(scenes[5]); await error.locator('.model-usage-details > summary').click(); await expect(error).toContainText('自有上游原错误'); await expect(metric(error.locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens'); await resize(390, 844); await shot('upstream-error-390', error.locator('.model-usage-details')); const legacy = center().locator('.mc-attempts article').filter({ hasText: 'owned-legacy-run' }); await legacy.locator('.model-usage-details > summary').focus(); await page.keyboard.press('Enter'); await expect(metric(legacy.locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens'); assert.deepEqual((await host.api('/model-center')).attempts.find(value => value.id === historical.id), historical); mark('Unknown, conflicting, failed and legacy receipts remain distinct; keyboard expansion works without rewriting legacy records or retrying'); await closeDialog(); await resize(1280); await viewContext(scenes[2]); await expect(metric(panel().locator('.model-usage-details'), '缓存读取')).toHaveText('100 tokens'); await panel().locator('.model-usage-sources > summary').click(); await expect(panel()).toContainText('usage.cache_creation.ephemeral_1h_input_tokens'); await shot('context-messages-source-1280', panel().locator('.model-usage-details')); await closeDialog(); await resize(390, 844); await viewContext(scenes[5]); await expect(panel()).toContainText('请求失败'); await expect(metric(panel().locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens'); await shot('context-error-390', panel().locator('.model-usage-details')); await closeDialog(); await viewContext(scenes[3]); await expect(panel()).toContainText('服务未返回'); assert.equal(await panel().locator('.model-usage-details').count(), 0); await closeDialog(); await ledger(); assert.deepEqual((await host.api('/model-center')).attempts.find(value => value.id === historical.id), historical); for (const scene of scenes) assert.equal(scene.model.requests.length, 1); assert.deepEqual(errors, []); assert.deepEqual(externalBlocked, []); mark('Existing task record and reload show accurate success/error/unknown context usage at 1280/390 with no additional model dispatch'); const config = (await host.api('/bootstrap')).config; assert.equal(config.executionMode, 'unrestricted'); assert.equal(config.capacityMode, 'passthrough'); assert.equal(config.historyPolicy, 'preserve'); fs.writeFileSync(path.join(directory, 'final-model-center.json'), JSON.stringify(await host.api('/model-center'), null, 2)); result = { success: true, appRoot, actualElectron: desktop, checks, screenshots, downloads, errors, externalBlocked, hostUrl: host.url, syntheticRequests: scenes.reduce((sum, scene) => sum + scene.model.requests.length, 0), realModelRequests: 0, realCredentialsRead: 0, productionWrites: 0, viewportWidths: [1280, 390] }; } catch (cause) { if (page) { await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {}); fs.writeFileSync(path.join(directory, 'failure.txt'), await page.locator('body').innerText().catch(() => '')); } result = { success: false, appRoot, actualElectron: desktop, checks, screenshots, errors, externalBlocked, error: cause.stack, syntheticRequests: scenes.reduce((sum, scene) => sum + scene.model.requests.length, 0), realModelRequests: 0 }; } finally { fs.writeFileSync(path.join(directory, 'requests.json'), JSON.stringify(scenes.map(scene => ({ name: scene.name, requests: scene.model.requests })), null, 2)); await app?.close(); await browser?.close(); if (controller && await controller.health()) await controller.stop(); await host?.close?.(); for (const dispose of disposers.reverse()) await dispose(); if (desktop) { fs.writeFileSync(path.join(directory, 'desktop-stdout.txt'), desktopStdout); fs.writeFileSync(path.join(directory, 'desktop-stderr.txt'), desktopStderr); } fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); } console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;