diff --git a/scripts/verify-model-usage-ui.mjs b/scripts/verify-model-usage-ui.mjs new file mode 100644 index 00000000..48dae4bf --- /dev/null +++ b/scripts/verify-model-usage-ui.mjs @@ -0,0 +1,182 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import net from 'node:net'; +import assert from 'node:assert/strict'; +import { randomUUID, createHash } from 'node:crypto'; +import { chromium, _electron } from 'playwright'; +import { expect } from 'playwright/test'; +import { fixture, launchWorkbench, until } from '../tests/helpers.mjs'; +import { Provider } from '../src/server/provider.mjs'; +import { HostController } from '../src/core/desktop/host-controller.mjs'; +import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs'; +import { projectEnvironment } from '../src/core/process-environment.mjs'; +import { usageSamples, usageModelId, cacheUsageFixture } from '../tests/fixtures/cache-usage-model.mjs'; +import { openWorkbenchTool } from './ui-navigation.mjs'; + +assert.ok(process.argv.slice(2).every(value => value === '--desktop')); +assert.ok(process.env.AGENT_WORKBENCH_QA_APP_ROOT, 'Use a frozen independent candidate'); +const desktop = process.argv.includes('--desktop'), appRoot = fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT); +const directory = path.resolve('.qa/m2-cache-usage-2026-11-07', (desktop ? 'electron-' : 'web-') + randomUUID()); +fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory); +const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true }); +const hub = Object.assign(Object.create(Provider.prototype), { apiKey: '' }).modelHub(f.store); +const historical = { id: randomUUID(), connectionId: 'owned-legacy-connection', runId: 'owned-legacy-run', ordinal: 1, + model: 'owned-historical-cache', protocol: 'chat-completions', status: 'completed', startedAt: '2026-11-01T00:00:00Z', usageStatus: 'reported', + usage: { prompt_tokens: 1000, completion_tokens: 100, total_tokens: 1100, protocol: 'chat-completions', raw: structuredClone(usageSamples.chat.raw) } }; +hub.attempts.push(historical); hub.saveAttempts(); +const conflict = structuredClone(usageSamples.responses); conflict.raw.total_tokens = 1; +const scenes = [ + { name: 'chat', ...usageSamples.chat }, { name: 'responses', ...usageSamples.responses, streaming: true }, + { name: 'messages', ...usageSamples.messages, streaming: true }, { name: 'missing', protocol: 'chat-completions' }, + { name: 'conflict', ...conflict }, { name: 'upstream-error', ...usageSamples.chat, status: 400 }, +]; +for (const scene of scenes) { + scene.model = await cacheUsageFixture(t, scene); + scene.connection = hub.upsert({ name: '自有用量验证 · ' + scene.name, model: usageModelId, protocol: scene.protocol, streaming: !!scene.streaming, + baseUrl: scene.model.baseUrl, requiresKey: false, modelIdentityPolicy: 'require-match' }); +} +f.store.close(); +const checks = [], screenshots = [], errors = [], externalBlocked = [], downloads = []; +let host, controller, browser, app, page, window, result, desktopStdout = '', desktopStderr = ''; +const sha = bytes => createHash('sha256').update(bytes).digest('hex'); +const mark = value => { checks.push(value); console.log('PASS ' + value); }; +const center = () => page.getByRole('dialog', { name: '模型中心', exact: true }); +const panel = () => page.getByLabel('请求上下文', { exact: true }); +const metric = (scope, name) => scope.locator(':scope > dl > div').filter({ has: page.locator('dt').filter({ hasText: name }) }).locator('dd'); +async function closeDialog() { + if (await page.getByRole('dialog').count()) await page.getByRole('dialog').getByRole('button', { name: '关闭窗口', exact: true }).click(); + else await page.getByRole('button', { name: '关闭检视区', exact: true }).click(); +} +async function resize(width, height = 960) { + if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); } + else await page.setViewportSize({ width, height }); +} +async function shot(name, scope) { + await scope.scrollIntoViewIfNeeded(); + assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), 'No unintended page overflow'); + const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); + screenshots.push({ file: path.basename(file), sha256: sha(fs.readFileSync(file)) }); + fs.writeFileSync(path.join(directory, name + '.txt'), await scope.innerText()); +} +async function ledger() { await openWorkbenchTool(page, '打开模型中心'); await center().getByRole('button', { name: '请求账本', exact: true }).click(); } +function rowFor(scene) { return center().locator('.mc-attempts article').filter({ has: page.locator('code').filter({ hasText: 'Run ' + scene.session.runId }) }); } +async function originalDownload(scene, row) { + const file = path.join(directory, scene.name + '-original-response.sse'); let native = null; + if (app) { + const id = randomUUID(); + await app.evaluate(({ session }, { id, file }) => { + globalThis.workbenchQaDownloads ??= {}; globalThis.workbenchQaDownloads[id] = { state: 'waiting' }; + session.defaultSession.once('will-download', (_event, item) => { + const value = globalThis.workbenchQaDownloads[id]; value.suggestedFilename = item.getFilename(); value.state = 'started'; + item.setSavePath(file); + item.once('done', (_event, state) => { value.state = state; value.bytes = item.getReceivedBytes(); value.totalBytes = item.getTotalBytes(); }); + }); + }, { id, file }); + await row.getByRole('button', { name: '下载已保存的响应', exact: true }).click(); + await expect.poll(() => app.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id].state, id), { timeout: 30000 }).toBe('completed'); + native = await app.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id], id); + assert.equal(native.bytes, native.totalBytes); assert.equal(native.bytes, fs.statSync(file).size); + } else { + const downloadEvent = page.waitForEvent('download'); + await row.getByRole('button', { name: '下载已保存的响应', exact: true }).click(); + const download = await downloadEvent; await download.saveAs(file); + } + assert.equal(sha(fs.readFileSync(file)), sha(scene.model.responses[0])); downloads.push({ file: path.basename(file), sha256: sha(fs.readFileSync(file)), + ...(native ? { native, scope: 'Actual Electron DownloadItem; QA selects owned path, not the Windows save dialog' } : { scope: 'Actual browser download' }) }); +} +async function viewContext(scene) { + await page.evaluate(id => localStorage.setItem('awb-session', id), scene.session.id); await page.reload(); await page.waitForLoadState('networkidle'); + await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await panel().waitFor({ state: 'visible' }); +} +try { + if (desktop) { + const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve)); const port = socket.address().port; await new Promise(resolve => socket.close(resolve)); + assert.notEqual(port, 4318); controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 }); await controller.start(); + host = { url: controller.url, api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) }; + app = await _electron.launch({ executablePath: desktopExecutableIdentity(appRoot).path, + args: [path.join(appRoot, 'dist-desktop/main.cjs'), '--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window'], + cwd: appRoot, env: projectEnvironment(), timeout: 60000 }); + app.process().stdout?.on('data', bytes => { desktopStdout += bytes.toString('utf8'); }); app.process().stderr?.on('data', bytes => { desktopStderr += bytes.toString('utf8'); }); + page = await app.firstWindow(); window = await app.browserWindow(page); await window.evaluate(win => win.showInactive()); + } else { + host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318'); + browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ acceptDownloads: true }); + } + page.on('pageerror', error => errors.push(error.message)); + await page.route('**/*', route => { + const url = new URL(route.request().url()); + if (url.origin === new URL(host.url).origin || ['data:', 'blob:'].includes(url.protocol)) return route.continue(); + externalBlocked.push(url.origin); return route.abort(); + }); + await page.addInitScript(id => localStorage.setItem('awb-project', id), f.project.id); + for (const scene of scenes) { + await host.api('/model-center/select', { type: 'connection', id: scene.connection.id }); + scene.session = await host.api('/sessions', { projectId: f.project.id }); + await host.api('/sessions/' + scene.session.id + '/messages', { content: '自有用量样本 ' + scene.name + ':只说明收到完整回复,不修改文件或调用工具。' }); + await until(async () => ['completed', 'failed'].includes((await host.api('/sessions/' + scene.session.id)).status), 30000); + scene.session = await host.api('/sessions/' + scene.session.id); + scene.plan = (await host.api('/sessions/' + scene.session.id + '/context-plans'))[0]; + scene.attempt = (await host.api('/model-center')).attempts.find(value => value.sessionId === scene.session.id); + assert.equal(scene.model.requests.length, 1); assert.equal(scene.session.execution.progress.totalTools, 0); + assert.equal(scene.session.status, scene.status === 400 ? 'failed' : 'completed'); + assert.deepEqual(scene.plan.actualUsage?.raw, scene.raw); + fs.writeFileSync(path.join(directory, scene.name + '-receipt.json'), JSON.stringify({ session: scene.session, plan: scene.plan, attempt: scene.attempt }, null, 2)); + fs.writeFileSync(path.join(directory, scene.name + '-original-response.bin'), scene.model.responses[0]); + } + await resize(1280); await page.goto(host.url); await page.waitForLoadState('networkidle'); await ledger(); + const chat = rowFor(scenes[0]); await chat.locator('.model-usage-details > summary').click(); const details = chat.locator('.model-usage-details'); + await expect(metric(details, '缓存读取')).toHaveText('600 tokens'); await expect(metric(details, '缓存写入')).toHaveText('300 tokens'); + await expect(metric(details, '未缓存输入')).toHaveText('100 tokens'); await expect(metric(details, '推理输出')).toHaveText('40 tokens'); + await details.locator('.model-usage-sources > summary').click(); await expect(details).toContainText('usage.prompt_tokens_details.cache_write_tokens'); + await shot('chat-source-1280', details); mark('Real local Chat receipt displays exclusive categories and original field sources'); + + const responses = rowFor(scenes[1]); await responses.locator('.model-usage-details > summary').click(); const responsesDetails = responses.locator('.model-usage-details'); + await expect(metric(responsesDetails, '缓存写入')).toHaveText('7,000 tokens'); await expect(metric(responsesDetails, '未缓存输入')).toHaveText('1,000 tokens'); + await expect(responsesDetails).toContainText('按上游用量计算'); await expect(metric(responsesDetails, '30 分钟写入')).toHaveText('3,000 tokens'); + await expect(metric(responsesDetails, '12 小时写入')).toHaveText('4,000 tokens'); await originalDownload(scenes[1], responses); + await resize(390, 844); await shot('responses-ttl-390', responsesDetails); mark('Responses SSE derived ordinary input and two lifetime categories fit 390px; actual downloaded bytes match the original'); + + const messages = rowFor(scenes[2]); await messages.locator('.model-usage-details > summary').click(); const messagesDetails = messages.locator('.model-usage-details'); + await expect(metric(messagesDetails, '5 分钟写入')).toHaveText('148 tokens'); await expect(metric(messagesDetails, '1 小时写入')).toHaveText('100 tokens'); + await expect(metric(messagesDetails, '推理输出')).toHaveText('上游未报告'); await expect(metric(messagesDetails, '未缓存输入')).toHaveText('50 tokens'); + await originalDownload(scenes[2], messages); await resize(1280); await shot('messages-ttl-1280', messagesDetails); + mark('Messages SSE separates uncached/read/write and 5m/1h; absent reasoning stays unknown and original download remains exact'); + + const missing = rowFor(scenes[3]); await expect(missing).toContainText('用量未知'); assert.equal(await missing.locator('.model-usage-details').count(), 0); + const bad = rowFor(scenes[4]); await bad.locator('.model-usage-details > summary').click(); await bad.locator('.model-usage-sources > summary').click(); + await expect(bad).toContainText('上游用量字段存在不一致'); await expect(bad).toContainText('输入输出合计不一致'); + const error = rowFor(scenes[5]); await error.locator('.model-usage-details > summary').click(); await expect(error).toContainText('自有上游原错误'); + await expect(metric(error.locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens'); + await resize(390, 844); await shot('upstream-error-390', error.locator('.model-usage-details')); + const legacy = center().locator('.mc-attempts article').filter({ hasText: 'owned-legacy-run' }); + await legacy.locator('.model-usage-details > summary').focus(); await page.keyboard.press('Enter'); await expect(metric(legacy.locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens'); + assert.deepEqual((await host.api('/model-center')).attempts.find(value => value.id === historical.id), historical); + mark('Unknown, conflicting, failed and legacy receipts remain distinct; keyboard expansion works without rewriting legacy records or retrying'); + await closeDialog(); await resize(1280); await viewContext(scenes[2]); + await expect(metric(panel().locator('.model-usage-details'), '缓存读取')).toHaveText('100 tokens'); + await panel().locator('.model-usage-sources > summary').click(); await expect(panel()).toContainText('usage.cache_creation.ephemeral_1h_input_tokens'); + await shot('context-messages-source-1280', panel().locator('.model-usage-details')); await closeDialog(); + await resize(390, 844); await viewContext(scenes[5]); + await expect(panel()).toContainText('请求失败'); await expect(metric(panel().locator('.model-usage-details'), '缓存写入')).toHaveText('300 tokens'); + await shot('context-error-390', panel().locator('.model-usage-details')); await closeDialog(); + await viewContext(scenes[3]); await expect(panel()).toContainText('服务未返回'); assert.equal(await panel().locator('.model-usage-details').count(), 0); await closeDialog(); + await ledger(); assert.deepEqual((await host.api('/model-center')).attempts.find(value => value.id === historical.id), historical); + for (const scene of scenes) assert.equal(scene.model.requests.length, 1); + assert.deepEqual(errors, []); assert.deepEqual(externalBlocked, []); + mark('Existing task record and reload show accurate success/error/unknown context usage at 1280/390 with no additional model dispatch'); + const config = (await host.api('/bootstrap')).config; assert.equal(config.executionMode, 'unrestricted'); assert.equal(config.capacityMode, 'passthrough'); assert.equal(config.historyPolicy, 'preserve'); + fs.writeFileSync(path.join(directory, 'final-model-center.json'), JSON.stringify(await host.api('/model-center'), null, 2)); + result = { success: true, appRoot, actualElectron: desktop, checks, screenshots, downloads, errors, externalBlocked, hostUrl: host.url, + syntheticRequests: scenes.reduce((sum, scene) => sum + scene.model.requests.length, 0), realModelRequests: 0, realCredentialsRead: 0, productionWrites: 0, viewportWidths: [1280, 390] }; +} catch (cause) { + if (page) { await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {}); fs.writeFileSync(path.join(directory, 'failure.txt'), await page.locator('body').innerText().catch(() => '')); } + result = { success: false, appRoot, actualElectron: desktop, checks, screenshots, errors, externalBlocked, error: cause.stack, + syntheticRequests: scenes.reduce((sum, scene) => sum + scene.model.requests.length, 0), realModelRequests: 0 }; +} finally { + fs.writeFileSync(path.join(directory, 'requests.json'), JSON.stringify(scenes.map(scene => ({ name: scene.name, requests: scene.model.requests })), null, 2)); + await app?.close(); await browser?.close(); if (controller && await controller.health()) await controller.stop(); await host?.close?.(); + for (const dispose of disposers.reverse()) await dispose(); + if (desktop) { fs.writeFileSync(path.join(directory, 'desktop-stdout.txt'), desktopStdout); fs.writeFileSync(path.join(directory, 'desktop-stderr.txt'), desktopStderr); } + fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); +} +console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1; diff --git a/src/client/ModelCenter.tsx b/src/client/ModelCenter.tsx index cbfdd25a..0145b2c3 100644 --- a/src/client/ModelCenter.tsx +++ b/src/client/ModelCenter.tsx @@ -3,6 +3,8 @@ import { Activity, Check, CircleHelp, KeyRound, Layers3, Plus, Power, RefreshCw, import { api, downloadArtifact } from './api'; import type { Config, ModelSelection, ModelSnapshot, RuntimeFailure } from './types'; import './ModelCenter.css'; +import { ModelUsageDetails } from './ModelUsageDetails'; +import type { TokenUsage } from '../shared/contracts/model-usage.mjs'; import { ModelIdentityDetails } from './ModelIdentityDetails'; import { ModelRoutingDetails, type RoutingDecision } from './ModelRoutingDetails'; import type { ModelIdentity } from '../shared/contracts/model-identity.mjs'; @@ -25,7 +27,7 @@ const upstreamCapacity = (value: { model: string; upstreamModel?: CatalogModel } type Connection = { id: string; name: string; brand: string; protocol: string; parameterProfile?: string; messagesAuth?: string; baseUrl: string; model: string; enabled: boolean; revision: number; hasKey: boolean; credentialSource?: { connectionId: string }; upstreamModel?: CatalogModel; requiresKey: boolean; modelIdentityPolicy?: string; modelIdentityAliases?: string[]; modelIdentityAliasSource?: string; nativeParameters?: Record; outputTokenField?: string; streaming?: boolean; capabilityDeclarations?: Record; capabilities?: Record; capabilityEvidence?: { checkedAt: string }; contextLimit: number; maxOutputTokens: number; concurrency: number; status: string; activeRequests: number; savedCredential?: { status: string }; health: { state: string; reason?: string; cooldownUntil?: number } }; type Pool = { id?: string; name: string; members: { connectionId: string; weight: number }[]; strategy: string; sticky: boolean; maxAttempts: number; revision?: number; queueTimeoutMs: number }; -type Attempt = { routingDecision?: RoutingDecision; modelIdentity?: ModelIdentity; diagnostic?: RuntimeFailure; requestUrl?: string; upstreamResponse?: { requestId?: string | null; requestUrl?: string; responseUrl?: string | null; redirectMode?: string }; id: string; connectionId: string; poolId: string | null; model: string; protocol?: string; streaming?: boolean; rejections?: { connectionName: string; capability: string; reason: string }[]; status: string; startedAt: string; ordinal: number; usageStatus: string; reason?: string; taskId?: string; sessionId?: string; runId: string; requestArtifactId?: string; responseArtifactId?: string; responseSha256?: string; responseRedacted?: boolean; stream?: { bytes: number; capturedBytes?: number; frames: number; sha256: string; terminal: boolean }; usage?: { prompt_tokens?: number; completion_tokens?: number } }; +type Attempt = { routingDecision?: RoutingDecision; modelIdentity?: ModelIdentity; diagnostic?: RuntimeFailure; requestUrl?: string; upstreamResponse?: { requestId?: string | null; requestUrl?: string; responseUrl?: string | null; redirectMode?: string }; id: string; connectionId: string; poolId: string | null; model: string; protocol?: string; streaming?: boolean; rejections?: { connectionName: string; capability: string; reason: string }[]; status: string; startedAt: string; ordinal: number; usageStatus: string; reason?: string; taskId?: string; sessionId?: string; runId: string; requestArtifactId?: string; responseArtifactId?: string; responseSha256?: string; responseRedacted?: boolean; stream?: { bytes: number; capturedBytes?: number; frames: number; sha256: string; terminal: boolean }; usage?: TokenUsage | null }; type ImportRecord = { id: string; source: string; path: string; enabled: boolean; lastCheckedAt?: string; lastSyncedAt?: string; lastError?: string; lastResult?: { imported?: number; conflicts?: string[] } }; type Center = { revision: number; selected: { type: string; id: string }; waitSettings: WaitSettings; connections: Connection[]; pools: Pool[]; imports: ImportRecord[]; attempts: Attempt[]; waiting: number }; type Operations = { checkedAt: string; version: string; scope: string; activeTasks: number; queue: number; checks: { id: string; title: string; status: string; detail: string }[]; @@ -205,7 +207,7 @@ export function ModelCenter({ onConfig, busy, runModel }: { onConfig: (value: Co
模型池成员与权重{connections.map(connection => { const member = pool.members.find(value => value.connectionId === connection.id); return ; })}
} } - {tab === 'attempts' && <>

请求账本

最近100次物理请求;最多保存1000次。未知用量保持未知。

{view.attempts.length} 条记录
{!view.attempts.length &&

当前没有模型中心请求。选择连接或模型池执行任务后,实际尝试会保存在这里。

}
{view.attempts.map(value =>
{states[value.status]}{value.model}

{connections.find(connection => connection.id === value.connectionId)?.name || '未派发'} · {value.protocol ? protocolLabel(value.protocol) : '能力筛选'}{value.streaming ? ' · 流式' : ''} · {value.ordinal ? '尝试 ' + value.ordinal : '未发送'} · {value.usageStatus === 'reported' ? `${value.usage?.prompt_tokens ?? '?'} 输入 / ${value.usage?.completion_tokens ?? '?'} 输出 tokens` : value.usageStatus === 'not_dispatched' ? '未产生请求用量' : '用量未知'}

Run {value.runId}{value.stream &&

{value.stream.terminal ? "响应完整终结" : "响应未完整,片段保留待核对"} · 已接收 {value.stream.bytes.toLocaleString()} bytes · {value.stream.frames} 个事件{value.stream.capturedBytes !== undefined && value.stream.capturedBytes < value.stream.bytes ? " · 仅保存上限内片段" : ""}

原流 SHA256 {value.stream.sha256}{value.responseSha256 && 工件 SHA256 {value.responseSha256}{value.responseRedacted ? "(已脱敏)" : ""}}{value.sessionId && value.responseArtifactId && }
}{value.requestUrl &&

请求地址:{value.requestUrl}{value.upstreamResponse?.responseUrl && <>
响应地址:{value.upstreamResponse.responseUrl}}

}{value.reason &&

{value.reason}

}{value.diagnostic && }{!value.diagnostic && value.upstreamResponse?.requestId &&

上游请求 ID:{value.upstreamResponse.requestId}

}{value.requestArtifactId && value.sessionId && }{!!value.rejections?.length &&

{value.rejections.map(item => item.connectionName + ':' + (capabilityLabels[item.capability] ?? item.capability) + ' · ' + item.reason).join(';')}

}
)}
} + {tab === 'attempts' && <>

请求账本

最近100次物理请求;最多保存1000次。未知用量保持未知。

{view.attempts.length} 条记录
{!view.attempts.length &&

当前没有模型中心请求。选择连接或模型池执行任务后,实际尝试会保存在这里。

}
{view.attempts.map(value =>
{states[value.status]}{value.model}

{connections.find(connection => connection.id === value.connectionId)?.name || '未派发'} · {value.protocol ? protocolLabel(value.protocol) : '能力筛选'}{value.streaming ? ' · 流式' : ''} · {value.ordinal ? '尝试 ' + value.ordinal : '未发送'} · {value.usageStatus === 'reported' ? `${value.usage?.prompt_tokens ?? '?'} 输入 / ${value.usage?.completion_tokens ?? '?'} 输出 tokens` : value.usageStatus === 'not_dispatched' ? '未产生请求用量' : '用量未知'}

Run {value.runId}{value.stream &&

{value.stream.terminal ? "响应完整终结" : "响应未完整,片段保留待核对"} · 已接收 {value.stream.bytes.toLocaleString()} bytes · {value.stream.frames} 个事件{value.stream.capturedBytes !== undefined && value.stream.capturedBytes < value.stream.bytes ? " · 仅保存上限内片段" : ""}

原流 SHA256 {value.stream.sha256}{value.responseSha256 && 工件 SHA256 {value.responseSha256}{value.responseRedacted ? "(已脱敏)" : ""}}{value.sessionId && value.responseArtifactId && }
}{value.requestUrl &&

请求地址:{value.requestUrl}{value.upstreamResponse?.responseUrl && <>
响应地址:{value.upstreamResponse.responseUrl}}

}{value.reason &&

{value.reason}

}{value.diagnostic && }{!value.diagnostic && value.upstreamResponse?.requestId &&

上游请求 ID:{value.upstreamResponse.requestId}

}{value.requestArtifactId && value.sessionId && }{!!value.rejections?.length &&

{value.rejections.map(item => item.connectionName + ':' + (capabilityLabels[item.capability] ?? item.capability) + ' · ' + item.reason).join(';')}

}
)}
} {tab === 'imports' && <>

已有配置导入

先预览来源和模型,再导入。原客户端配置保持原样。

{view.imports.length > 0 && <>

已登记来源

{view.imports.map(value =>
{value.source}{value.path}{value.enabled ? '每分钟检查' : '自动同步已关闭'}{value.lastCheckedAt &&

最近检查 {new Date(value.lastCheckedAt).toLocaleString()}

}{value.lastError &&

{value.lastError}

}{!!value.lastResult?.conflicts?.length &&

有 {value.lastResult.conflicts.length} 项与手动修改冲突,已保留工作台中的修改。

}
)}} {discovery.map(value =>
{value.source}{value.path || '当前版本尚无可发现的配置路径'}{value.available ? value.supported ? '可预览' : '当前不支持' : '未发现'}{value.detail &&

{value.detail}

}
)} diff --git a/src/client/ModelUsageDetails.css b/src/client/ModelUsageDetails.css new file mode 100644 index 00000000..62dae01f --- /dev/null +++ b/src/client/ModelUsageDetails.css @@ -0,0 +1,14 @@ +.model-usage-details { margin-top: 10px; color: var(--muted, #657773); font-size: 12px; line-height: 1.6; } +.model-usage-details > summary { width: fit-content; cursor: pointer; color: var(--ink, #2c4844); font-weight: 600; } +.model-usage-details dl { margin: 8px 0; } +.model-usage-details dl > div { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); gap: 8px; padding: 3px 0; } +.model-usage-details dt, .model-usage-details dd { margin: 0; overflow-wrap: anywhere; } +.model-usage-details dd { text-align: right; font-variant-numeric: tabular-nums; } +.model-usage-details dt small { display: block; font-size: 11px; font-weight: 400; } +.model-usage-details > p, .model-usage-sources > p { margin: 6px 0; } +.model-usage-details .model-usage-warning { color: var(--warning-ink, #a05c36); } +.model-usage-sources { margin-top: 7px; } +.model-usage-sources > summary { cursor: pointer; } +.model-usage-sources dl > div { grid-template-columns: 75px minmax(0, 1fr); } +.model-usage-sources dd { text-align: left; } +.model-usage-sources code { font-size: 11px; white-space: normal; overflow-wrap: anywhere; } diff --git a/src/client/ModelUsageDetails.tsx b/src/client/ModelUsageDetails.tsx new file mode 100644 index 00000000..03d8b411 --- /dev/null +++ b/src/client/ModelUsageDetails.tsx @@ -0,0 +1,30 @@ +import { tokenAccountingView, type TokenMetric, type TokenUsage } from '../shared/contracts/model-usage.mjs'; +import './ModelUsageDetails.css'; + +const count = (metric: TokenMetric) => metric.status === 'invalid' ? '上游数值无效' : metric.tokens === null ? '上游未报告' : metric.tokens.toLocaleString('zh-CN') + ' tokens'; +const protocols: Record = { 'anthropic-messages': 'Messages', responses: 'Responses', 'chat-completions': 'Chat' }; +const labels = { cacheRead: '缓存读取', cacheWrite: '缓存写入', uncached: '未缓存输入', reasoning: '推理输出' } as const; +const sourceLabels = { input: '全部输入', output: '全部输出', total: '合计', ...labels } as const; +const checkLabel = (value: string) => value === 'consistent' ? '一致' : value === 'inconsistent' ? '不一致' : '尚未完整报告'; +const lifetimeNames: Record = { ephemeral_5m_input_tokens: '5 分钟写入', ephemeral_1h_input_tokens: '1 小时写入', ttl_30m_tokens: '30 分钟写入', ttl_12h_tokens: '12 小时写入' }; + +export function ModelUsageDetails({ usage, expanded = false }: { usage: TokenUsage | null | undefined; expanded?: boolean }) { + const view = tokenAccountingView(usage); + if (!view) return null; + return
+ 缓存与用量明细 +
{(Object.keys(labels) as (keyof typeof labels)[]).map(key =>
+
{key === 'uncached' && view.scheme === 'read-and-miss' ? '未命中缓存输入' : labels[key]}{view.metrics[key].status === 'derived' && 按上游用量计算}
+
{count(view.metrics[key])}
+
)}{view.lifetimes.map(value =>
{lifetimeNames[value.name] ?? value.name}
{count(value)}
)}
+ {view.issues.length > 0 &&

上游用量字段存在不一致或无效值,原值已保留,请查看来源与原响应。

} +

缓存写入不等于命中;推理已包含在输出中。这些 token 统计不代表实际账单。

+
用量来源与核对 +

{protocols[view.protocol] ?? view.protocol} · {view.projection === 'original-usage' ? '原始上游 usage' : '旧归一记录'}
输入分类合计{checkLabel(view.checks.inputPartition)} · 输入输出合计{checkLabel(view.checks.total)}{view.lifetimes.length > 0 && <> · 写入时长分类{checkLabel(view.checks.cacheWriteLifetimes)}}

+
{(Object.keys(sourceLabels) as (keyof typeof sourceLabels)[]).map(key =>
{sourceLabels[key]}
{view.metrics[key].sources.join(';')}
)} + {view.lifetimes.map(value =>
{lifetimeNames[value.name] ?? value.name}
{value.sources.join(';')}
)} +
+ {view.issues.length > 0 && {view.issues.join(' · ')}} +
+
; +} diff --git a/src/client/RequestContextPanel.tsx b/src/client/RequestContextPanel.tsx index 368db360..e0a8a8fd 100644 --- a/src/client/RequestContextPanel.tsx +++ b/src/client/RequestContextPanel.tsx @@ -2,6 +2,7 @@ import { useEffect, useState } from 'react'; import { Archive, BrainCircuit, FileText, LoaderCircle } from 'lucide-react'; import { api } from './api'; import { Modal } from './components'; +import { ModelUsageDetails } from './ModelUsageDetails'; import type { ContextHistory, ContextHistoryGroup, ContextPlan, Session } from './types'; const number = (value: number | null | undefined) => (value ?? 0).toLocaleString('zh-CN'); @@ -44,7 +45,7 @@ export function RequestContextPanel({ session, runId, openArtifact, onError }: {
请求输入估算
{number(plan.estimatedInputTokens)} tokens
输出 / 安全余量
{plan.budget.outputReservation === null ? '不附加上限' : number(plan.budget.outputReservation) + ' tokens'} / {number(plan.budget.safetyTokens)} tokens
模型上下文窗口
{plan.budget.contextLimit === null ? '由上游处理 · 无本地窗口限制' : number(plan.budget.contextLimit) + ' tokens'}
已报告输入
{plan.actualUsage?.prompt_tokens === undefined ? '服务未返回' : number(plan.actualUsage.prompt_tokens) + ' tokens'}
历史记录
保留 {plan.retainedMessages} 条 · 保存 {plan.archivedGroups} 组原记录
-
上游缓存命中
{plan.actualUsage?.prompt_tokens_details?.cached_tokens === undefined ? '上游未报告' : number(plan.actualUsage.prompt_tokens_details.cached_tokens) + ' tokens'}
+
已报告输出
{plan.actualUsage?.completion_tokens === undefined ? '服务未返回' : number(plan.actualUsage.completion_tokens) + ' tokens'}
{plan.prefixAudit && <>
缓存前缀核对
{plan.prefixAudit.variants.map((variant, index) =>
{variant.model} · {variant.previousPromptBytes === null ? '首次请求或新路线' : '共同前缀 ' + number(variant.commonPrefixBytes) + ' B'}{variant.reason === 'tool_surface_changed' ? ' · 可用工具已变化' : ''}
)}共同前缀用于核对稳定性,不等于上游命中。
} {observation && <>
请求归属
{({ main: '主任务', diagnostic: '独立诊断', worker: '子任务' } as Record)[observation.path.kind] ?? observation.path.kind}
} {userMessage && <>
用户消息实例
{userMessage.id.slice(0, 12)}同样的文字分别记录,不按文本合并。
} @@ -54,6 +55,7 @@ export function RequestContextPanel({ session, runId, openArtifact, onError }: { {!!plan.skillCatalogEntries && <>
技能目录
{plan.skillCatalogEntries} 项本次目录记录
}
发送内容
{number(plan.requestBytes)} B{plan.requestSha256.slice(0, 16)}
+ {surface &&
本轮工具 · 可见 {surface.entries.filter(entry => entry.visible).length} / 已登记 {surface.entries.length}

这是本次实际发送的工具范围。资源和权限在执行时重新核对;登记不代表本轮可用。

    {surface.entries.map(entry =>
  • diff --git a/src/client/types.ts b/src/client/types.ts index a2a97285..9d697dc6 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -1,3 +1,5 @@ +import type { TokenUsage } from '../shared/contracts/model-usage.mjs'; + export interface Config { approvalMode?: 'ask' | 'auto' | 'full'; mode: 'demo' | 'compatible'; baseUrl: string; model: string; @@ -193,7 +195,7 @@ export interface ContextPlan { latestToolsArtifactId?: string | null; latestToolsSha256?: string | null; latestToolsEntries?: number; projectMemoryArtifactId?: string | null; projectMemorySha256?: string | null; projectMemoryEntries?: number; skillCatalogArtifactId?: string | null; skillCatalogSha256?: string | null; skillCatalogEntries?: number; - actualUsage: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number; prompt_tokens_details?: { cached_tokens?: number; cache_creation_tokens?: number } } | null; requestAttempts: number; error?: string; + actualUsage: TokenUsage | null; requestAttempts: number; error?: string; historyPolicy?: 'preserve' | 'compact'; contextLayout?: string; prefixAudit?: { schemaVersion: number; previousPlanId: string | null; variants: { connectionId: string | null; model: string; commonPrefixBytes: number; previousPromptBytes: number | null; reason: string; toolNames: string[]; toolsSha256: string; firstChange?: { path: string; kind: string; sha256: string | null; previousSha256: string | null } | null; diff --git a/src/core/context/request-plan.mjs b/src/core/context/request-plan.mjs index 74c29176..0070dd87 100644 --- a/src/core/context/request-plan.mjs +++ b/src/core/context/request-plan.mjs @@ -461,10 +461,10 @@ export class RequestPlanner { return { messages, body, record }; } - finish(session, record, { response = null, error = null, attempts = 0 } = {}) { + finish(session, record, { response = null, error = null, attempts = 0, usage = response?.usage ?? null } = {}) { this.store.owned('context_plan', record.id, session.id); record.status = error ? error.name === 'AbortError' || session.status === 'stopping' ? 'interrupted' : 'failed' : 'completed'; - record.finishedAt = new Date().toISOString(); record.actualUsage = response?.usage ?? null; + record.finishedAt = new Date().toISOString(); record.actualUsage = usage; record.responseSha256 = response?.message ? hash(response.message) : null; record.requestAttempts = attempts; if (error) record.error = String(error.message ?? error).slice(0, 800); this.store.save(session, { type: 'context.' + record.status, detail: { recordId: record.id } }); @@ -482,10 +482,13 @@ export function verifyPlanEvidence(store, session, plan, cache = new Map()) { const runModel = store.owned('run', plan.runId, session.id).model; if(plan.tokenReservation)verifyModelReservation(store,session,plan.taskId,{...store.owned('run',plan.runId,session.id).model, modelRouting:store.owned('run',plan.runId,session.id).model.routing},body,plan.tokenReservation); - const invalidOutput = body.kind === 'model-route-plan' && runModel.capacityMode === 'model' + // Passthrough can carry explicit native output settings and Messages' required + // max_tokens without a local reservation. The wire SHA and frozen budget are + // still checked below; a protocol field is not a local budget. + const invalidOutput = !isPassthrough(runModel) && (body.kind === 'model-route-plan' && runModel.capacityMode === 'model' ? body.variants.some(variant => outputReservation(variant.body) !== runModel.routing?.members.find(member => member.id === variant.connectionId)?.maxOutputTokens) || Math.max(...requestVariants(body).map(outputReservation)) !== plan.budget.outputReservation - : requestVariants(body).some(value => outputReservation(value) !== plan.budget.outputReservation); + : requestVariants(body).some(value => outputReservation(value) !== plan.budget.outputReservation)); if (metrics.bytes !== plan.requestBytes || metrics.estimatedInputTokens !== plan.estimatedInputTokens || invalidOutput || plan.budget.inputBudget !== null && metrics.estimatedInputTokens > plan.budget.inputBudget) throw new Error('请求预算与原正文不匹配。'); historyGroups(requestHistory(body)); diff --git a/src/core/models/protocols.mjs b/src/core/models/protocols.mjs index b5f0cc12..de5872a1 100644 --- a/src/core/models/protocols.mjs +++ b/src/core/models/protocols.mjs @@ -1,5 +1,6 @@ import { unrestricted, ceiling } from '../../shared/contracts/execution-policy.mjs'; import { ModelRequestError } from '../../server/diagnostics.mjs'; +import { withTokenAccounting } from '../../shared/contracts/model-usage.mjs'; export const protocolFailure = (message, kind = 'protocol', extra = null) => new ModelRequestError({ kind, source: 'model_connection', origin: 'local', stage: 'model_request', title: '模型响应未通过完整性核对', message, retryable: false, occurredAt: new Date().toISOString(), ...(extra ? extra : {}) }); @@ -12,6 +13,10 @@ export const outputTruncatedFailure = (kind = 'protocol') => protocolFailure(OUT export { protocolRoute } from '../../shared/contracts/model-endpoint.mjs'; export function reportedUsage(config, value) { + return withTokenAccounting(baseReportedUsage(config, value)); +} + +function baseReportedUsage(config, value) { const raw = value?.usage; if (!raw || typeof raw !== 'object') return null; if (config.protocol === 'anthropic-messages') { @@ -30,6 +35,7 @@ export function reportedUsage(config, value) { const output = config.protocol === 'responses' ? raw.output_tokens : raw.completion_tokens; const valid = number => Number.isSafeInteger(number) && number >= 0; if (!valid(raw.total_tokens) && !(valid(input) && valid(output))) return null; + if (!valid(raw.total_tokens) && !valid(input + output)) return null; const cached = config.protocol === 'responses' ? raw.input_tokens_details?.cached_tokens : raw.prompt_tokens_details?.cached_tokens ?? raw.prompt_cache_hit_tokens; const cacheDetails = valid(cached) && valid(input) && cached <= input ? { cached_tokens: cached } : null; const reasoning = config.protocol === 'responses' ? raw.output_tokens_details?.reasoning_tokens : raw.completion_tokens_details?.reasoning_tokens; diff --git a/src/server/runtime.mjs b/src/server/runtime.mjs index 49094a12..514f82b4 100644 --- a/src/server/runtime.mjs +++ b/src/server/runtime.mjs @@ -520,7 +520,7 @@ export class Runtime { let requestEvent; const externalEngine = !!run.worker?.engine; if (externalEngine) { planned.record.engineStepOnly = true; planned.record.engine = 'deepseek-workbench'; } - let response, requestStarted = false, responseAccounted = externalEngine; + let response, finalAttemptUsage = null, requestStarted = false, responseAccounted = externalEngine; try { signal.throwIfAborted(); checkBudget(run.record, task); if (run.worker) sealWorkerDispatch(this.store, session, run.worker, run.record, planned.record); @@ -528,6 +528,7 @@ export class Runtime { requestStarted = true; const progressHandler = progress => { if (progress.consumption) { + finalAttemptUsage = progress.consumption.usage ?? null; this.store.recordConsumption(session, run.record, { ...progress.consumption, requestId: planned.record.id }, planned.record.tokenReservation?.reservedTokens??planned.record.estimatedInputTokens + config.maxOutputTokens); responseAccounted = true; if (run.worker) { planned.record.workerUsageAccounted = true; this.store.save(session); } @@ -585,7 +586,7 @@ export class Runtime { } } catch (error) { if (requestEvent?.stream && requestEvent.stream.status !== 'completed') { requestEvent.stream.status = signal.aborted ? 'interrupted' : 'incomplete'; requestEvent.phase = signal.aborted ? 'interrupted' : 'failed'; requestEvent.finishedAt = now(); } - this.requestPlanner().finish(session, planned.record, { error: Object.assign(new Error(this.provider.redact(error.message ?? String(error))), { name: signal.aborted ? 'AbortError' : error.name }), attempts: planned.record.requestAttempts }); + this.requestPlanner().finish(session, planned.record, { error: Object.assign(new Error(this.provider.redact(error.message ?? String(error))), { name: signal.aborted ? 'AbortError' : error.name }), attempts: planned.record.requestAttempts, usage: finalAttemptUsage }); if (run.worker && requestStarted && !responseAccounted) { this.store.recordResponse(session, run.record, null, planned.record.tokenReservation?.reservedTokens??planned.record.estimatedInputTokens + config.maxOutputTokens); planned.record.workerUsageAccounted = true; diff --git a/src/shared/contracts/model-usage.d.mts b/src/shared/contracts/model-usage.d.mts new file mode 100644 index 00000000..ecdf032b --- /dev/null +++ b/src/shared/contracts/model-usage.d.mts @@ -0,0 +1,15 @@ +export type TokenMetricStatus = 'reported' | 'derived' | 'unknown' | 'invalid'; +export type TokenMetricName = 'input' | 'output' | 'total' | 'cacheRead' | 'cacheWrite' | 'uncached' | 'reasoning'; +export interface TokenMetric { tokens: number | null; status: TokenMetricStatus; sources: string[]; formula?: string } +export interface TokenAccounting { + schemaVersion: 1; protocol: string; projection: 'original-usage' | 'legacy-normalized'; scheme: 'read-write-uncached' | 'read-and-miss'; + metrics: Record; lifetimes: (TokenMetric & { name: string })[]; + checks: Record<'inputPartition' | 'total' | 'cacheWriteLifetimes', 'consistent' | 'partial' | 'inconsistent'>; issues: string[]; +} +export interface TokenUsage { + prompt_tokens?: number; completion_tokens?: number; total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number; cache_creation_tokens?: number }; + completion_tokens_details?: { reasoning_tokens?: number }; raw?: Record; protocol?: string; tokenAccounting?: TokenAccounting; +} +export function tokenAccountingView(usage: TokenUsage | null | undefined): TokenAccounting | null; +export function withTokenAccounting(usage: T): T; diff --git a/src/shared/contracts/model-usage.mjs b/src/shared/contracts/model-usage.mjs new file mode 100644 index 00000000..d2d36d2e --- /dev/null +++ b/src/shared/contracts/model-usage.mjs @@ -0,0 +1,68 @@ +const object = value => !!value && typeof value === 'object' && !Array.isArray(value); +const valid = value => Number.isSafeInteger(value) && value >= 0; +const at = (value, path) => path.split('.').reduce((part, key) => object(part) ? part[key] : undefined, value); +const present = value => value !== undefined && value !== null; +const metric = (value, source, status = 'reported') => ({ tokens: valid(value) ? value : null, + status: !present(value) ? 'unknown' : valid(value) ? status : 'invalid', sources: Array.isArray(source) ? source : [source] }); + +// These are token categories from the received protocol fields, not a bill or +// proof that any particular provider served the connection. Missing is not zero. +export function tokenAccountingView(usage) { + if (!object(usage)) return null; + if (usage.tokenAccounting?.schemaVersion === 1 && object(usage.tokenAccounting.metrics)) return usage.tokenAccounting; + const protocol = usage.protocol ?? 'chat-completions', raw = object(usage.raw) ? usage.raw : null; + const metrics = { input: metric(usage.prompt_tokens, 'normalized.prompt_tokens'), output: metric(usage.completion_tokens, 'normalized.completion_tokens'), + total: metric(usage.total_tokens, 'normalized.total_tokens'), cacheRead: metric(undefined, 'unreported'), cacheWrite: metric(undefined, 'unreported'), + uncached: metric(undefined, 'unreported'), reasoning: metric(undefined, 'unreported') }; + const issues = [], lifetimes = []; let scheme = 'read-write-uncached'; + const read = path => metric(at(raw, path), 'usage.' + path); + const first = paths => read(paths.find(path => present(at(raw, path))) ?? paths[0]); + let lifetimePaths = []; + if (!raw) { + metrics.cacheRead = metric(usage.prompt_tokens_details?.cached_tokens, 'normalized.prompt_tokens_details.cached_tokens'); + metrics.cacheWrite = metric(usage.prompt_tokens_details?.cache_creation_tokens, 'normalized.prompt_tokens_details.cache_creation_tokens'); + metrics.reasoning = metric(usage.completion_tokens_details?.reasoning_tokens, 'normalized.completion_tokens_details.reasoning_tokens'); + } else if (protocol === 'anthropic-messages') { + metrics.output = read('output_tokens'); metrics.cacheRead = read('cache_read_input_tokens'); metrics.cacheWrite = read('cache_creation_input_tokens'); metrics.uncached = read('input_tokens'); + const parts = [metrics.uncached, metrics.cacheRead, metrics.cacheWrite]; + const input = parts.every(value => valid(value.tokens)) ? parts.reduce((sum, value) => sum + value.tokens, 0) : undefined; + metrics.input = metric(input, parts.flatMap(value => value.sources), 'derived'); + lifetimePaths = ['cache_creation.ephemeral_5m_input_tokens', 'cache_creation.ephemeral_1h_input_tokens']; + } else { + const responses = protocol === 'responses', details = responses ? 'input_tokens_details' : 'prompt_tokens_details'; + metrics.input = read(responses ? 'input_tokens' : 'prompt_tokens'); metrics.output = read(responses ? 'output_tokens' : 'completion_tokens'); + metrics.cacheRead = first(responses ? [details + '.cached_tokens'] : [details + '.cached_tokens', 'prompt_cache_hit_tokens', 'cached_tokens']); + metrics.cacheWrite = read(details + '.cache_write_tokens'); + metrics.uncached = first(responses ? [details + '.uncached_tokens'] : [details + '.uncached_tokens', 'prompt_cache_miss_tokens']); + metrics.reasoning = read((responses ? 'output_tokens_details' : 'completion_tokens_details') + '.reasoning_tokens'); + lifetimePaths = [details + '.cache_write_tokens_details.ttl_30m_tokens', details + '.cache_write_tokens_details.ttl_12h_tokens']; + if (!responses && metrics.cacheWrite.status === 'unknown' && metrics.uncached.sources[0] === 'usage.prompt_cache_miss_tokens') scheme = 'read-and-miss'; + if (!responses && new Set([at(raw, details + '.cached_tokens'), raw.prompt_cache_hit_tokens, raw.cached_tokens].filter(valid)).size > 1) issues.push('cache_read_alias_mismatch'); + if (!responses && valid(at(raw, details + '.uncached_tokens')) && valid(raw.prompt_cache_miss_tokens) + && at(raw, details + '.uncached_tokens') !== raw.prompt_cache_miss_tokens) issues.push('uncached_alias_mismatch'); + if (metrics.uncached.status === 'unknown' && [metrics.input.tokens, metrics.cacheRead.tokens, metrics.cacheWrite.tokens].every(valid)) { + const remaining = metrics.input.tokens - metrics.cacheRead.tokens - metrics.cacheWrite.tokens; + if (valid(remaining)) metrics.uncached = { tokens: remaining, status: 'derived', sources: [...metrics.input.sources, ...metrics.cacheRead.sources, ...metrics.cacheWrite.sources], formula: 'input - cacheRead - cacheWrite' }; + else issues.push('cache_categories_exceed_input'); + } + } + if (raw) { + if (present(raw.total_tokens)) metrics.total = read('total_tokens'); + else metrics.total = metric(valid(metrics.input.tokens) && valid(metrics.output.tokens) ? metrics.input.tokens + metrics.output.tokens : undefined, + [...metrics.input.sources, ...metrics.output.sources], 'derived'); + for (const key of ['cacheRead', 'cacheWrite', 'uncached']) if (valid(metrics[key].tokens) && valid(metrics.input.tokens) && metrics[key].tokens > metrics.input.tokens) issues.push(key + '_exceeds_input'); + if (valid(metrics.reasoning.tokens) && valid(metrics.output.tokens) && metrics.reasoning.tokens > metrics.output.tokens) issues.push('reasoning_exceeds_output'); + if (lifetimePaths.some(path => present(at(raw, path)))) for (const path of lifetimePaths) lifetimes.push({ name: path.split('.').at(-1), ...read(path) }); + } + for (const [key, value] of Object.entries(metrics)) if (value.status === 'invalid') issues.push('invalid_' + key); + for (const value of lifetimes) if (value.status === 'invalid') issues.push('invalid_write_lifetime'); + const check = (values, total) => !valid(total) || !values.every(valid) ? 'partial' : values.reduce((sum, value) => sum + value, 0) === total ? 'consistent' : 'inconsistent'; + const checks = { inputPartition: check((scheme === 'read-and-miss' ? ['cacheRead', 'uncached'] : ['cacheRead', 'cacheWrite', 'uncached']).map(key => metrics[key].tokens), metrics.input.tokens), + total: check([metrics.input.tokens, metrics.output.tokens], metrics.total.tokens), cacheWriteLifetimes: lifetimes.length ? check(lifetimes.map(value => value.tokens), metrics.cacheWrite.tokens) : 'partial' }; + for (const [key, value] of Object.entries(checks)) if (value === 'inconsistent') issues.push(key + '_mismatch'); + return { schemaVersion: 1, protocol, projection: raw ? 'original-usage' : 'legacy-normalized', scheme, metrics, lifetimes, checks, issues }; +} + +export function withTokenAccounting(usage) { + return usage ? { ...usage, tokenAccounting: tokenAccountingView(usage) } : usage; +} diff --git a/tests/fixtures/cache-usage-model.mjs b/tests/fixtures/cache-usage-model.mjs new file mode 100644 index 00000000..3032ecae --- /dev/null +++ b/tests/fixtures/cache-usage-model.mjs @@ -0,0 +1,47 @@ +import http from 'node:http'; + +export const usageSamples = { + chat: { protocol: 'chat-completions', raw: { prompt_tokens: 1000, completion_tokens: 100, total_tokens: 1100, + prompt_tokens_details: { cached_tokens: 600, cache_write_tokens: 300, uncached_tokens: 100 }, completion_tokens_details: { reasoning_tokens: 40 } } }, + responses: { protocol: 'responses', raw: { input_tokens: 12000, output_tokens: 100, total_tokens: 12100, + input_tokens_details: { cached_tokens: 4000, cache_write_tokens: 7000, cache_write_tokens_details: { ttl_30m_tokens: 3000, ttl_12h_tokens: 4000 } }, output_tokens_details: { reasoning_tokens: 40 } } }, + messages: { protocol: 'anthropic-messages', raw: { input_tokens: 50, output_tokens: 12, cache_read_input_tokens: 100, cache_creation_input_tokens: 248, + cache_creation: { ephemeral_5m_input_tokens: 148, ephemeral_1h_input_tokens: 100 } } }, +}; +export const usageModelId = 'owned-cache-usage'; +const event = value => 'data: ' + (typeof value === 'string' ? value : JSON.stringify(value)) + '\r\n\r\n'; +const messageEvent = (type, value) => 'event: ' + type + '\r\n' + event({ type, ...value }); +export function usageResponse(protocol, usage, text = '这是本机用量夹具的完整回复,没有执行工具。') { + const output = { type: 'message', id: 'msg_owned_usage', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text }] }; + if (protocol === 'responses') return { id: 'resp_owned_usage', model: usageModelId, status: 'completed', output: [output], ...(usage ? { usage } : {}) }; + if (protocol === 'anthropic-messages') return { id: 'msg_owned_usage', type: 'message', model: usageModelId, role: 'assistant', content: [{ type: 'text', text }], stop_reason: 'end_turn', stop_sequence: null, ...(usage ? { usage } : {}) }; + return { id: 'chat_owned_usage', model: usageModelId, choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }], ...(usage ? { usage } : {}) }; +} +export function usageStream(protocol, usage, terminal = true) { + const final = usageResponse(protocol, usage); + if (protocol === 'responses') return event({ type: 'response.created', sequence_number: 0, response: { id: final.id, model: usageModelId } }) + + (terminal ? event({ type: 'response.completed', sequence_number: 1, response: final }) : ''); + if (protocol === 'anthropic-messages') return messageEvent('message_start', { message: { ...final, content: [], stop_reason: null, usage: { ...usage, output_tokens: 0 } } }) + + messageEvent('content_block_start', { index: 0, content_block: { type: 'text', text: '' } }) + + messageEvent('content_block_delta', { index: 0, delta: { type: 'text_delta', text: final.content[0].text } }) + + messageEvent('content_block_stop', { index: 0 }) + + messageEvent('message_delta', { delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: usage.output_tokens } }) + + (terminal ? messageEvent('message_stop', {}) : ''); + return event({ id: final.id, model: usageModelId, choices: [{ index: 0, delta: { role: 'assistant', content: final.choices[0].message.content }, finish_reason: null }] }) + + event({ id: final.id, model: usageModelId, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }) + + (usage ? event({ id: final.id, model: usageModelId, choices: [], usage }) : '') + (terminal ? event('[DONE]') : ''); +} +export async function cacheUsageFixture(t, { protocol, raw, streaming = false, terminal = true, status = 200 }) { + const requests = [], responses = []; + const server = http.createServer(async (request, response) => { + const chunks = []; for await (const chunk of request) chunks.push(chunk); + requests.push({ url: request.url, input: JSON.parse(Buffer.concat(chunks)) }); + const value = status === 200 ? usageResponse(protocol, raw) : { error: { type: 'owned_usage_error', code: 'owned_usage_400', message: '自有上游原错误,不重试' }, usage: raw }; + const bytes = Buffer.from(streaming && status === 200 ? usageStream(protocol, raw, terminal) : JSON.stringify(value)); responses.push(bytes); + response.writeHead(status, { 'Content-Type': streaming && status === 200 ? 'text/event-stream' : 'application/json', 'x-request-id': 'owned-usage-request' }); + for (let offset = 0; offset < bytes.length; offset += 7) response.write(bytes.subarray(offset, offset + 7)); response.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); })); + return { baseUrl: 'http://127.0.0.1:' + server.address().port + '/v1', requests, responses }; +} diff --git a/tests/model-usage-accounting.test.mjs b/tests/model-usage-accounting.test.mjs new file mode 100644 index 00000000..6e1cf1ed --- /dev/null +++ b/tests/model-usage-accounting.test.mjs @@ -0,0 +1,120 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { fixture, until } from './helpers.mjs'; +import { Provider } from '../src/server/provider.mjs'; +import { Runtime } from '../src/server/runtime.mjs'; +import { Store } from '../src/server/store.mjs'; +import { reportedUsage } from '../src/core/models/protocols.mjs'; +import { tokenAccountingView } from '../src/shared/contracts/model-usage.mjs'; +import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs'; +import { usageSamples, usageModelId, cacheUsageFixture } from './fixtures/cache-usage-model.mjs'; + +const normalized = sample => reportedUsage({ protocol: sample.protocol }, { usage: sample.raw }); +const counts = view => Object.fromEntries(Object.entries(view.metrics).map(([key, value]) => [key, value.tokens])); + +test('Kimi-style exclusive read/write/uncached usage retains original fields and reasoning without double counting', () => { + const sample = structuredClone(usageSamples.chat), before = JSON.stringify(sample.raw), usage = normalized(sample), view = usage.tokenAccounting; + assert.deepEqual(counts(view), { input: 1000, output: 100, total: 1100, cacheRead: 600, cacheWrite: 300, uncached: 100, reasoning: 40 }); + assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.checks.total, 'consistent'); assert.deepEqual(view.issues, []); + assert.equal(view.metrics.uncached.status, 'reported'); assert.equal(view.metrics.cacheWrite.sources[0], 'usage.prompt_tokens_details.cache_write_tokens'); + assert.equal(JSON.stringify(sample.raw), before); assert.deepEqual(usage.raw, sample.raw); assert.equal(usage.total_tokens, 1100); + delete sample.raw.prompt_tokens_details.uncached_tokens; const derived = normalized(sample).tokenAccounting.metrics.uncached; + assert.equal(derived.tokens, 100); assert.equal(derived.status, 'derived'); assert.equal(derived.formula, 'input - cacheRead - cacheWrite'); +}); + +test('Responses cache write lifetimes remain disjoint, derived ordinary input is labeled and reasoning remains inside output', () => { + const usage = normalized(usageSamples.responses), view = usage.tokenAccounting; + assert.deepEqual(counts(view), { input: 12000, output: 100, total: 12100, cacheRead: 4000, cacheWrite: 7000, uncached: 1000, reasoning: 40 }); + assert.equal(view.metrics.uncached.status, 'derived'); assert.deepEqual(view.lifetimes.map(value => value.tokens), [3000, 4000]); + assert.equal(view.checks.cacheWriteLifetimes, 'consistent'); assert.equal(view.checks.total, 'consistent'); assert.deepEqual(usage.raw, usageSamples.responses.raw); +}); + +test('Messages input is uncached and totals add read/write exactly once; optional cache fields and absent reasoning stay unknown', () => { + const usage = normalized(usageSamples.messages), view = usage.tokenAccounting; + assert.deepEqual(counts(view), { input: 398, output: 12, total: 410, cacheRead: 100, cacheWrite: 248, uncached: 50, reasoning: null }); + assert.equal(view.metrics.input.status, 'derived'); assert.equal(view.metrics.uncached.status, 'reported'); assert.deepEqual(view.lifetimes.map(value => value.tokens), [148, 100]); + assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.checks.cacheWriteLifetimes, 'consistent'); + const absent = reportedUsage({ protocol: 'anthropic-messages' }, { usage: { input_tokens: 50, output_tokens: 12 } }); + assert.equal(absent.prompt_tokens_details, undefined); assert.equal(absent.tokenAccounting.metrics.cacheRead.tokens, null); assert.equal(absent.tokenAccounting.metrics.cacheWrite.tokens, null); + assert.equal(absent.tokenAccounting.checks.inputPartition, 'partial'); + assert.equal(absent.tokenAccounting.metrics.input.status, 'unknown'); assert.equal(absent.tokenAccounting.metrics.total.status, 'unknown'); + assert.equal(absent.tokenAccounting.metrics.uncached.tokens, 50); assert.equal(absent.total_tokens, 62, 'Legacy normalized totals remain unchanged; the new breakdown does not invent missing categories.'); +}); + +test('DeepSeek cache misses retain their own meaning and never fabricate cache-write counts or provider identity', () => { + const raw = { prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_cache_hit_tokens: 900, prompt_cache_miss_tokens: 100 }; + const usage = reportedUsage({ protocol: 'chat-completions', model: 'arbitrary-proxy-alias' }, { usage: raw }), view = usage.tokenAccounting; + assert.equal(view.scheme, 'read-and-miss'); assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.metrics.cacheWrite.tokens, null); + assert.equal(view.metrics.uncached.tokens, 100); assert.equal(view.metrics.uncached.sources[0], 'usage.prompt_cache_miss_tokens'); + assert.deepEqual(usage.raw, raw); assert.equal('provider' in view, false); +}); + +test('missing differs from explicit zero; incomplete, invalid and overflowing receipts never become invented categories', () => { + const partial = reportedUsage({}, { usage: { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 700 } } }).tokenAccounting; + assert.equal(partial.metrics.cacheWrite.tokens, null); assert.equal(partial.metrics.uncached.tokens, null); assert.equal(partial.checks.inputPartition, 'partial'); + const zero = reportedUsage({}, { usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, prompt_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 } } }).tokenAccounting; + assert.equal(zero.metrics.cacheWrite.tokens, 0); assert.equal(zero.metrics.uncached.tokens, 0); assert.equal(zero.checks.inputPartition, 'consistent'); + for (const value of [-1, '300', 1.2, Number.MAX_SAFE_INTEGER + 1]) { + const view = reportedUsage({}, { usage: { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 600, cache_write_tokens: value } } }).tokenAccounting; + assert.equal(view.metrics.cacheWrite.tokens, null); assert.equal(view.metrics.cacheWrite.status, 'invalid'); assert.ok(view.issues.includes('invalid_cacheWrite')); assert.equal(view.metrics.uncached.tokens, null); + } + assert.equal(reportedUsage({}, { usage: { prompt_tokens: Number.MAX_SAFE_INTEGER, completion_tokens: 1 } }), null); + assert.equal(reportedUsage({}, {}), null); assert.equal(tokenAccountingView(null), null); +}); + +test('conflicting categories, aliases, totals, lifetime partitions and excessive reasoning keep raw values with explicit issues', () => { + const sample = structuredClone(usageSamples.responses); + sample.raw.total_tokens = 1; sample.raw.input_tokens_details.cache_write_tokens_details.ttl_30m_tokens = 1; sample.raw.output_tokens_details.reasoning_tokens = 101; + const usage = normalized(sample), view = usage.tokenAccounting; + assert.equal(view.checks.total, 'inconsistent'); assert.equal(view.checks.cacheWriteLifetimes, 'inconsistent'); assert.ok(view.issues.includes('reasoning_exceeds_output')); assert.deepEqual(usage.raw, sample.raw); + const raw = { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 800, cache_write_tokens: 300, uncached_tokens: 100 }, prompt_cache_hit_tokens: 700 }; + const conflict = reportedUsage({}, { usage: raw }).tokenAccounting; + assert.equal(conflict.checks.inputPartition, 'inconsistent'); assert.ok(conflict.issues.includes('cache_read_alias_mismatch')); +}); + +test('legacy raw and normalized usage can be displayed without rewriting old receipt bytes or inventing unavailable cache history', () => { + const old = { prompt_tokens: 1000, completion_tokens: 100, total_tokens: 1100, raw: structuredClone(usageSamples.chat.raw), protocol: 'chat-completions' }, before = JSON.stringify(old); + assert.equal(tokenAccountingView(old).metrics.cacheWrite.tokens, 300); assert.equal(JSON.stringify(old), before); assert.equal(old.tokenAccounting, undefined); + const legacy = { prompt_tokens: 150, completion_tokens: 12, total_tokens: 162, prompt_tokens_details: { cached_tokens: 30, cache_creation_tokens: 20 } }; + const view = tokenAccountingView(legacy); assert.equal(view.projection, 'legacy-normalized'); assert.equal(view.metrics.cacheWrite.tokens, 20); assert.equal(view.metrics.uncached.tokens, null); +}); + +async function run(t, sample, extra = {}) { + const f = fixture(t, { modernDefaults: true }), model = await cacheUsageFixture(t, { ...sample, ...extra }); + const provider = new Provider(), hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned usage fixture', model: usageModelId, protocol: sample.protocol, + baseUrl: model.baseUrl, requiresKey: false, streaming: !!extra.streaming, modelIdentityPolicy: 'require-match' }); + hub.select({ type: 'connection', id: connection.id }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); + runtime.start(f.session.id, '仅解释用量字段含义,不修改文件或运行命令。'); await until(() => !runtime.isActive(f.session.id), 15000); + assert.equal(model.requests.length, 1); const attempt = hub.view().attempts[0], plans = [...f.store.core.context_plan.values()]; assert.equal(plans.length, 1); + assert.equal(f.store.core.tool.size, 0); return { f, model, hub, runtime, provider, attempt, plan: plans[0] }; +} + +for (const [name, sample] of Object.entries(usageSamples)) for (const streaming of [false, true]) test(`actual ${name} ${streaming ? 'SSE' : 'JSON'} persists original usage and source-bound categories in plan, attempt, Run and restart`, async t => { + const { f, model, attempt, plan, runtime } = await run(t, sample, { streaming }), expected = normalized(sample); + assert.equal(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported'); assert.equal(attempt.modelIdentity.status, 'matched'); + assert.deepEqual(attempt.usage, expected); assert.deepEqual(plan.actualUsage, expected); + const execution = f.store.core.run.get(f.session.runId); assert.equal(execution.usage.totalTokens, expected.total_tokens); assert.equal(execution.usage.unknownRequests, 0); + assert.deepEqual(execution.consumption[0].usage, expected); assert.doesNotThrow(() => verifyPlanEvidence(f.store, f.session, plan)); + assert.throws(() => verifyPlanEvidence(f.store, f.session, { ...plan, requestBytes: plan.requestBytes + 1 }), /不匹配/); + assert.throws(() => verifyPlanEvidence(f.store, f.session, { ...plan, budget: { ...plan.budget, outputReservation: 1 } }), /不匹配/); + const artifact = f.store.owned('artifact', attempt.responseArtifactId, f.session.id); assert.equal(artifact.sha256, createHash('sha256').update(model.responses[0]).digest('hex')); + assert.equal(attempt.responseSha256, artifact.sha256); const original = JSON.stringify(attempt); await runtime.stopAll(); f.store.close(); + const reopened = new Store(f.data); f.onCleanup(() => reopened.close()); const restored = new Provider().modelHub(reopened).view().attempts.find(value => value.id === attempt.id); + assert.equal(JSON.stringify(restored), original); assert.equal(model.requests.length, 1); +}); + +test('original upstream 400 retains reported cache usage without failover, retries or zero-cost inference', async t => { + const { f, model, attempt, plan } = await run(t, usageSamples.chat, { status: 400 }); + assert.equal(f.session.status, 'failed'); assert.notEqual(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported'); + assert.deepEqual(attempt.usage.raw, usageSamples.chat.raw); assert.equal(attempt.usage.tokenAccounting.metrics.cacheWrite.tokens, 300); + assert.equal(plan.actualUsage.tokenAccounting.metrics.cacheRead.tokens, 600); assert.match(f.session.failure.message, /自有上游原错误/); assert.equal(model.requests.length, 1); +}); + +test('missing usage stays unknown, while unended SSE keeps reported categories without treating its reply as completed', async t => { + const missing = await run(t, { protocol: 'chat-completions', raw: undefined }); assert.equal(missing.attempt.usageStatus, 'unknown'); assert.equal(missing.attempt.usage, null); + assert.equal(missing.f.store.core.run.get(missing.f.session.runId).usage.unknownRequests, 1); + const partial = await run(t, usageSamples.chat, { streaming: true, terminal: false }); assert.notEqual(partial.attempt.status, 'completed'); + assert.equal(partial.attempt.usageStatus, 'reported'); assert.equal(partial.attempt.usage.tokenAccounting.metrics.cacheWrite.tokens, 300); + assert.equal(partial.model.requests.length, 1); assert.equal(partial.f.store.core.tool.size, 0); +});