import test from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { fixture, until } from './helpers.mjs'; import { Provider } from '../src/server/provider.mjs'; import { Runtime } from '../src/server/runtime.mjs'; import { Store } from '../src/server/store.mjs'; import { reportedUsage } from '../src/core/models/protocols.mjs'; import { tokenAccountingView } from '../src/shared/contracts/model-usage.mjs'; import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs'; import { usageSamples, usageModelId, cacheUsageFixture } from './fixtures/cache-usage-model.mjs'; const normalized = sample => reportedUsage({ protocol: sample.protocol }, { usage: sample.raw }); const counts = view => Object.fromEntries(Object.entries(view.metrics).map(([key, value]) => [key, value.tokens])); test('Kimi-style exclusive read/write/uncached usage retains original fields and reasoning without double counting', () => { const sample = structuredClone(usageSamples.chat), before = JSON.stringify(sample.raw), usage = normalized(sample), view = usage.tokenAccounting; assert.deepEqual(counts(view), { input: 1000, output: 100, total: 1100, cacheRead: 600, cacheWrite: 300, uncached: 100, reasoning: 40 }); assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.checks.total, 'consistent'); assert.deepEqual(view.issues, []); assert.equal(view.metrics.uncached.status, 'reported'); assert.equal(view.metrics.cacheWrite.sources[0], 'usage.prompt_tokens_details.cache_write_tokens'); assert.equal(JSON.stringify(sample.raw), before); assert.deepEqual(usage.raw, sample.raw); assert.equal(usage.total_tokens, 1100); delete sample.raw.prompt_tokens_details.uncached_tokens; const derived = normalized(sample).tokenAccounting.metrics.uncached; assert.equal(derived.tokens, 100); assert.equal(derived.status, 'derived'); assert.equal(derived.formula, 'input - cacheRead - cacheWrite'); }); test('Responses cache write lifetimes remain disjoint, derived ordinary input is labeled and reasoning remains inside output', () => { const usage = normalized(usageSamples.responses), view = usage.tokenAccounting; assert.deepEqual(counts(view), { input: 12000, output: 100, total: 12100, cacheRead: 4000, cacheWrite: 7000, uncached: 1000, reasoning: 40 }); assert.equal(view.metrics.uncached.status, 'derived'); assert.deepEqual(view.lifetimes.map(value => value.tokens), [3000, 4000]); assert.equal(view.checks.cacheWriteLifetimes, 'consistent'); assert.equal(view.checks.total, 'consistent'); assert.deepEqual(usage.raw, usageSamples.responses.raw); }); test('Messages input is uncached and totals add read/write exactly once; optional cache fields and absent reasoning stay unknown', () => { const usage = normalized(usageSamples.messages), view = usage.tokenAccounting; assert.deepEqual(counts(view), { input: 398, output: 12, total: 410, cacheRead: 100, cacheWrite: 248, uncached: 50, reasoning: null }); assert.equal(view.metrics.input.status, 'derived'); assert.equal(view.metrics.uncached.status, 'reported'); assert.deepEqual(view.lifetimes.map(value => value.tokens), [148, 100]); assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.checks.cacheWriteLifetimes, 'consistent'); const absent = reportedUsage({ protocol: 'anthropic-messages' }, { usage: { input_tokens: 50, output_tokens: 12 } }); assert.equal(absent.prompt_tokens_details, undefined); assert.equal(absent.tokenAccounting.metrics.cacheRead.tokens, null); assert.equal(absent.tokenAccounting.metrics.cacheWrite.tokens, null); assert.equal(absent.tokenAccounting.checks.inputPartition, 'partial'); assert.equal(absent.tokenAccounting.metrics.input.status, 'unknown'); assert.equal(absent.tokenAccounting.metrics.total.status, 'unknown'); assert.equal(absent.tokenAccounting.metrics.uncached.tokens, 50); assert.equal(absent.total_tokens, 62, 'Legacy normalized totals remain unchanged; the new breakdown does not invent missing categories.'); }); test('DeepSeek cache misses retain their own meaning and never fabricate cache-write counts or provider identity', () => { const raw = { prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_cache_hit_tokens: 900, prompt_cache_miss_tokens: 100 }; const usage = reportedUsage({ protocol: 'chat-completions', model: 'arbitrary-proxy-alias' }, { usage: raw }), view = usage.tokenAccounting; assert.equal(view.scheme, 'read-and-miss'); assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.metrics.cacheWrite.tokens, null); assert.equal(view.metrics.uncached.tokens, 100); assert.equal(view.metrics.uncached.sources[0], 'usage.prompt_cache_miss_tokens'); assert.deepEqual(usage.raw, raw); assert.equal('provider' in view, false); }); test('missing differs from explicit zero; incomplete, invalid and overflowing receipts never become invented categories', () => { const partial = reportedUsage({}, { usage: { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 700 } } }).tokenAccounting; assert.equal(partial.metrics.cacheWrite.tokens, null); assert.equal(partial.metrics.uncached.tokens, null); assert.equal(partial.checks.inputPartition, 'partial'); const zero = reportedUsage({}, { usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, prompt_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 } } }).tokenAccounting; assert.equal(zero.metrics.cacheWrite.tokens, 0); assert.equal(zero.metrics.uncached.tokens, 0); assert.equal(zero.checks.inputPartition, 'consistent'); for (const value of [-1, '300', 1.2, Number.MAX_SAFE_INTEGER + 1]) { const view = reportedUsage({}, { usage: { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 600, cache_write_tokens: value } } }).tokenAccounting; assert.equal(view.metrics.cacheWrite.tokens, null); assert.equal(view.metrics.cacheWrite.status, 'invalid'); assert.ok(view.issues.includes('invalid_cacheWrite')); assert.equal(view.metrics.uncached.tokens, null); } assert.equal(reportedUsage({}, { usage: { prompt_tokens: Number.MAX_SAFE_INTEGER, completion_tokens: 1 } }), null); assert.equal(reportedUsage({}, {}), null); assert.equal(tokenAccountingView(null), null); }); test('conflicting categories, aliases, totals, lifetime partitions and excessive reasoning keep raw values with explicit issues', () => { const sample = structuredClone(usageSamples.responses); sample.raw.total_tokens = 1; sample.raw.input_tokens_details.cache_write_tokens_details.ttl_30m_tokens = 1; sample.raw.output_tokens_details.reasoning_tokens = 101; const usage = normalized(sample), view = usage.tokenAccounting; assert.equal(view.checks.total, 'inconsistent'); assert.equal(view.checks.cacheWriteLifetimes, 'inconsistent'); assert.ok(view.issues.includes('reasoning_exceeds_output')); assert.deepEqual(usage.raw, sample.raw); const raw = { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 800, cache_write_tokens: 300, uncached_tokens: 100 }, prompt_cache_hit_tokens: 700 }; const conflict = reportedUsage({}, { usage: raw }).tokenAccounting; assert.equal(conflict.checks.inputPartition, 'inconsistent'); assert.ok(conflict.issues.includes('cache_read_alias_mismatch')); }); test('legacy raw and normalized usage can be displayed without rewriting old receipt bytes or inventing unavailable cache history', () => { const old = { prompt_tokens: 1000, completion_tokens: 100, total_tokens: 1100, raw: structuredClone(usageSamples.chat.raw), protocol: 'chat-completions' }, before = JSON.stringify(old); assert.equal(tokenAccountingView(old).metrics.cacheWrite.tokens, 300); assert.equal(JSON.stringify(old), before); assert.equal(old.tokenAccounting, undefined); const legacy = { prompt_tokens: 150, completion_tokens: 12, total_tokens: 162, prompt_tokens_details: { cached_tokens: 30, cache_creation_tokens: 20 } }; const view = tokenAccountingView(legacy); assert.equal(view.projection, 'legacy-normalized'); assert.equal(view.metrics.cacheWrite.tokens, 20); assert.equal(view.metrics.uncached.tokens, null); }); async function run(t, sample, extra = {}) { const f = fixture(t, { modernDefaults: true }), model = await cacheUsageFixture(t, { ...sample, ...extra }); const provider = new Provider(), hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned usage fixture', model: usageModelId, protocol: sample.protocol, baseUrl: model.baseUrl, requiresKey: false, streaming: !!extra.streaming, modelIdentityPolicy: 'require-match' }); hub.select({ type: 'connection', id: connection.id }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '仅解释用量字段含义,不修改文件或运行命令。'); await until(() => !runtime.isActive(f.session.id), 15000); assert.equal(model.requests.length, 1); const attempt = hub.view().attempts[0], plans = [...f.store.core.context_plan.values()]; assert.equal(plans.length, 1); assert.equal(f.store.core.tool.size, 0); return { f, model, hub, runtime, provider, attempt, plan: plans[0] }; } for (const [name, sample] of Object.entries(usageSamples)) for (const streaming of [false, true]) test(`actual ${name} ${streaming ? 'SSE' : 'JSON'} persists original usage and source-bound categories in plan, attempt, Run and restart`, async t => { const { f, model, attempt, plan, runtime } = await run(t, sample, { streaming }), expected = normalized(sample); assert.equal(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported'); assert.equal(attempt.modelIdentity.status, 'matched'); assert.deepEqual(attempt.usage, expected); assert.deepEqual(plan.actualUsage, expected); const execution = f.store.core.run.get(f.session.runId); assert.equal(execution.usage.totalTokens, expected.total_tokens); assert.equal(execution.usage.unknownRequests, 0); assert.deepEqual(execution.consumption[0].usage, expected); assert.doesNotThrow(() => verifyPlanEvidence(f.store, f.session, plan)); assert.throws(() => verifyPlanEvidence(f.store, f.session, { ...plan, requestBytes: plan.requestBytes + 1 }), /不匹配/); assert.throws(() => verifyPlanEvidence(f.store, f.session, { ...plan, budget: { ...plan.budget, outputReservation: 1 } }), /不匹配/); const artifact = f.store.owned('artifact', attempt.responseArtifactId, f.session.id); assert.equal(artifact.sha256, createHash('sha256').update(model.responses[0]).digest('hex')); assert.equal(attempt.responseSha256, artifact.sha256); const original = JSON.stringify(attempt); await runtime.stopAll(); f.store.close(); const reopened = new Store(f.data); f.onCleanup(() => reopened.close()); const restored = new Provider().modelHub(reopened).view().attempts.find(value => value.id === attempt.id); assert.equal(JSON.stringify(restored), original); assert.equal(model.requests.length, 1); }); test('original upstream 400 retains reported cache usage without failover, retries or zero-cost inference', async t => { const { f, model, attempt, plan } = await run(t, usageSamples.chat, { status: 400 }); assert.equal(f.session.status, 'failed'); assert.notEqual(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported'); assert.deepEqual(attempt.usage.raw, usageSamples.chat.raw); assert.equal(attempt.usage.tokenAccounting.metrics.cacheWrite.tokens, 300); assert.equal(plan.actualUsage.tokenAccounting.metrics.cacheRead.tokens, 600); assert.match(f.session.failure.message, /自有上游原错误/); assert.equal(model.requests.length, 1); }); test('missing usage stays unknown, while unended SSE keeps reported categories without treating its reply as completed', async t => { const missing = await run(t, { protocol: 'chat-completions', raw: undefined }); assert.equal(missing.attempt.usageStatus, 'unknown'); assert.equal(missing.attempt.usage, null); assert.equal(missing.f.store.core.run.get(missing.f.session.runId).usage.unknownRequests, 1); const partial = await run(t, usageSamples.chat, { streaming: true, terminal: false }); assert.notEqual(partial.attempt.status, 'completed'); assert.equal(partial.attempt.usageStatus, 'reported'); assert.equal(partial.attempt.usage.tokenAccounting.metrics.cacheWrite.tokens, 300); assert.equal(partial.model.requests.length, 1); assert.equal(partial.f.store.core.tool.size, 0); });