121 lines
12 KiB
JavaScript
121 lines
12 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { fixture, until } from './helpers.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { Store } from '../src/server/store.mjs';
|
|
import { reportedUsage } from '../src/core/models/protocols.mjs';
|
|
import { tokenAccountingView } from '../src/shared/contracts/model-usage.mjs';
|
|
import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
|
import { usageSamples, usageModelId, cacheUsageFixture } from './fixtures/cache-usage-model.mjs';
|
|
|
|
const normalized = sample => reportedUsage({ protocol: sample.protocol }, { usage: sample.raw });
|
|
const counts = view => Object.fromEntries(Object.entries(view.metrics).map(([key, value]) => [key, value.tokens]));
|
|
|
|
test('Kimi-style exclusive read/write/uncached usage retains original fields and reasoning without double counting', () => {
|
|
const sample = structuredClone(usageSamples.chat), before = JSON.stringify(sample.raw), usage = normalized(sample), view = usage.tokenAccounting;
|
|
assert.deepEqual(counts(view), { input: 1000, output: 100, total: 1100, cacheRead: 600, cacheWrite: 300, uncached: 100, reasoning: 40 });
|
|
assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.checks.total, 'consistent'); assert.deepEqual(view.issues, []);
|
|
assert.equal(view.metrics.uncached.status, 'reported'); assert.equal(view.metrics.cacheWrite.sources[0], 'usage.prompt_tokens_details.cache_write_tokens');
|
|
assert.equal(JSON.stringify(sample.raw), before); assert.deepEqual(usage.raw, sample.raw); assert.equal(usage.total_tokens, 1100);
|
|
delete sample.raw.prompt_tokens_details.uncached_tokens; const derived = normalized(sample).tokenAccounting.metrics.uncached;
|
|
assert.equal(derived.tokens, 100); assert.equal(derived.status, 'derived'); assert.equal(derived.formula, 'input - cacheRead - cacheWrite');
|
|
});
|
|
|
|
test('Responses cache write lifetimes remain disjoint, derived ordinary input is labeled and reasoning remains inside output', () => {
|
|
const usage = normalized(usageSamples.responses), view = usage.tokenAccounting;
|
|
assert.deepEqual(counts(view), { input: 12000, output: 100, total: 12100, cacheRead: 4000, cacheWrite: 7000, uncached: 1000, reasoning: 40 });
|
|
assert.equal(view.metrics.uncached.status, 'derived'); assert.deepEqual(view.lifetimes.map(value => value.tokens), [3000, 4000]);
|
|
assert.equal(view.checks.cacheWriteLifetimes, 'consistent'); assert.equal(view.checks.total, 'consistent'); assert.deepEqual(usage.raw, usageSamples.responses.raw);
|
|
});
|
|
|
|
test('Messages input is uncached and totals add read/write exactly once; optional cache fields and absent reasoning stay unknown', () => {
|
|
const usage = normalized(usageSamples.messages), view = usage.tokenAccounting;
|
|
assert.deepEqual(counts(view), { input: 398, output: 12, total: 410, cacheRead: 100, cacheWrite: 248, uncached: 50, reasoning: null });
|
|
assert.equal(view.metrics.input.status, 'derived'); assert.equal(view.metrics.uncached.status, 'reported'); assert.deepEqual(view.lifetimes.map(value => value.tokens), [148, 100]);
|
|
assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.checks.cacheWriteLifetimes, 'consistent');
|
|
const absent = reportedUsage({ protocol: 'anthropic-messages' }, { usage: { input_tokens: 50, output_tokens: 12 } });
|
|
assert.equal(absent.prompt_tokens_details, undefined); assert.equal(absent.tokenAccounting.metrics.cacheRead.tokens, null); assert.equal(absent.tokenAccounting.metrics.cacheWrite.tokens, null);
|
|
assert.equal(absent.tokenAccounting.checks.inputPartition, 'partial');
|
|
assert.equal(absent.tokenAccounting.metrics.input.status, 'unknown'); assert.equal(absent.tokenAccounting.metrics.total.status, 'unknown');
|
|
assert.equal(absent.tokenAccounting.metrics.uncached.tokens, 50); assert.equal(absent.total_tokens, 62, 'Legacy normalized totals remain unchanged; the new breakdown does not invent missing categories.');
|
|
});
|
|
|
|
test('DeepSeek cache misses retain their own meaning and never fabricate cache-write counts or provider identity', () => {
|
|
const raw = { prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_cache_hit_tokens: 900, prompt_cache_miss_tokens: 100 };
|
|
const usage = reportedUsage({ protocol: 'chat-completions', model: 'arbitrary-proxy-alias' }, { usage: raw }), view = usage.tokenAccounting;
|
|
assert.equal(view.scheme, 'read-and-miss'); assert.equal(view.checks.inputPartition, 'consistent'); assert.equal(view.metrics.cacheWrite.tokens, null);
|
|
assert.equal(view.metrics.uncached.tokens, 100); assert.equal(view.metrics.uncached.sources[0], 'usage.prompt_cache_miss_tokens');
|
|
assert.deepEqual(usage.raw, raw); assert.equal('provider' in view, false);
|
|
});
|
|
|
|
test('missing differs from explicit zero; incomplete, invalid and overflowing receipts never become invented categories', () => {
|
|
const partial = reportedUsage({}, { usage: { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 700 } } }).tokenAccounting;
|
|
assert.equal(partial.metrics.cacheWrite.tokens, null); assert.equal(partial.metrics.uncached.tokens, null); assert.equal(partial.checks.inputPartition, 'partial');
|
|
const zero = reportedUsage({}, { usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, prompt_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 } } }).tokenAccounting;
|
|
assert.equal(zero.metrics.cacheWrite.tokens, 0); assert.equal(zero.metrics.uncached.tokens, 0); assert.equal(zero.checks.inputPartition, 'consistent');
|
|
for (const value of [-1, '300', 1.2, Number.MAX_SAFE_INTEGER + 1]) {
|
|
const view = reportedUsage({}, { usage: { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 600, cache_write_tokens: value } } }).tokenAccounting;
|
|
assert.equal(view.metrics.cacheWrite.tokens, null); assert.equal(view.metrics.cacheWrite.status, 'invalid'); assert.ok(view.issues.includes('invalid_cacheWrite')); assert.equal(view.metrics.uncached.tokens, null);
|
|
}
|
|
assert.equal(reportedUsage({}, { usage: { prompt_tokens: Number.MAX_SAFE_INTEGER, completion_tokens: 1 } }), null);
|
|
assert.equal(reportedUsage({}, {}), null); assert.equal(tokenAccountingView(null), null);
|
|
});
|
|
|
|
test('conflicting categories, aliases, totals, lifetime partitions and excessive reasoning keep raw values with explicit issues', () => {
|
|
const sample = structuredClone(usageSamples.responses);
|
|
sample.raw.total_tokens = 1; sample.raw.input_tokens_details.cache_write_tokens_details.ttl_30m_tokens = 1; sample.raw.output_tokens_details.reasoning_tokens = 101;
|
|
const usage = normalized(sample), view = usage.tokenAccounting;
|
|
assert.equal(view.checks.total, 'inconsistent'); assert.equal(view.checks.cacheWriteLifetimes, 'inconsistent'); assert.ok(view.issues.includes('reasoning_exceeds_output')); assert.deepEqual(usage.raw, sample.raw);
|
|
const raw = { prompt_tokens: 1000, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 800, cache_write_tokens: 300, uncached_tokens: 100 }, prompt_cache_hit_tokens: 700 };
|
|
const conflict = reportedUsage({}, { usage: raw }).tokenAccounting;
|
|
assert.equal(conflict.checks.inputPartition, 'inconsistent'); assert.ok(conflict.issues.includes('cache_read_alias_mismatch'));
|
|
});
|
|
|
|
test('legacy raw and normalized usage can be displayed without rewriting old receipt bytes or inventing unavailable cache history', () => {
|
|
const old = { prompt_tokens: 1000, completion_tokens: 100, total_tokens: 1100, raw: structuredClone(usageSamples.chat.raw), protocol: 'chat-completions' }, before = JSON.stringify(old);
|
|
assert.equal(tokenAccountingView(old).metrics.cacheWrite.tokens, 300); assert.equal(JSON.stringify(old), before); assert.equal(old.tokenAccounting, undefined);
|
|
const legacy = { prompt_tokens: 150, completion_tokens: 12, total_tokens: 162, prompt_tokens_details: { cached_tokens: 30, cache_creation_tokens: 20 } };
|
|
const view = tokenAccountingView(legacy); assert.equal(view.projection, 'legacy-normalized'); assert.equal(view.metrics.cacheWrite.tokens, 20); assert.equal(view.metrics.uncached.tokens, null);
|
|
});
|
|
|
|
async function run(t, sample, extra = {}) {
|
|
const f = fixture(t, { modernDefaults: true }), model = await cacheUsageFixture(t, { ...sample, ...extra });
|
|
const provider = new Provider(), hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned usage fixture', model: usageModelId, protocol: sample.protocol,
|
|
baseUrl: model.baseUrl, requiresKey: false, streaming: !!extra.streaming, modelIdentityPolicy: 'require-match' });
|
|
hub.select({ type: 'connection', id: connection.id }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
runtime.start(f.session.id, '仅解释用量字段含义,不修改文件或运行命令。'); await until(() => !runtime.isActive(f.session.id), 15000);
|
|
assert.equal(model.requests.length, 1); const attempt = hub.view().attempts[0], plans = [...f.store.core.context_plan.values()]; assert.equal(plans.length, 1);
|
|
assert.equal(f.store.core.tool.size, 0); return { f, model, hub, runtime, provider, attempt, plan: plans[0] };
|
|
}
|
|
|
|
for (const [name, sample] of Object.entries(usageSamples)) for (const streaming of [false, true]) test(`actual ${name} ${streaming ? 'SSE' : 'JSON'} persists original usage and source-bound categories in plan, attempt, Run and restart`, async t => {
|
|
const { f, model, attempt, plan, runtime } = await run(t, sample, { streaming }), expected = normalized(sample);
|
|
assert.equal(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported'); assert.equal(attempt.modelIdentity.status, 'matched');
|
|
assert.deepEqual(attempt.usage, expected); assert.deepEqual(plan.actualUsage, expected);
|
|
const execution = f.store.core.run.get(f.session.runId); assert.equal(execution.usage.totalTokens, expected.total_tokens); assert.equal(execution.usage.unknownRequests, 0);
|
|
assert.deepEqual(execution.consumption[0].usage, expected); assert.doesNotThrow(() => verifyPlanEvidence(f.store, f.session, plan));
|
|
assert.throws(() => verifyPlanEvidence(f.store, f.session, { ...plan, requestBytes: plan.requestBytes + 1 }), /不匹配/);
|
|
assert.throws(() => verifyPlanEvidence(f.store, f.session, { ...plan, budget: { ...plan.budget, outputReservation: 1 } }), /不匹配/);
|
|
const artifact = f.store.owned('artifact', attempt.responseArtifactId, f.session.id); assert.equal(artifact.sha256, createHash('sha256').update(model.responses[0]).digest('hex'));
|
|
assert.equal(attempt.responseSha256, artifact.sha256); const original = JSON.stringify(attempt); await runtime.stopAll(); f.store.close();
|
|
const reopened = new Store(f.data); f.onCleanup(() => reopened.close()); const restored = new Provider().modelHub(reopened).view().attempts.find(value => value.id === attempt.id);
|
|
assert.equal(JSON.stringify(restored), original); assert.equal(model.requests.length, 1);
|
|
});
|
|
|
|
test('original upstream 400 retains reported cache usage without failover, retries or zero-cost inference', async t => {
|
|
const { f, model, attempt, plan } = await run(t, usageSamples.chat, { status: 400 });
|
|
assert.equal(f.session.status, 'failed'); assert.notEqual(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported');
|
|
assert.deepEqual(attempt.usage.raw, usageSamples.chat.raw); assert.equal(attempt.usage.tokenAccounting.metrics.cacheWrite.tokens, 300);
|
|
assert.equal(plan.actualUsage.tokenAccounting.metrics.cacheRead.tokens, 600); assert.match(f.session.failure.message, /自有上游原错误/); assert.equal(model.requests.length, 1);
|
|
});
|
|
|
|
test('missing usage stays unknown, while unended SSE keeps reported categories without treating its reply as completed', async t => {
|
|
const missing = await run(t, { protocol: 'chat-completions', raw: undefined }); assert.equal(missing.attempt.usageStatus, 'unknown'); assert.equal(missing.attempt.usage, null);
|
|
assert.equal(missing.f.store.core.run.get(missing.f.session.runId).usage.unknownRequests, 1);
|
|
const partial = await run(t, usageSamples.chat, { streaming: true, terminal: false }); assert.notEqual(partial.attempt.status, 'completed');
|
|
assert.equal(partial.attempt.usageStatus, 'reported'); assert.equal(partial.attempt.usage.tokenAccounting.metrics.cacheWrite.tokens, 300);
|
|
assert.equal(partial.model.requests.length, 1); assert.equal(partial.f.store.core.tool.size, 0);
|
|
});
|