import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { createHash } from 'node:crypto'; import { fileContinuation } from '../src/core/context/read-continuation.mjs'; import { filePage, readMessageBudget, readBatchBudget, spendReadBudget } from '../src/core/context/read-budget.mjs'; import { readLines } from '../src/server/files.mjs'; import { Runtime, validateToolArgs } from '../src/server/runtime.mjs'; import { Provider } from '../src/server/provider.mjs'; import { requestHistory, requestMetrics } from '../src/shared/contracts/model-request.mjs'; import { RequestPlanner, verifyPlanEvidence } from '../src/core/context/request-plan.mjs'; import { fixture, compatibleFixture, call, completion, until, assertToolPairs } from './helpers.mjs'; const sha = text => createHash('sha256').update(text).digest('hex'); test('continuations distinguish unread source, later original line ranges and already returned result wrappers', t => { const f = fixture(t), content = Array.from({ length: 310 }, (_, i) => '行' + i + '🙂').join('\r\n'); fs.writeFileSync(path.join(f.projectRoot, 'source.txt'), content); const selected = readLines(f.project, { path: 'source.txt', start_line: 2, end_line: 301 }); const first = filePage(selected, {}, { maximumMessageBytes: 1500 }); const next = fileContinuation(first); assert.equal(next.range_complete, false); assert.deepEqual(next.next_read_file, { path: 'source.txt', start_line: 2, end_line: 301, offset: first.next_offset, expected_sha256: sha(content) }); const tail = filePage(selected, { offset: selected.content.length - 3 }); assert.equal(tail.truncated, true, 'truncated can remain true after reaching the end of a selected range'); assert.equal(fileContinuation(tail).range_complete, true); assert.deepEqual(fileContinuation(tail).next_read_file, { path: 'source.txt', start_line: 302, expected_sha256: sha(content) }); const entire = filePage(readLines(f.project, { path: 'source.txt', start_line: 302 }), {}); assert.equal(fileContinuation(entire).next_read_file, null); assert.equal(fileContinuation({ path: 'source.txt', sha256: sha(content), truncated: true }), null, 'Legacy missing range is not invented'); assert.equal(readLines(f.project, next.next_read_file).sha256, first.sha256); fs.appendFileSync(path.join(f.projectRoot, 'source.txt'), '\nchanged'); assert.throws(() => readLines(f.project, next.next_read_file), /SHA256已变化/); assert.throws(() => validateToolArgs('read_file', { path: 'source.txt', expected_sha256: 'invalid' }), /SHA256/); }); test('pure-file batch returns unused small-page space while mixed and historical allocation stays conservative', () => { const plan = { readBudgetPolicyVersion: 2, budget: { inputBudget: 12352 }, nextReadFixedInputTokens: 9000, workingRetentionTokens: 600 }; const message = { role: 'assistant', content: '', tool_calls: [call('small', 'read_file', { path: 'small' }), call('large', 'read_file', { path: 'large' })] }; const batch = readBatchBudget(plan, message), before = batch.remainingBytes / 2; spendReadBudget(batch, message.tool_calls[0], { path: 'small', content: 'x' }); assert.ok(batch.remainingBytes > before); assert.equal(batch.remainingCalls, 1); assert.ok(readMessageBudget(plan, message) > readMessageBudget({ ...plan, readBudgetPolicyVersion: undefined }, message)); const mixed = { ...message, tool_calls: [...message.tool_calls, call('plan', 'plan_checks', {})] }; assert.equal(readBatchBudget(plan, mixed), null); assert.equal(readMessageBudget(plan, mixed), readMessageBudget({ ...plan, readBudgetPolicyVersion: undefined }, mixed)); assert.equal(readBatchBudget(plan, { ...message, tool_calls: [call('artifact', 'read_artifact', { artifact_id: 'raw' })] }), null); }); test('observed upstream high input pauses before a new request, preserves reported usage and distinguishes the limiting ledger', t => { const f = fixture(t), config = { ...f.store.config, model: 'owned', contextLimit: 16000, maxOutputTokens: 2048, maxRunTokens: 90000, maxTaskTokens: 90000 }; const run = f.store.beginRun(f.session, 'Original goal and all requirements remain.', false, config), task = f.store.core.task.get(run.taskId); f.session.providerMessages.push({ role: 'user', content: task.goal }); run.usage.totalTokens = 72571; task.usage.totalTokens = 72571; run.consumption = [{ runId: run.id, attemptId: 'owned-high-input', model: config.model, protocol: 'chat-completions', endpoint: config.baseUrl, usageStatus: 'reported', usage: { prompt_tokens: 27164, completion_tokens: 303, total_tokens: 27467 }, localEstimate: { schemaVersion: 1, inputTokens: 12168, outputTokens: 2048 } }]; const original = structuredClone(run.consumption), planner = new RequestPlanner(f.store); const prepare = system => planner.prepare(f.session, run, config, { system, tools: [], stepId: 'owned-step' }); for (const expected of ['token_limit', 'task_token_limit']) { if (expected === 'task_token_limit') task.usage.totalTokens++; assert.throws(() => prepare('Read only original input.'), error => { assert.equal(error.reason, expected); assert.equal(error.contextBudget.pauseKind, 'request_reservation'); assert.ok(error.contextBudget.requiredInputTokens <= error.contextBudget.inputBudget); assert.ok(error.contextBudget.tokenReservation.reservedTokens > error.contextBudget.remainingTokens); assert.match(error.message, /输入可容纳/); return true; }); } assert.throws(() => prepare('完整原目标'.repeat(10000)), error => { assert.equal(error.reason, 'context_limit'); assert.equal(error.contextBudget.pauseKind, undefined); return true; }); assert.deepEqual(run.consumption, original); assert.equal(f.store.core.context_plan.size, 0, 'No unaffordable request plan is dispatched'); }); for (const protocol of ['chat-completions', 'responses', 'anthropic-messages']) { test(protocol + ': complete actual Unicode/CRLF pages and small source survive the next bounded wire request', { timeout: 30000 }, async t => { const f = fixture(t), text = ('long single line 雪🙂"\\'.repeat(100) + '\r\n').repeat(3), small = 'export const value = "原文🙂";\r\n'; fs.writeFileSync(path.join(f.projectRoot, 'long.txt'), text); fs.writeFileSync(path.join(f.projectRoot, 'small.mjs'), small); let n = 0, next = 0, reconstructed = '', sawSmall = false; const server = await compatibleFixture(t, (_route, body) => { n++; const messages = requestHistory(body); assertToolPairs(messages); const actual = messages.filter(m => m.role === 'tool').map(m => JSON.parse(m.content)); const page = actual.filter(value => value.path === 'long.txt').at(-1); if (n > 1) { assert.ok(page, 'The actual newest page must reach the model in full'); assert.equal(page.offset, next); assert.equal(page.content, text.slice(page.offset, page.next_offset)); assert.ok(page.content.isWellFormed()); reconstructed += page.content; next = page.next_offset; sawSmall ||= actual.some(value => value.path === 'small.mjs' && value.content === small); } const calls = n === 1 ? [call('small-first', 'read_file', { path: 'small.mjs' }), call('long-first', 'read_file', { path: 'long.txt', limit: 16000 })] : next < text.length ? [call('long-next-' + n, 'read_file', { path: 'long.txt', offset: next, start_line: 1, end_line: 4, limit: 16000, expected_sha256: sha(text) })] : []; if (protocol === 'chat-completions') return { body: completion(calls, calls.length ? '' : 'Done') }; if (protocol === 'responses') return { body: { id: 'resp_' + n, status: 'completed', output: calls.length ? calls.map(c => ({ type: 'function_call', id: 'fc_' + c.id, call_id: c.id, name: c.function.name, arguments: c.function.arguments, status: 'completed' })) : [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'Done' }] }], usage: { input_tokens: 123, output_tokens: 45 } } }; return { body: { id: 'msg_' + n, type: 'message', role: 'assistant', model: 'owned-pages', stop_reason: calls.length ? 'tool_use' : 'end_turn', content: calls.length ? calls.map(c => ({ type: 'tool_use', id: c.id, name: c.function.name, input: JSON.parse(c.function.arguments) })) : [{ type: 'text', text: 'Done' }], usage: { input_tokens: 123, output_tokens: 45 } } }; }); f.store.saveConfig({ contextLimit: 16000, maxOutputTokens: 2048, maxSteps: 12 }); const provider = new Provider({ retryPolicy: { attempts: 1 } }), hub = provider.modelHub(f.store); const connection = hub.upsert({ name: 'Owned pages', protocol, baseUrl: server.baseUrl, model: 'owned-pages', requiresKey: false, contextLimit: 16000, maxOutputTokens: 2048 }); hub.select({ type: 'connection', id: connection.id }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, 'Read these two original source files exactly.'); await until(() => !runtime.isActive(f.session.id), 25000); assert.equal(f.session.status, 'completed', JSON.stringify({ failure: f.session.failure, run: f.store.core.run.get(f.session.runId), next, expected: text.length })); assert.equal(reconstructed, text); assert.ok(sawSmall); for (const plan of f.store.core.context_plan.values()) { verifyPlanEvidence(f.store, f.session, plan); const body = JSON.parse(f.store.artifactStore.read(f.store.core.artifact.get(plan.requestArtifactId))); assert.ok(requestMetrics(body).estimatedInputTokens <= 12352); assert.equal(plan.budget.safetyTokens, 1600); assert.equal(plan.readBudgetPolicyVersion, 2); } assert.equal(fs.readFileSync(path.join(f.projectRoot, 'long.txt'), 'utf8'), text); }); }