Files
Agent-Workbench/tests/reading-reliability.test.mjs
T

159 lines
13 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { fileContinuation } from '../src/core/context/read-continuation.mjs';
import { filePage, readMessageBudget, readBatchBudget, spendReadBudget, orderReadCalls } from '../src/core/context/read-budget.mjs';
import { readLines } from '../src/server/files.mjs';
import { Runtime, validateToolArgs } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { requestHistory, requestMetrics } from '../src/shared/contracts/model-request.mjs';
import { RequestPlanner, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
import { fixture, compatibleFixture, call, completion, until, assertToolPairs } from './helpers.mjs';
const sha = text => createHash('sha256').update(text).digest('hex');
test('continuations distinguish unread source, later original line ranges and already returned result wrappers', t => {
const f = fixture(t), content = Array.from({ length: 310 }, (_, i) => '行' + i + '🙂').join('\r\n');
fs.writeFileSync(path.join(f.projectRoot, 'source.txt'), content);
const selected = readLines(f.project, { path: 'source.txt', start_line: 2, end_line: 301 });
const first = filePage(selected, {}, { maximumMessageBytes: 1500 });
const next = fileContinuation(first); assert.equal(next.range_complete, false);
assert.deepEqual(next.next_read_file, { path: 'source.txt', start_line: 2, end_line: 301, offset: first.next_offset, expected_sha256: sha(content) });
const tail = filePage(selected, { offset: selected.content.length - 3 });
assert.equal(tail.truncated, true, 'truncated can remain true after reaching the end of a selected range');
assert.equal(fileContinuation(tail).range_complete, true);
assert.deepEqual(fileContinuation(tail).next_read_file, { path: 'source.txt', start_line: 302, expected_sha256: sha(content) });
const entire = filePage(readLines(f.project, { path: 'source.txt', start_line: 302 }), {});
assert.equal(fileContinuation(entire).next_read_file, null);
assert.equal(fileContinuation({ path: 'source.txt', sha256: sha(content), truncated: true }), null, 'Legacy missing range is not invented');
assert.equal(readLines(f.project, next.next_read_file).sha256, first.sha256);
fs.appendFileSync(path.join(f.projectRoot, 'source.txt'), '\nchanged');
assert.throws(() => readLines(f.project, next.next_read_file), /SHA256已变化/);
assert.throws(() => validateToolArgs('read_file', { path: 'source.txt', expected_sha256: 'invalid' }), /SHA256/);
});
test('pure-file batch returns unused small-page space while mixed and historical allocation stays conservative', () => {
const plan = { readBudgetPolicyVersion: 2, budget: { inputBudget: 12352 }, nextReadFixedInputTokens: 9000, workingRetentionTokens: 600 };
const message = { role: 'assistant', content: '', tool_calls: [call('small', 'read_file', { path: 'small' }), call('large', 'read_file', { path: 'large' })] };
const batch = readBatchBudget(plan, message), before = batch.remainingBytes / 2;
spendReadBudget(batch, message.tool_calls[0], { path: 'small', content: 'x' });
assert.ok(batch.remainingBytes > before); assert.equal(batch.remainingCalls, 1);
assert.ok(readMessageBudget(plan, message) > readMessageBudget({ ...plan, readBudgetPolicyVersion: undefined }, message));
const mixed = { ...message, tool_calls: [...message.tool_calls, call('plan', 'plan_checks', {})] };
assert.equal(readBatchBudget(plan, mixed), null);
assert.equal(readMessageBudget(plan, mixed), readMessageBudget({ ...plan, readBudgetPolicyVersion: undefined }, mixed));
assert.equal(readBatchBudget(plan, { ...message, tool_calls: [call('artifact', 'read_artifact', { artifact_id: 'raw' })] }), null);
});
test('v3 pools mixed file/log reads, serves small sources first and leaves historical/mutating batches unchanged', () => {
const plan={readBudgetPolicyVersion:3,budget:{inputBudget:12352},nextReadFixedInputTokens:9000,workingRetentionTokens:600};
const calls=[call('large','read_file',{path:'large'}),call('log','read_artifact',{artifact_id:'log'}),call('small','read_file',{path:'small'})];
const message={role:'assistant',content:'',tool_calls:calls},batch=readBatchBudget(plan,message);
assert.ok(batch); assert.equal(batch.policyVersion,3);
assert.deepEqual(orderReadCalls(batch,calls,c=>({large:2584,small:51,log:6000})[c.id]).map(c=>c.id),['small','large','log']);
assert.deepEqual(calls.map(c=>c.id),['large','log','small']);
assert.ok(readMessageBudget(plan,message)>readMessageBudget({...plan,readBudgetPolicyVersion:2},message));
assert.equal(readBatchBudget({...plan,readBudgetPolicyVersion:2},message),null);
assert.equal(orderReadCalls(null,calls,()=>0),calls);
assert.equal(readBatchBudget(plan,{...message,tool_calls:[...calls,call('write','propose_file',{path:'out',content:'x'})]}),null);
assert.deepEqual(orderReadCalls(batch,calls,c=>{if(c.id==='large')throw Error('unavailable');return 10;}).map(c=>c.id),['log','small','large']);
});
test('v3 W-shaped mixed reading returns complete small modules and substantially larger first test pages in the next bounded request', async t => {
const f=fixture(t),text=fs.readFileSync(new URL('../benchmarks/live-delivery-v1/node/test/expense.test.mjs',import.meta.url),'utf8');
fs.writeFileSync(path.join(f.projectRoot,'expense.test.mjs'),text);
for(const [name,content] of [['amount','export const amount = 0;\n'],['summary','export const summary = [];\n'],['report','export const report = "";\n']])fs.writeFileSync(path.join(f.projectRoot,name+'.mjs'),content);
const artifact=f.store.artifactStore.write(f.session,null,'Original failure trace\n'.repeat(500),{kind:'check-log'});
f.store.core.artifact.set(artifact.id,artifact);f.store.save(f.session);
let requests=0,page;
const model=await compatibleFixture(t,(route,body)=>{
requests++;
if(requests===1)return{body:completion([call('large-first','read_file',{path:'expense.test.mjs',limit:8000}),
...['amount','summary','report'].map(name=>call(name,'read_file',{path:name+'.mjs',limit:4000})),
call('log','read_artifact',{artifact_id:artifact.id,limit:8000})])};
const results=body.messages.filter(m=>m.role==='tool').map(m=>JSON.parse(m.content));page=results.find(v=>v.path==='expense.test.mjs');
assert.ok(page);assert.ok(page.next_offset>945,JSON.stringify(page));assert.equal(page.content,text.slice(0,page.next_offset));
for(const name of ['amount','summary','report']){
const source=results.find(v=>v.path===name+'.mjs');assert.ok(source);assert.equal(source.next_offset,source.total_characters);
}
assert.ok(results.find(v=>v.artifact_id===artifact.id));assertToolPairs(body.messages);
assert.ok(requestMetrics(body).estimatedInputTokens<=12352);return{body:completion([],'完成指定只读核对。')};
});
f.store.saveConfig({mode:'compatible',model:'mixed-reading-fixture',baseUrl:model.baseUrl,contextLimit:16000,maxOutputTokens:2048,maxSteps:8});
const runtime=new Runtime(f.store,new Provider());f.onCleanup(()=>runtime.stopAll());
runtime.start(f.session.id,'只读取指定测试、三模块和原检查日志,核对真实依据;无需测试,不修改文件。');
await runtime.active.get(f.session.id).promise;
assert.equal(requests,2,JSON.stringify(f.session.failure));assert.equal(f.session.status,'completed');
assert.ok(page.next_offset>945);assertToolPairs(f.session.providerMessages);
for(const plan of f.store.core.context_plan.values())assert.doesNotThrow(()=>verifyPlanEvidence(f.store,f.session,plan));
});
test('observed upstream high input pauses before a new request, preserves reported usage and distinguishes the limiting ledger', t => {
const f = fixture(t), config = { ...f.store.config, model: 'owned', contextLimit: 16000, maxOutputTokens: 2048,
maxRunTokens: 90000, maxTaskTokens: 90000 };
const run = f.store.beginRun(f.session, 'Original goal and all requirements remain.', false, config), task = f.store.core.task.get(run.taskId);
f.session.providerMessages.push({ role: 'user', content: task.goal });
run.usage.totalTokens = 72571; task.usage.totalTokens = 72571;
run.consumption = [{ runId: run.id, attemptId: 'owned-high-input', model: config.model, protocol: 'chat-completions', endpoint: config.baseUrl,
usageStatus: 'reported', usage: { prompt_tokens: 27164, completion_tokens: 303, total_tokens: 27467 },
localEstimate: { schemaVersion: 1, inputTokens: 12168, outputTokens: 2048 } }];
const original = structuredClone(run.consumption), planner = new RequestPlanner(f.store);
const prepare = system => planner.prepare(f.session, run, config, { system, tools: [], stepId: 'owned-step' });
for (const expected of ['token_limit', 'task_token_limit']) {
if (expected === 'task_token_limit') task.usage.totalTokens++;
assert.throws(() => prepare('Read only original input.'), error => {
assert.equal(error.reason, expected); assert.equal(error.contextBudget.pauseKind, 'request_reservation');
assert.ok(error.contextBudget.requiredInputTokens <= error.contextBudget.inputBudget);
assert.ok(error.contextBudget.tokenReservation.reservedTokens > error.contextBudget.remainingTokens);
assert.match(error.message, /输入可容纳/); return true;
});
}
assert.throws(() => prepare('完整原目标'.repeat(10000)), error => { assert.equal(error.reason, 'context_limit'); assert.equal(error.contextBudget.pauseKind, undefined); return true; });
assert.deepEqual(run.consumption, original); assert.equal(f.store.core.context_plan.size, 0, 'No unaffordable request plan is dispatched');
});
for (const protocol of ['chat-completions', 'responses', 'anthropic-messages']) {
test(protocol + ': complete actual Unicode/CRLF pages and small source survive the next bounded wire request', { timeout: 30000 }, async t => {
const f = fixture(t), text = ('long single line 雪🙂"\\'.repeat(100) + '\r\n').repeat(3), small = 'export const value = "原文🙂";\r\n';
fs.writeFileSync(path.join(f.projectRoot, 'long.txt'), text); fs.writeFileSync(path.join(f.projectRoot, 'small.mjs'), small);
let n = 0, next = 0, reconstructed = '', sawSmall = false;
const server = await compatibleFixture(t, (_route, body) => {
n++; const messages = requestHistory(body); assertToolPairs(messages);
const actual = messages.filter(m => m.role === 'tool').map(m => JSON.parse(m.content));
const page = actual.filter(value => value.path === 'long.txt').at(-1);
if (n > 1) {
assert.ok(page, 'The actual newest page must reach the model in full'); assert.equal(page.offset, next);
assert.equal(page.content, text.slice(page.offset, page.next_offset));
assert.ok(page.content.isWellFormed());
reconstructed += page.content; next = page.next_offset;
sawSmall ||= actual.some(value => value.path === 'small.mjs' && value.content === small);
}
const calls = n === 1 ? [call('small-first', 'read_file', { path: 'small.mjs' }), call('long-first', 'read_file', { path: 'long.txt', limit: 16000 })]
: next < text.length ? [call('long-next-' + n, 'read_file', { path: 'long.txt', offset: next, start_line: 1, end_line: 4, limit: 16000, expected_sha256: sha(text) })] : [];
if (protocol === 'chat-completions') return { body: completion(calls, calls.length ? '' : 'Done') };
if (protocol === 'responses') return { body: { id: 'resp_' + n, status: 'completed',
output: calls.length ? calls.map(c => ({ type: 'function_call', id: 'fc_' + c.id, call_id: c.id, name: c.function.name, arguments: c.function.arguments, status: 'completed' }))
: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'Done' }] }], usage: { input_tokens: 123, output_tokens: 45 } } };
return { body: { id: 'msg_' + n, type: 'message', role: 'assistant', model: 'owned-pages', stop_reason: calls.length ? 'tool_use' : 'end_turn',
content: calls.length ? calls.map(c => ({ type: 'tool_use', id: c.id, name: c.function.name, input: JSON.parse(c.function.arguments) })) : [{ type: 'text', text: 'Done' }], usage: { input_tokens: 123, output_tokens: 45 } } };
});
f.store.saveConfig({ contextLimit: 16000, maxOutputTokens: 2048, maxSteps: 12 });
const provider = new Provider({ retryPolicy: { attempts: 1 } }), hub = provider.modelHub(f.store);
const connection = hub.upsert({ name: 'Owned pages', protocol, baseUrl: server.baseUrl, model: 'owned-pages', requiresKey: false, contextLimit: 16000, maxOutputTokens: 2048 });
hub.select({ type: 'connection', id: connection.id }); const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Read these two original source files exactly.');
await until(() => !runtime.isActive(f.session.id), 25000);
assert.equal(f.session.status, 'completed', JSON.stringify({ failure: f.session.failure, run: f.store.core.run.get(f.session.runId), next, expected: text.length }));
assert.equal(reconstructed, text); assert.ok(sawSmall);
for (const plan of f.store.core.context_plan.values()) {
verifyPlanEvidence(f.store, f.session, plan);
const body = JSON.parse(f.store.artifactStore.read(f.store.core.artifact.get(plan.requestArtifactId)));
assert.ok(requestMetrics(body).estimatedInputTokens <= 12352); assert.equal(plan.budget.safetyTokens, 1600);
assert.equal(plan.readBudgetPolicyVersion, 3);
}
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'long.txt'), 'utf8'), text);
});
}