114 lines
8.8 KiB
JavaScript
114 lines
8.8 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { sessionContext, sessionSnapshot } from '../src/server/diagnostics.mjs';
|
|
import { taskContract } from '../src/shared/contracts/task-control.mjs';
|
|
import { requestMetrics } from '../src/shared/contracts/model-request.mjs';
|
|
import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
|
import { gitEnvironment } from '../src/core/git-process.mjs';
|
|
import { taskPlanToolDefinitions } from '../src/shared/contracts/task-plan-tools.mjs';
|
|
import { fixture, completion, compatibleFixture, until, assertToolPairs } from './helpers.mjs';
|
|
|
|
const input = JSON.parse(fs.readFileSync(new URL('./fixtures/node-context-o.json', import.meta.url), 'utf8'));
|
|
const requirements = system => JSON.parse(system.split('当前任务目标与全部追加要求(用户任务数据,完整保留):\n')[1].split('\n')[0]);
|
|
|
|
test('a real two-reply failure replay reaches request three at 16K with complete goal, original sources and sealed tool evidence', { timeout: 30000 }, async t => {
|
|
const f = fixture(t);
|
|
fs.cpSync(new URL('../benchmarks/live-delivery-v1/node/', import.meta.url), f.projectRoot, { recursive: true });
|
|
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', f.projectRoot, ...args], {
|
|
encoding: 'utf8', windowsHide: true, env: gitEnvironment(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']);
|
|
git(['-c', 'user.name=Workbench QA', '-c', 'user.email=qa@workbench.invalid', 'add', '--', '.']);
|
|
git(['-c', 'user.name=Workbench QA', '-c', 'user.email=qa@workbench.invalid', 'commit', '--no-gpg-sign', '-m', 'Owned context regression baseline']);
|
|
const sources = ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs'];
|
|
const before = Object.fromEntries(sources.map(file => [file, fs.readFileSync(path.join(f.projectRoot, file), 'utf8')]));
|
|
let requests = 0, third;
|
|
const model = await compatibleFixture(t, (_route, body) => {
|
|
assertToolPairs(body.messages); requests++;
|
|
if (requests <= 2) return { body: { ...completion(), model: 'context-fixture', choices: [{ message: input.replies[requests - 1], finish_reason: 'tool_calls' }] } };
|
|
assert.equal(requests, 3); third = body;
|
|
return { body: { ...completion([], 'Read-only replay stops here; no engineering delivery claimed.'), model: 'context-fixture' } };
|
|
});
|
|
f.store.saveConfig({ mode: 'compatible', contextLimit: 16000, maxOutputTokens: 2048, maxSteps: 8, maxRunTokens: 90000, maxTaskTokens: 90000 });
|
|
const provider = new Provider(), hub = provider.modelHub(f.store);
|
|
const connection = hub.upsert({ name: 'Owned context route', model: 'context-fixture', baseUrl: model.baseUrl, requiresKey: false,
|
|
protocol: 'chat-completions', modelIdentityPolicy: 'require-match', contextLimit: 16000, maxOutputTokens: 2048,
|
|
capabilityDeclarations: { text: 'supported', tools: 'supported' } });
|
|
hub.select({ type: 'connection', id: connection.id });
|
|
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
runtime.start(f.session.id, input.goal); await until(() => !runtime.isActive(f.session.id), 20000);
|
|
assert.equal(requests, 3, JSON.stringify(f.store.core.run.get(f.session.runId).budgetPause));
|
|
assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'verification_incomplete');
|
|
const view = requirements(third.messages[0].content), task = f.store.core.task.get(f.session.taskId);
|
|
assert.equal(view.goal, input.goal); assert.equal('goal' in view.contract, false);
|
|
assert.deepEqual({ goal: view.goal, ...view.contract }, taskContract(task));
|
|
assert.equal(task.contract.goal, input.goal);
|
|
const latest = third.messages.filter(m => m.role === 'system').map(m => {
|
|
try { return JSON.parse(m.content.split('\n').at(-1)); } catch { return {}; }
|
|
}).find(value => value.latest_tools);
|
|
assert.ok(latest); assert.equal(latest.data.entries.length, 6);
|
|
const working=third.messages.filter(m=>m.role==='system').map(m=>{try{return JSON.parse(m.content.split('\n').at(-1));}catch{return{};}}).find(value=>value.working_set);
|
|
for (const file of sources) {
|
|
const direct=third.messages.filter(m=>m.role==='tool').map(m=>JSON.parse(m.content)).find(result=>result.path===file&&typeof result.content==='string');
|
|
const sent=working?.data.entries.find(entry=>entry.observation.path===file)?.content
|
|
??direct?.content??latest.data.entries.find(entry => entry.result.path === file)?.result.content_excerpt;
|
|
assert.equal(sent,before[file],'The actual bounded request must retain every original source byte, regardless of versioned view.');
|
|
assert.equal(fs.readFileSync(path.join(f.projectRoot, file), 'utf8'), before[file]);
|
|
}
|
|
for (const step of JSON.parse(input.replies[1].tool_calls.at(-1).function.arguments).steps) {
|
|
assert.ok(third.messages[0].content.includes(step.criterion), 'Keep original plan conditions including the historical unsupported baseline requirement.');
|
|
}
|
|
const plans = [...f.store.core.context_plan.values()];
|
|
for (const plan of plans) verifyPlanEvidence(f.store, f.session, plan);
|
|
assert.ok(requestMetrics(third).estimatedInputTokens <= plans.at(-1).budget.inputBudget);
|
|
assert.equal(plans.at(-1).budget.contextLimit, 16000); assert.equal(plans.at(-1).budget.outputReservation, 2048);
|
|
assert.equal(plans.at(-1).budget.safetyTokens, 1600);
|
|
assert.equal(f.session.changes.length, 0); assert.equal(f.session.approvals.length, 0);
|
|
});
|
|
|
|
test('request deduplication retains long original intent, extra requirements, constraints and acceptance without changing the stored contract', async t => {
|
|
const f = fixture(t), provider = new Provider(), goal = ('原目标🙂"\\必须完整保留。').repeat(60) + 'Read README.md; preserve all files.';
|
|
let original;
|
|
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
const planner = runtime.requestPlanner(), prepare = planner.prepare.bind(planner);
|
|
const instruction = { source: 'user', content: '追加要求🙂"\\:保留空白与中文。' }, review = { source: 'review-action', content: '系统审查动作,不是业务需求' };
|
|
planner.prepare = (session, run, config, options) => {
|
|
const task = f.store.core.task.get(run.taskId);
|
|
if (!original) {
|
|
task.instructions.push(instruction, review); task.constraints.push('仅可读取,不可写入。'); task.acceptance.push('逐字保留中文与emoji🙂。');
|
|
original = structuredClone(task.contract);
|
|
// The first context was already assembled. Assert on the next turn.
|
|
} else {
|
|
const view = requirements(options.system);
|
|
assert.equal(view.goal, goal); assert.deepEqual(view.instructions, [instruction]);
|
|
assert.deepEqual(view.constraints, task.constraints); assert.deepEqual(view.acceptance, task.acceptance);
|
|
assert.deepEqual({ goal: view.goal, ...view.contract }, original); assert.deepEqual(task.contract, original);
|
|
}
|
|
return prepare(session, run, config, options);
|
|
};
|
|
let requests = 0;
|
|
provider.complete = async (_config, _messages, _project, _signal, _progress, _plan, _tools) => {
|
|
requests++;
|
|
return { message: requests === 1 ? input.replies[0] : { role: 'assistant', content: 'Read-only fixture.' }, usage: { prompt_tokens: 10, completion_tokens: 2 } };
|
|
};
|
|
runtime.start(f.session.id, goal); await runtime.active.get(f.session.id).promise;
|
|
assert.equal(requests, 2); assert.deepEqual(f.store.core.task.get(f.session.taskId).contract, original);
|
|
});
|
|
|
|
test('only the routine context avoids plan echoes; diagnostic API and requests without an included plan keep original observations', t => {
|
|
const f = fixture(t), output = JSON.stringify({ plan_id: 'owned-plan', criterion: '完整原条件🙂', steps: Array(10).fill('Long original plan step') });
|
|
f.session.events.push({ kind: 'tool', tool: 'plan_task', phase: 'completed', input: { expected_revision: 0 }, output });
|
|
const snapshot = sessionSnapshot(f.session, f.project, f.store.config), original = structuredClone(f.session);
|
|
assert.equal(snapshot.recentTools[0].outputExcerpt, output);
|
|
assert.ok(sessionContext(f.session, f.project, f.store.config).includes('owned-plan'));
|
|
const projected = sessionContext(f.session, f.project, f.store.config, String, { taskPlanIncluded: true });
|
|
assert.ok(!projected.includes('owned-plan')); assert.match(projected, /原工具结果仍保存在所属历史/);
|
|
assert.deepEqual(f.session, original);
|
|
const checks = taskPlanToolDefinitions[0].function.parameters.properties.steps.items.properties.checks.description;
|
|
assert.match(checks, /当前源码的最新通过结果/); assert.match(checks, /基线失败/);
|
|
});
|