203 lines
15 KiB
JavaScript
203 lines
15 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { applyChange } from '../src/server/files.mjs';
|
|
import { isDiagnosticQuestion, taskContract, updateTaskContract, taskToolAllowed } from '../src/shared/contracts/task-control.mjs';
|
|
import { fixture, call, assertToolPairs } from './helpers.mjs';
|
|
|
|
const answer = content => ({ message: { role: 'assistant', content }, usage: { prompt_tokens: 11, completion_tokens: 3 } });
|
|
const calls = (...tools) => ({ ...answer(''), message: { role: 'assistant', content: '', tool_calls: tools } });
|
|
test('operational questions are diagnostic while explicit repair requests remain execution instructions', () => {
|
|
for (const text of ['前面的错误是怎么回事?', '这个报错是什么意思', '解释下这个错误', '请解释刚才执行的错误', '你使用的是什么模型?', 'Why did your fix fail?']) assert.equal(isDiagnosticQuestion(text), true, text);
|
|
for (const text of ['先修复系统', '为什么失败,请帮我修复', '这个错误怎么回事,然后修一下', 'fix this error', '继续完成']) assert.equal(isDiagnosticQuestion(text), false, text);
|
|
});
|
|
|
|
test('an explicit prohibition on running engineering checks is not interpreted as a requirement to run them', () => {
|
|
for (const goal of ['只读核对指定文件范围,不修改文件,不运行工程检查。', '不执行工程检查,只解释原结果',
|
|
'不需要做任何测试', '无需工程检查', 'Read the evidence; do not run checks.', "Don't run tests; explain the original failure.",
|
|
'Inspect the directory once and answer only. Do not modify files or run checks.']) {
|
|
assert.equal(taskContract({goal}).checks, 'skip', goal);
|
|
const task = { goal: 'Run checks.', instructions: [] };
|
|
updateTaskContract(task, goal); assert.equal(task.contract.checks, 'skip', goal);
|
|
assert.equal(taskToolAllowed(task, { purpose: 'execution' }, 'run_checks'), false);
|
|
updateTaskContract(task, 'Now run checks.'); assert.equal(task.contract.checks, 'required');
|
|
}
|
|
for (const goal of ['修复并运行工程检查', '运行并通过原测试', 'Fix the source and run checks.']) assert.equal(taskContract({goal}).checks, 'required', goal);
|
|
});
|
|
|
|
test('file delivery requires a positive instruction, not saved evidence or a prohibited write', () => {
|
|
for (const goal of ['读取保存的原工件,分段取得真实依据,不执行命令或修改文件。',
|
|
'读取保存过的 HTML 文件并解释内容', '不要创建文件,只分析结果', '读取工件,不执行命令或创建文件',
|
|
'Do not create a file; explain the evidence.', 'Read the saved file without creating HTML.',
|
|
'不需要做任何修改 HTML 文件,只核对结果', '解释如何修改 HTML 文件', 'Explain how to modify an HTML file.']) {
|
|
assert.equal(taskContract({ goal }).kind, 'general', goal);
|
|
}
|
|
for (const goal of ['创建一个 HTML 文件,无需测试', '新建 result.txt', '把结果保存为 report.md 文件',
|
|
'请写入 SVG 文件', 'Create an HTML file without running tests.', '不要执行命令,创建一个 HTML 文件',
|
|
'文件不存在时创建 report.md 文件', '不要修改原有文件而是新建 HTML 文件']) {
|
|
assert.equal(taskContract({ goal }).kind, 'file', goal);
|
|
}
|
|
});
|
|
|
|
test('an unsubstantiated text claim cannot complete a requested file delivery', async t => {
|
|
const f = harness(t, () => answer('HTML 已创建完成。'));
|
|
f.runtime.start(f.session.id, '创建一个 HTML 文件,无需测试'); await f.wait();
|
|
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'delivery_incomplete');
|
|
assert.equal(f.session.changes.length, 0); assert.equal(f.session.approvals.length, 0);
|
|
});
|
|
function harness(t, respond, options = {}) {
|
|
const f = fixture(t, options), provider = new Provider(); let round = 0;
|
|
provider.complete = async (config, messages, project, signal, progress, plan, tools) => respond(round++, { messages, tools, f });
|
|
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
return { ...f, runtime, wait: () => runtime.active.get(f.session.id).promise };
|
|
}
|
|
|
|
test('HTML without tests ends at reviewed file delivery and never offers automatic rechecking', async t => {
|
|
const f = harness(t, (round, { tools }) => {
|
|
assert.ok(!tools.some(tool => ['run_checks', 'plan_checks'].includes(tool.function.name)));
|
|
return round === 0 ? calls(call('html', 'propose_file', { path: 'bike.html', content: '<!doctype html><svg><circle r="4"/></svg>', reason: '原请求' })) : answer('文件提案已生成。');
|
|
});
|
|
f.runtime.start(f.session.id, '创建一个 HTML,使用 SVG 动画,无需测试。'); await f.wait();
|
|
const task = f.store.core.task.get(f.session.taskId), id = task.id;
|
|
assert.equal(f.session.status, 'needs_review');
|
|
applyChange(f.store, f.session, f.session.changes[0].id);
|
|
assert.equal(f.session.status, 'completed'); assert.equal(task.status, 'completed');
|
|
assert.equal(task.delivery.checks, 'not_requested'); assert.deepEqual(task.delivery.paths, ['bike.html']);
|
|
assert.match(fs.readFileSync(path.join(f.projectRoot, 'bike.html'), 'utf8'), /<svg>/);
|
|
assert.throws(() => f.runtime.continueReviewed(f.session.id), { status: 409 });
|
|
assert.equal(f.store.core.check_job.size, 0); assert.equal(f.session.approvals.length, 0);
|
|
f.runtime.start(f.session.id, '继续完成'); await f.wait();
|
|
assert.equal(f.session.taskId, id); assert.match(task.contract.goal, /SVG 动画/);
|
|
assertToolPairs(f.session.providerMessages);
|
|
});
|
|
|
|
test('diagnostic answers retain the unfinished task, limits and check result, and enforce read-only tools', async t => {
|
|
const f = harness(t, (round, { tools }) => {
|
|
if (round === 0) return answer('尚未交付。');
|
|
if (round === 1) {
|
|
assert.ok(tools.every(tool => !['run_command', 'propose_file', 'run_checks'].includes(tool.function.name)));
|
|
return calls(call('bad-diagnostic-write', 'propose_file', { path: 'unwanted.txt', content: 'wrong', reason: '诊断不应写入' }), call('inspect-error', 'inspect_session', {}));
|
|
}
|
|
return answer('所属任务的检查失败,原任务仍需修复。');
|
|
});
|
|
f.runtime.start(f.session.id, '修复工程并通过测试'); await f.wait();
|
|
const task = f.store.core.task.get(f.session.taskId), taskId = task.id;
|
|
task.verification = { status: 'failed', checkJobId: null }; f.session.status = 'paused'; f.store.save(f.session);
|
|
const instructions = structuredClone(task.instructions);
|
|
const mainHistory = structuredClone(f.session.providerMessages);
|
|
f.runtime.start(f.session.id, '前面的错误是怎么回事?'); await f.wait();
|
|
assert.equal(f.session.taskId, taskId); assert.equal(task.status, 'paused'); assert.equal(f.session.status, 'paused');
|
|
assert.deepEqual(task.instructions, instructions); assert.equal(task.verification.status, 'failed');
|
|
const run = f.store.core.run.get(f.session.runId);
|
|
assert.equal(run.purpose, 'diagnostic'); assert.equal(run.status, 'completed'); assert.equal(run.stopReason, 'diagnostic_answered');
|
|
assert.equal(f.session.changes.length, 0); assert.ok(!fs.existsSync(path.join(f.projectRoot, 'unwanted.txt')));
|
|
assert.deepEqual(f.session.providerMessages, mainHistory, 'Diagnostic exchanges stay outside the main model history.');
|
|
const denied = JSON.parse(run.diagnosticMessages.find(message => message.tool_call_id === 'bad-diagnostic-write').content);
|
|
assert.equal(denied.status, 'error'); assert.match(denied.message, /只读/);
|
|
const inspected = JSON.parse(run.diagnosticMessages.find(message => message.tool_call_id === 'inspect-error').content);
|
|
assert.equal(inspected.diagnosticContext.evidenceVerified, true); assert.equal(inspected.diagnosticContext.taskId, taskId);
|
|
f.runtime.start(f.session.id, '继续完成'); await f.wait();
|
|
assert.equal(f.session.taskId, taskId); assert.equal(f.session.status, 'paused');
|
|
assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'verification_incomplete');
|
|
});
|
|
|
|
test('open full-access requests advertise task intent while diagnostic history and the existing execution contract stay intact', async t => {
|
|
const f = harness(t, (round, { tools }) => {
|
|
if (round === 0) {
|
|
assert.ok(!tools.some(tool => ['plan_checks', 'run_checks'].includes(tool.function.name)));
|
|
assert.ok(tools.some(tool => tool.function.name === 'run_command'));
|
|
return answer('尚未交付。');
|
|
}
|
|
if (round === 1) {
|
|
assert.ok(tools.some(tool => tool.function.name === 'read_file'));
|
|
assert.ok(!tools.some(tool => ['run_command', 'propose_file', 'prepare_change_set', 'run_checks', 'plan_delivery'].includes(tool.function.name)));
|
|
return calls(call('open-diagnostic-write', 'propose_file', { path: 'unwanted.txt', content: 'wrong', reason: 'diagnostic must retain the original task' }), call('open-inspect-error', 'inspect_session', {}));
|
|
}
|
|
return answer('所属原记录已核对,未修改文件或重跑检查。');
|
|
}, { modernDefaults: true });
|
|
f.store.saveConfig({ approvalMode: 'full' });
|
|
assert.equal(f.store.config.executionMode, 'unrestricted'); assert.equal(f.store.config.capacityMode, 'passthrough'); assert.equal(f.store.config.historyPolicy, 'preserve');
|
|
f.runtime.start(f.session.id, '读取README.md并解释原记录,无需测试。'); await f.wait();
|
|
const task = f.store.core.task.get(f.session.taskId), taskId = task.id;
|
|
task.verification = { status: 'failed', checkJobId: null }; f.session.status = 'paused'; f.store.save(f.session);
|
|
const instructions = structuredClone(task.instructions), budget = structuredClone(task.budget), history = structuredClone(f.session.providerMessages), original = fs.readFileSync(path.join(f.projectRoot, 'README.md'));
|
|
f.runtime.start(f.session.id, '前面的错误是怎么回事?'); await f.wait();
|
|
const run = f.store.core.run.get(f.session.runId);
|
|
assert.equal(run.purpose, 'diagnostic'); assert.equal(run.status, 'completed'); assert.equal(run.stopReason, 'diagnostic_answered');
|
|
assert.equal(run.budget.maxTokens, 0); assert.equal(run.budget.maxDurationMs, 0);
|
|
assert.equal(f.session.taskId, taskId); assert.equal(task.status, 'paused'); assert.equal(f.session.status, 'paused');
|
|
assert.deepEqual(task.instructions, instructions); assert.deepEqual(task.budget, budget); assert.equal(task.verification.status, 'failed');
|
|
assert.deepEqual(f.session.providerMessages, history); assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, 'README.md')), original);
|
|
assert.equal(f.session.changes.length, 0); assert.equal(f.session.approvals.length, 0); assert.equal(fs.existsSync(path.join(f.projectRoot, 'unwanted.txt')), false);
|
|
const denied = JSON.parse(run.diagnosticMessages.find(message => message.tool_call_id === 'open-diagnostic-write').content);
|
|
assert.equal(denied.status, 'error'); assert.match(denied.message, /只读/);
|
|
const inspected = JSON.parse(run.diagnosticMessages.find(message => message.tool_call_id === 'open-inspect-error').content);
|
|
assert.equal(inspected.diagnosticContext.evidenceVerified, true); assert.equal(inspected.diagnosticContext.taskId, taskId); assertToolPairs(run.diagnosticMessages);
|
|
});
|
|
|
|
test('review continuation is a system action and adds no testing demand or fake user instruction', async t => {
|
|
let continued;
|
|
const f = harness(t, (round, { messages }) => {
|
|
if (round === 0) return calls(call('doc', 'propose_file', { path: 'result.txt', content: 'done', reason: 'requested' }));
|
|
if (round === 2) continued = messages;
|
|
return answer('改动已生成。');
|
|
});
|
|
f.runtime.start(f.session.id, '创建 result.txt'); await f.wait();
|
|
applyChange(f.store, f.session, f.session.changes[0].id);
|
|
const task = f.store.core.task.get(f.session.taskId), users = f.session.messages.filter(message => message.role === 'user').length;
|
|
f.runtime.continueReviewed(f.session.id); await f.wait();
|
|
assert.equal(task.instructions.length, 0);
|
|
assert.equal(f.session.messages.filter(message => message.role === 'user').length, users);
|
|
assert.equal(f.store.core.run.get(f.session.runId).source, 'review-action');
|
|
assert.match(JSON.stringify(continued), /创建 result.txt/);
|
|
assert.doesNotMatch(JSON.stringify(continued), /识别并实际执行工程检查/);
|
|
});
|
|
|
|
test('a provider cannot execute tests against an explicit no-test task even if it ignores advertised tools', async t => {
|
|
const f = harness(t, round => round === 0 ? calls(call('blocked-check', 'plan_checks', {}), call('blocked-command', 'run_command', { command: 'npm test', reason: 'ignored constraint' })) : answer('按照原要求不测试。'));
|
|
f.runtime.start(f.session.id, '创建 HTML,无需测试'); await f.wait();
|
|
assert.equal(f.session.approvals.length, 0); assert.equal(f.store.core.check_plan.size, 0);
|
|
for (const id of ['blocked-check', 'blocked-command']) assert.match(JSON.parse(f.session.providerMessages.find(message => message.tool_call_id === id).content).message, /无需测试/);
|
|
assertToolPairs(f.session.providerMessages);
|
|
});
|
|
|
|
test('a budget pause becomes visible only after its input snapshot can be sealed with the original Run', async t => {
|
|
const f = harness(t, () => calls(call('one-read', 'read_file', { path: 'README.md' })));
|
|
f.store.config.maxSteps = 1;
|
|
let entered, release;
|
|
const capturing = new Promise(resolve => { entered = resolve; });
|
|
const held = new Promise(resolve => { release = resolve; });
|
|
const capture = f.runtime.capturePauseInputs.bind(f.runtime);
|
|
f.runtime.capturePauseInputs = async run => { entered(); await held; return capture(run); };
|
|
t.after(() => release());
|
|
f.runtime.start(f.session.id, '读取原文件');
|
|
await capturing;
|
|
const run = f.store.core.run.get(f.session.runId);
|
|
assert.equal(f.session.status, 'running'); assert.equal(run.finishedAt, null);
|
|
assert.equal(f.runtime.isActive(f.session.id), true);
|
|
assert.throws(() => f.runtime.resume(f.session.id), { status: 409 });
|
|
release(); await f.wait();
|
|
assert.equal(f.session.status, 'paused'); assert.equal(run.stopReason, 'step_limit');
|
|
assert.ok(run.finishedAt); assert.ok(run.finalSourceFingerprint);
|
|
assert.equal(f.runtime.isActive(f.session.id), false);
|
|
});
|
|
|
|
test('alternating duplicate file reads and listing files still pauses without treating new pages as duplicates', async t => {
|
|
const f = harness(t, round => calls(call('duplicate-read-' + round, 'read_file', { path: round % 2 ? 'README.md' : 'second.txt' }), call('list-between-' + round, 'list_files', { path: '' })));
|
|
fs.writeFileSync(path.join(f.projectRoot, 'second.txt'), 'another original source\n');
|
|
f.runtime.start(f.session.id, '查看两个文件'); await f.wait();
|
|
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'no_progress');
|
|
assert.equal(f.store.core.run.get(f.session.runId).steps, 5); assertToolPairs(f.session.providerMessages);
|
|
f.runtime.resume(f.session.id); await f.wait();
|
|
assert.equal(f.store.core.task.get(f.session.taskId).stagnation.count, 2);
|
|
f.runtime.resume(f.session.id); await f.wait();
|
|
assert.equal(f.store.core.run.get(f.session.runId).usage.requests, 0);
|
|
assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'no_progress');
|
|
fs.appendFileSync(path.join(f.projectRoot, 'second.txt'), 'new owner input\n');
|
|
f.runtime.resume(f.session.id); await f.wait();
|
|
assert.ok(f.store.core.run.get(f.session.runId).usage.requests > 0);
|
|
});
|