Files
Agent-Workbench/tests/task-plans.test.mjs

245 lines
22 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { randomUUID } from 'node:crypto';
import { spawnSync } from 'node:child_process';
import { fixture, call, assertToolPairs } from './helpers.mjs';
import { readingStep, checksStep, beginPlanTask, completeRead } from './fixtures/task-plan-fixture.mjs';
import { createDeliveryProject, applyDeliveryFix, deliveryGit } from './fixtures/delivery-project.mjs';
import { createBrowserProject, browserSteps } from './fixtures/browser-project.mjs';
import { taskPlanInput, PLAN_READ_TOOLS } from '../src/shared/contracts/task-plans.mjs';
import { taskPlanToolDefinitions } from '../src/shared/contracts/task-plan-tools.mjs';
import { TaskPlanManager } from '../src/core/tasks/task-plan-manager.mjs';
import { Store } from '../src/server/store.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { compactToolFeedback } from '../src/core/context/tool-feedback.mjs';
test('published plan_task analysis tools match the executable read-only contract', () => {
const step = taskPlanToolDefinitions.find(tool => tool.function.name === 'plan_task').function.parameters.properties.steps.items;
assert.deepEqual(step.properties.tools.items.enum, PLAN_READ_TOOLS);
assert.ok(!step.properties.tools.items.enum.includes('run_command'));
assert.throws(() => taskPlanInput({ expected_revision: 0, steps: [{ key: 'read', title: 'Inspect', criterion: 'Read evidence', kind: 'analysis', tools: ['run_command'] }] }), /只读工具/);
});
test('a real escaped long-file tool message keeps its proof identity within a tight request budget', async t => {
const f = fixture(t); beginPlanTask(f); await new TaskPlanManager(f.store).plan(f.session, { expected_revision: 0, steps: [readingStep()] });
fs.writeFileSync(path.join(f.projectRoot, 'long.mjs'), '"中文😀\\\\\n'.repeat(600));
const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
const invocation = call('bounded-proof-read', 'read_file', { path: 'long.mjs' });
f.session.providerMessages.push({ role: 'assistant', content: '', tool_calls: [invocation] });
const record = f.store.core.run.get(f.session.runId);
const result = await runtime.executeTool(f.session, invocation, { id: record.id, taskId: record.taskId, record, project: f.project, config: f.store.config, readMessageBytes: 1024 });
assert.ok(result.tool_invocation_id); assert.ok(result.budget_limited); assert.ok(result.returned_characters > 0);
assert.ok(Buffer.byteLength(JSON.stringify({ role: 'tool', tool_call_id: invocation.id, content: JSON.stringify(result) }), 'utf8') <= 1024);
assert.equal(compactToolFeedback(JSON.stringify(result)).tool_invocation_id, result.tool_invocation_id);
const proof = await new TaskPlanManager(f.store).verify(f.session, { plan_id: f.store.core.task.get(f.session.taskId).currentTaskPlanId,
expected_revision: 1, step_key: 'read', evidence_id: result.tool_invocation_id });
assert.equal(proof.status, 'completed');
});
import { CheckRunner } from '../src/core/check-runner.mjs';
import { BrowserManager } from '../src/core/browser/browser-manager.mjs';
import { ChangeSetManager } from '../src/core/workspaces/change-sets.mjs';
import { proposeFile } from '../src/server/files.mjs';
import { GitDelivery } from '../src/core/git-delivery.mjs';
import { DeliveryReporter } from '../src/core/delivery-report.mjs';
const save = (manager, session, steps, expected_revision = 0) => manager.plan(session, { steps, expected_revision });
const verify = (manager, session, plan, step_key, evidence_id) => manager.verify(session, { plan_id: plan.id, expected_revision: plan.revision, step_key, evidence_id });
const corrupt = (store, id) => fs.appendFileSync(path.join(store.artifactStore.root, store.core.artifact.get(id).relative), 'corruption');
const pause = f => { f.session.status = 'paused'; f.store.save(f.session); };
test('task plans reject fabricated completion, invalid scopes and cyclic or missing dependencies', () => {
assert.throws(() => taskPlanInput({ expected_revision: 0, steps: [{ ...readingStep(), status: 'verified' }] }), /有效证据类别/);
assert.throws(() => taskPlanInput({ expected_revision: 0, steps: [readingStep('a', ['b']), readingStep('b', ['a'])] }), /循环/);
assert.throws(() => taskPlanInput({ expected_revision: 0, steps: [readingStep('a', ['unknown'])] }), /必须存在/);
assert.throws(() => taskPlanInput({ expected_revision: 0, steps: [{ ...readingStep(), tools: ['run_command'] }] }), /只读/);
assert.throws(() => taskPlanInput({ expected_revision: 0, steps: [{ key: 'write', title: '修改', criterion: '修改已审查', kind: 'changes', paths: ['../other.mjs'] }] }), /相对文件/);
});
test('sealed plans retain goal, contract hash and original steps through SQLite reopen without executing tools', async t => {
const f = fixture(t); beginPlanTask(f); const manager = new TaskPlanManager(f.store), steps = [readingStep()];
const plan = await save(manager, f.session, steps), artifact = f.store.core.artifact.get(plan.contractArtifactId);
assert.equal(artifact.sha256, plan.contractSha256); assert.equal(f.store.core.tool.size, 0);
assert.equal((await save(manager, f.session, steps)).id, plan.id);
await assert.rejects(save(manager, f.session, [{ ...steps[0], criterion: '弱化条件' }], 1), /不能删改/);
await assert.rejects(save(manager, f.session, [...steps, readingStep('later')], 0), /视图已过期/);
f.store.close(); const restored = new Store(f.data); f.onCleanup(() => restored.close());
const view = await new TaskPlanManager(restored).inspect(restored.session(f.session.id));
assert.equal(view.plan_id, plan.id); assert.equal(view.revision, 1); assert.equal(view.steps[0].status, 'ready');
assert.equal(restored.core.tool.size, 0); assert.equal(restored.core.task.get(f.session.taskId).goal, '检查工程并按真实证据交付');
});
test('proofs require original owned reads, honor dependencies and reject altered invocation arguments and bytes', async t => {
const f = fixture(t); beginPlanTask(f); const manager = new TaskPlanManager(f.store), record = completeRead(f);
const plan = await save(manager, f.session, [readingStep('first'), readingStep('next', ['first'])]);
assert.equal((await verify(manager, f.session, plan, 'next', record.id)).status, 'error');
assert.equal((await manager.inspect(f.session)).steps[1].status, 'blocked');
assert.equal((await verify(manager, f.session, plan, 'first', record.id)).status, 'completed');
const again = await verify(manager, f.session, plan, 'first', record.id); assert.equal(again.reused, true);
assert.equal((await verify(manager, f.session, plan, 'next', record.id)).status, 'completed');
assert.equal((await manager.inspect(f.session)).all_verified, true); assert.equal(f.store.core.tool.size, 1);
const argumentsBefore = f.session.providerMessages[0].tool_calls[0].function.arguments;
f.session.providerMessages[0].tool_calls[0].function.arguments = '{"path":"different-file"}';
assert.equal((await manager.inspect(f.session)).all_verified, false);
f.session.providerMessages[0].tool_calls[0].function.arguments = argumentsBefore;
corrupt(f.store, record.resultArtifactId);
assert.equal((await manager.inspect(f.session)).steps[0].status, 'unverified');
assert.equal((await verify(manager, f.session, plan, 'first', record.id)).status, 'error');
assert.equal(f.store.core.tool.size, 1);
});
test('new user requirements invalidate current proofs and reseal the same original operations in a new revision', async t => {
const f = fixture(t); beginPlanTask(f); const manager = new TaskPlanManager(f.store), record = completeRead(f), original = await save(manager, f.session, [readingStep()]);
await verify(manager, f.session, original, 'read', record.id);
const run = f.store.beginRun(f.session, '追加约束:保留原文件格式', true, f.store.config, { instructionSource: 'user' }); pause(f);
assert.equal((await manager.inspect(f.session)).status, 'needs_update');
await assert.rejects(verify(manager, f.session, original, 'read', record.id), /目标已变化/);
const next = await save(manager, f.session, [readingStep()], 1); assert.equal(next.parentPlanId, original.id); assert.equal(next.runId, run.id);
assert.equal((await manager.inspect(f.session)).all_verified, false);
await verify(manager, f.session, next, 'read', record.id);
assert.equal((await manager.inspect(f.session)).all_verified, true); assert.equal(f.store.core.tool.size, 1);
assert.equal(f.store.core.task_plan.size, 2); assert.equal(f.store.core.task_proof.size, 2);
});
test('applied change-set evidence binds reviewed current files and invalidates dependent steps after outside edits', async t => {
const f = fixture(t); beginPlanTask(f); const manager = new TaskPlanManager(f.store), batches = new ChangeSetManager(f.store);
f.onCleanup(() => batches.stopAll());
const proposed = proposeFile(f.store, f.session, { path: 'src/result.mjs', content: 'export const result = 42;\n', reason: 'Reviewed plan fixture' });
const set = await batches.prepare(f.session, [proposed.change_id]);
const plan = await save(manager, f.session, [{ key: 'change', title: '完成源文件', criterion: '审查后应用实际源码', kind: 'changes', paths: ['src/result.mjs'], depends_on: [] }, readingStep('read', ['change'])]);
assert.equal((await verify(manager, f.session, plan, 'change', set.id)).status, 'error');
await (await batches.start(f.session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed');
assert.equal((await verify(manager, f.session, plan, 'change', set.id)).status, 'completed');
await verify(manager, f.session, plan, 'read', completeRead(f).id); assert.equal((await manager.inspect(f.session)).all_verified, true);
fs.appendFileSync(path.join(f.projectRoot, 'src/result.mjs'), '// user edit\n');
const changed = await manager.inspect(f.session); assert.equal(changed.steps[0].status, 'stale'); assert.equal(changed.steps[1].status, 'blocked');
assert.equal(set.status, 'completed', 'Read-only plan inspection must not rewrite the original applied result.');
});
test('actual npm checks are verified by original counted logs and current source, with failure history retained', async t => {
const f = fixture(t); createDeliveryProject(f); pause(f);
const manager = new TaskPlanManager(f.store), runner = new CheckRunner(f.store); f.onCleanup(() => runner.stopAll());
const plan = await save(manager, f.session, [checksStep()]);
const failedPlan = await runner.plan(f.session), failed = await (await runner.start(f.session, failedPlan.id)).promise;
assert.equal(failed.status, 'failed'); assert.equal((await verify(manager, f.session, plan, 'check', failed.id)).status, 'error');
applyDeliveryFix(f);
const passedPlan = await runner.plan(f.session), passed = await (await runner.start(f.session, passedPlan.id)).promise;
assert.equal(passed.status, 'passed'); assert.equal((await verify(manager, f.session, plan, 'check', passed.id)).status, 'completed');
assert.equal((await manager.inspect(f.session)).all_verified, true); assert.equal(passed.results[0].tests.total, 3);
passed.results[0].tests.total++; assert.equal((await manager.inspect(f.session)).all_verified, false); passed.results[0].tests.total--;
corrupt(f.store, passed.results[0].logArtifactId); assert.equal((await manager.inspect(f.session)).all_verified, false);
assert.equal(failed.status, 'failed'); assert.equal(f.store.core.task_proof.size, 2);
});
test('real desktop and mobile browser proofs bind required actions and PNGs, rejecting source changes, a later failed plan and changed builds', async t => {
const f = createBrowserProject(fixture(t)), manager = new TaskPlanManager(f.store), browser = new BrowserManager(f.store);
f.onCleanup(() => browser.stopAll());
const plan = await save(manager, f.session, [1280, 390].map((width, index) => ({ key: 'page' + index, title: width + 'px页面', criterion: '实际交互与可见断言通过', kind: 'browser', viewport_width: width, actions: ['fill', 'select', 'click'], depends_on: [] })));
for (const [index, width] of [1280, 390].entries()) {
const page = await browser.plan(f.session, { steps: browserSteps, viewport: { width, height: 844 } }), job = await browser.start(f.session, page.id).promise;
assert.equal(job.status, 'passed', job.summary); assert.equal((await verify(manager, f.session, plan, 'page' + index, job.id)).status, 'completed');
}
assert.equal((await manager.inspect(f.session)).all_verified, true);
const sourcePath = path.join(f.projectRoot, 'src/app.js'), sourceBefore = fs.readFileSync(sourcePath);
const builtBefore = fs.readFileSync(path.join(f.projectRoot, 'dist/app.js'));
const originalJobs = f.store.core.browser_job.size;
fs.appendFileSync(sourcePath, '\n// source changed without rebuilding\n');
const changedSource = await manager.inspect(f.session);
assert.equal(changedSource.all_verified, false);
assert.deepEqual(changedSource.steps.map(step => step.status), ['stale', 'stale']);
const originalProof = f.store.core.task_proof.get(changedSource.steps[0].proof_id);
assert.equal((await verify(manager, f.session, plan, 'page0', originalProof.evidenceId)).status, 'error');
assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, 'dist/app.js')), builtBefore);
assert.equal(f.store.core.browser_job.size, originalJobs, 'Inspecting and rejecting old evidence must not replay a browser.');
fs.writeFileSync(sourcePath, sourceBefore);
assert.equal((await manager.inspect(f.session)).all_verified, true);
const failingPlan = await browser.plan(f.session, { steps: [{ action: 'assert_count', selector: 'button', count: 9 }], viewport: { width: 390, height: 844 } });
assert.equal((await browser.start(f.session, failingPlan.id).promise).status, 'failed');
assert.equal((await manager.inspect(f.session)).steps[1].status, 'stale');
fs.appendFileSync(path.join(f.projectRoot, 'dist/app.js'), '// changed output\n');
assert.equal((await manager.inspect(f.session)).steps[0].status, 'stale');
});
test('Git delivery blocks incomplete plans and changed preview bindings, preserving HEAD and owner index before a fresh verified commit', async t => {
const f = fixture(t); createDeliveryProject(f); applyDeliveryFix(f); pause(f);
const manager = new TaskPlanManager(f.store), runner = new CheckRunner(f.store), delivery = new GitDelivery(f.store);
f.onCleanup(() => runner.stopAll()); f.onCleanup(() => delivery.stopAll());
const checkPlan = await runner.plan(f.session), check = await (await runner.start(f.session, checkPlan.id)).promise; assert.equal(check.status, 'passed');
const plan = await save(manager, f.session, [checksStep()]);
await assert.rejects(delivery.plan(f.session), /任务计划/);
const pending = await new DeliveryReporter(f.store).create(f.session); assert.equal(pending.report.status, 'needs_plan_verification');
await verify(manager, f.session, plan, 'check', check.id); const preview = await delivery.plan(f.session);
const head = deliveryGit(f, ['rev-parse', 'HEAD']).trim(), indexPath = path.join(deliveryGit(f, ['rev-parse', '--absolute-git-dir']).trim(), 'index'), originalIndex = fs.readFileSync(indexPath);
const next = await save(manager, f.session, [checksStep(), readingStep('read', ['check'])], 1);
const rejected = await (await delivery.start(f.session, preview.id)).promise;
assert.equal(rejected.status, 'stale'); assert.equal(deliveryGit(f, ['rev-parse', 'HEAD']).trim(), head); assert.deepEqual(fs.readFileSync(indexPath), originalIndex);
await verify(manager, f.session, next, 'check', check.id); await verify(manager, f.session, next, 'read', completeRead(f).id);
const fresh = await delivery.plan(f.session); assert.notEqual(fresh.taskPlanBindingSha256, preview.taskPlanBindingSha256);
const delivered = await (await delivery.start(f.session, fresh.id)).promise; assert.equal(delivered.status, 'completed', delivered.summary);
const report = await new DeliveryReporter(f.store).create(f.session); assert.equal(report.report.taskPlan.all_verified, true); assert.equal(report.report.status, 'verified_committed');
assert.equal(report.report.taskPlanHistory.length, 2); assert.equal(report.report.taskProofs.length, 3);
assert.match(f.store.artifactStore.read(f.store.core.artifact.get(report.markdownArtifactId)), /执行计划与实际核验/);
});
test('the runtime supplies owned read IDs and completes a plan only after original step evidence verification', async t => {
const f = fixture(t), provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let phase = 0, planId, readId;
provider.complete = async (_config, messages, _project, _signal, _progress, _requestPlan, tools) => {
assert.ok(tools.some(tool => tool.function.name === 'verify_task_step'));
const last = messages.filter(message => message.role === 'tool').at(-1), result = last ? JSON.parse(last.content) : null;
let calls;
if (phase === 0) calls = [call('runtime-plan', 'plan_task', { expected_revision: 0, steps: [readingStep()] })];
if (phase === 1) { planId = result.plan_id; calls = [call('runtime-read', 'read_file', { path: 'README.md' })]; }
if (phase === 2) { readId = result.tool_invocation_id; assert.ok(readId); calls = [call('runtime-proof', 'verify_task_step', { plan_id: planId, expected_revision: 1, step_key: 'read', evidence_id: readId })]; }
phase++; return { message: { role: 'assistant', content: calls ? '' : '原步骤核验完成。', ...(calls ? { tool_calls: calls } : {}) }, usage: { prompt_tokens: 120, completion_tokens: 20 } };
};
runtime.start(f.session.id, '按实际读取核验一项步骤'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'completed'); assert.equal((await runtime.taskPlanner().inspect(f.session)).all_verified, true);
assert.equal(f.store.core.tool.get(readId).tool, 'read_file'); assert.equal(phase, 4); assertToolPairs(f.session.providerMessages);
});
test('premature model completion pauses the original Task and resume preserves the existing plan without replay', async t => {
const f = fixture(t), provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let phase = 0, planId;
provider.complete = async (_config, messages) => {
const result = JSON.parse(messages.filter(message => message.role === 'tool').at(-1)?.content ?? '{}'); let calls;
if (phase === 0) calls = [call('early-plan', 'plan_task', { expected_revision: 0, steps: [readingStep()] })];
if (phase === 1) planId = result.plan_id;
if (phase === 2) calls = [call('resumed-read', 'read_file', { path: 'README.md' })];
if (phase === 3) calls = [call('resumed-proof', 'verify_task_step', { plan_id: planId, expected_revision: 1, step_key: 'read', evidence_id: result.tool_invocation_id })];
phase++; return { message: { role: 'assistant', content: calls ? '' : '模型自述结束。', ...(calls ? { tool_calls: calls } : {}) }, usage: { prompt_tokens: 120, completion_tokens: 20 } };
};
runtime.start(f.session.id, '核验读取后再完成任务'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'task_plan_incomplete');
const taskId = f.session.taskId, prefix = structuredClone(f.session.providerMessages);
runtime.start(f.session.id, '', true); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.taskId, taskId); assert.equal(f.session.status, 'completed'); assert.equal(f.store.core.task_plan.size, 1);
assert.deepEqual(f.session.providerMessages.slice(0, prefix.length), prefix); assertToolPairs(f.session.providerMessages);
});
test('plan operations participate in project activity and orderly shutdown', async t => {
const f = fixture(t); beginPlanTask(f); const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
let release; const gate = new Promise(resolve => { release = resolve; });
const pending = runtime.taskPlanner().mutate(f.session, 'user', () => gate);
assert.equal(runtime.isActive(f.session.id), true); assert.equal(runtime.projectHasActivity(f.project.id), true); assert.equal(runtime.hasActivity(), true);
assert.throws(() => runtime.start(f.session.id, '不能在核验时覆盖任务'), /已有任务/);
const stopped = runtime.stopAll(); release(); await pending; await stopped; assert.equal(runtime.hasActivity(), false);
});
for (const operation of ['plan', 'proof']) for (const point of ['before', 'after']) test(`actual process exit ${point} ${operation} save restores only committed plan evidence without replay`, async t => {
const f = fixture(t); beginPlanTask(f); const evidence = completeRead(f), manager = new TaskPlanManager(f.store);
const plan = operation === 'proof' ? await save(manager, f.session, [readingStep()]) : null;
const inputFile = path.join(f.root, 'crash-' + randomUUID() + '.json');
fs.writeFileSync(inputFile, JSON.stringify({ data: f.data, sessionId: f.session.id, operation, point, planId: plan?.id, evidenceId: evidence.id })); f.store.close();
const exited = spawnSync(process.execPath, ['tests/fixtures/task-plan-crash-worker.mjs', inputFile], { windowsHide: true, timeout: 10000, encoding: 'utf8' });
assert.equal(exited.status, 71, exited.stderr + exited.stdout);
const restored = new Store(f.data); f.onCleanup(() => restored.close());
const current = restored.session(f.session.id), view = await new TaskPlanManager(restored).inspect(current);
assert.equal(restored.core.tool.size, 1); assert.equal(restored.core.tool.get(evidence.id).state, 'completed');
assert.equal(view.configured, operation === 'proof' || point === 'after');
assert.equal(view.all_verified, operation === 'proof' && point === 'after');
assertToolPairs(current.providerMessages);
});