Files
Agent-Workbench/tests/delivery.test.mjs
2026-10-29 22:28:47 +08:00

267 lines
22 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { fixture, launchWorkbench, until, call, assertToolPairs, compatibleFixture, completion } from './helpers.mjs';
import { createDeliveryProject, applyDeliveryFix, checkedDelivery, deliveryGit as git, writeDeliveryFile as write } from './fixtures/delivery-project.mjs';
import { CheckRunner } from '../src/core/check-runner.mjs';
import { GitDelivery } from '../src/core/git-delivery.mjs';
import { gitEnvironment } from '../src/core/git-process.mjs';
import { DeliveryReporter } from '../src/core/delivery-report.mjs';
import { inspectProject } from '../src/core/project-inspector.mjs';
import { proposeFile, applyChange } from '../src/server/files.mjs';
import { Store } from '../src/server/store.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
test('actual failed tests become a reviewed local commit and evidence report while preserving unrelated index and files', async t => {
const f = fixture(t); createDeliveryProject(f);
const parent = git(f, ['rev-parse', 'HEAD']).trim(), protectedStage = git(f, ['show', ':owner-staged.txt']);
const ownerFiles = ['owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(f.projectRoot, name))]);
const checks = new CheckRunner(f.store), delivery = new GitDelivery(f.store); f.onCleanup(() => delivery.stopAll()); f.onCleanup(() => checks.stopAll());
let started = await checks.start(f.session, (await checks.plan(f.session)).id), failed = await started.promise;
assert.equal(failed.status, 'failed'); assert.equal(failed.results[0].tests.failed, 3);
await assert.rejects(delivery.plan(f.session), /通过记录/);
applyDeliveryFix(f);
started = await checks.start(f.session, (await checks.plan(f.session)).id); const passed = await started.promise;
assert.equal(passed.status, 'passed'); assert.equal(passed.results[0].tests.passed, 3);
const before = await inspectProject(f.project), plan = await delivery.plan(f.session, { message: 'feat: deliver reviewed addition repair' });
assert.deepEqual(plan.paths, ['src/add.mjs', 'src/math.mjs']); assert.equal(plan.author.name, 'Workbench QA'); assert.equal(plan.policy.hooks, 'disabled');
assert.equal(git(f, ['rev-parse', 'HEAD']).trim(), parent); assert.match(plan.files[1].diff, /sum/);
const accepted = await delivery.start(f.session, plan.id), job = await accepted.promise;
assert.equal(job.status, 'completed', job.summary); assert.equal(job.requiresVerification, false); assert.equal(job.indexPreserved, true); assert.equal(job.workspacePreserved, true);
assert.equal(git(f, ['rev-parse', 'HEAD']).trim(), job.commit); assert.equal(git(f, ['rev-parse', 'HEAD^']).trim(), parent);
assert.deepEqual(git(f, ['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split('\n'), ['src/add.mjs', 'src/math.mjs']);
assert.equal(git(f, ['show', ':owner-staged.txt']), protectedStage);
for (const [name, bytes] of ownerFiles) assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, name)), bytes);
assert.match(git(f, ['status', '--porcelain=v1']), /MM owner-staged.txt/); assert.match(git(f, ['status', '--porcelain=v1']), /\?\? owner-untracked.txt/);
const after = await inspectProject(f.project);
assert.notEqual(before.fingerprint, after.fingerprint); assert.equal(before.sourceFingerprint, after.sourceFingerprint);
assert.equal(f.store.core.task.get(f.session.taskId).verification.status, 'passed');
const report = JSON.parse(f.store.artifactStore.read(f.store.core.artifact.get(job.reportArtifactId)));
assert.equal(report.status, 'verified_committed'); assert.equal(report.verifiedCheckJobId, passed.id); assert.equal(report.task.requirementsAcceptance, 'not_recorded');
assert.equal(report.checks[0].results[0].tests.failed, 3); assert.equal(report.checks[1].results[0].tests.passed, 3); assert.equal(report.checks[1].stale, false);
assert.equal(report.deliveries[0].commit, job.commit); assert.equal(report.usage.amount, null);
assert.match(f.store.artifactStore.read(f.store.core.artifact.get(job.reportMarkdownArtifactId)), /检查通过并已提交/);
await assert.rejects(delivery.start(f.session, plan.id), { status: 409 });
await assert.rejects(delivery.plan(f.session), /没有新的/);
f.store.close(); const recovered = new Store(f.data); f.onCleanup(() => recovered.close());
assert.equal(recovered.owned('delivery_job', job.id, f.session.id).commit, job.commit);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
});
test('selected preexisting unstaged, staged and untracked owner edits cannot enter a task commit', async t => {
for (const kind of ['unstaged', 'staged', 'untracked']) {
const f = fixture(t); createDeliveryProject(f, { broken: false });
const relative = kind === 'untracked' ? 'owner-new.mjs' : 'src/add.mjs';
write(f, relative, kind === 'untracked' ? 'export const owner = true;\n' : 'export const add = (a, b) => a + b + 0;\n');
if (kind === 'staged') git(f, ['add', '--', relative]);
const proposal = proposeFile(f.store, f.session, { path: relative, content: kind === 'untracked' ? 'export const owner = true; export const reviewed = true;\n' : 'export const add = (a, b) => a + b + 0 + 0;\n', reason: 'Task starts after an owner edit' });
applyChange(f.store, f.session, proposal.change_id);
const checks = new CheckRunner(f.store), plan = await checks.plan(f.session);
assert.equal((await (await checks.start(f.session, plan.id)).promise).status, 'passed');
const before = git(f, ['status', '--porcelain=v1']), head = git(f, ['rev-parse', 'HEAD']);
await assert.rejects(new GitDelivery(f.store).plan(f.session), /其他工作区修改|暂存修改|任务之前已经存在/);
assert.equal(git(f, ['status', '--porcelain=v1']), before); assert.equal(git(f, ['rev-parse', 'HEAD']), head);
}
});
test('source, unrelated stage and branch changes after review invalidate a commit plan without changing HEAD', async t => {
for (const kind of ['source', 'stage', 'branch']) {
const f = fixture(t), { delivery } = await checkedDelivery(f), plan = await delivery.plan(f.session);
if (kind === 'source') write(f, 'src/add.mjs', "import { sum } from './math.mjs'; export const add = sum;\n");
if (kind === 'stage') { write(f, 'owner-unstaged.txt', 'Owner staged after review\n'); git(f, ['add', '--', 'owner-unstaged.txt']); }
if (kind === 'branch') git(f, ['switch', '-c', 'owner-branch']);
const head = git(f, ['rev-parse', 'HEAD']), stage = git(f, ['ls-files', '--stage', '-z']);
const job = await (await delivery.start(f.session, plan.id)).promise;
assert.equal(job.status, 'stale', job.summary); assert.equal(job.commit, null); assert.equal(git(f, ['rev-parse', 'HEAD']), head);
assert.equal(git(f, ['ls-files', '--stage', '-z']), stage); assert.equal(fs.existsSync(path.join(f.projectRoot, '.git/index.lock')), false);
await assert.rejects(delivery.start(f.session, plan.id), { status: 409 });
}
});
test('a later Git metadata change keeps matching source evidence valid but a real source edit requires checks', async t => {
const f = fixture(t), { delivery, check } = await checkedDelivery(f), source = (await inspectProject(f.project)).sourceFingerprint;
git(f, ['switch', '-c', 'delivery-branch']);
const plan = await delivery.plan(f.session), job = await (await delivery.start(f.session, plan.id)).promise;
assert.equal(job.status, 'completed', job.summary); assert.equal(plan.snapshot.branch, 'delivery-branch');
assert.equal((await inspectProject(f.project)).sourceFingerprint, source);
const reporter = new DeliveryReporter(f.store); assert.equal((await reporter.create(f.session)).report.verifiedCheckJobId, check.id);
write(f, 'src/math.mjs', 'export const sum = (a, b) => a + b + 1;\n');
const stale = await reporter.create(f.session);
assert.equal(stale.report.status, 'needs_checks'); assert.equal(stale.report.verifiedCheckJobId, null); assert.equal(stale.report.checks[0].stale, true);
});
test('a corrupt passed-check log and a pending proposal block Git delivery instead of inventing evidence', async t => {
const f = fixture(t), { delivery, check } = await checkedDelivery(f);
const proposal = proposeFile(f.store, f.session, { path: 'review.md', content: 'Still pending\n', reason: 'Pending item' });
await assert.rejects(delivery.plan(f.session), /待审查/);
f.session.changes.find(change => change.id === proposal.change_id).status = 'rejected'; f.session.status = 'completed'; f.store.save(f.session);
const log = f.store.core.artifact.get(check.results[0].logArtifactId);
fs.writeFileSync(path.join(f.store.artifactStore.root, log.relative), 'Corrupted result');
await assert.rejects(delivery.plan(f.session), /日志完整/);
assert.equal((await new DeliveryReporter(f.store).create(f.session)).report.status, 'needs_checks');
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '1');
});
test('Git filters, merge state, detached HEAD and old task ownership are explicitly rejected', async t => {
for (const kind of ['filter', 'merge', 'detached', 'old-task']) {
const f = fixture(t), { delivery } = await checkedDelivery(f);
if (kind === 'filter') {
write(f, '.gitattributes', '*.mjs filter=owner-filter\n');
git(f, ['config', 'filter.owner-filter.clean', 'node scripts/filter-marker.mjs']);
write(f, 'scripts/filter-marker.mjs', "import fs from 'node:fs'; fs.writeFileSync('filter-was-run.txt', 'unexpected');\n");
}
if (kind === 'merge') fs.writeFileSync(path.join(f.projectRoot, '.git/MERGE_HEAD'), git(f, ['rev-parse', 'HEAD']));
if (kind === 'detached') git(f, ['switch', '--detach']);
if (kind === 'old-task') {
const run = f.store.beginRun(f.session, 'A different task with no reviewed changes', false, f.store.config);
f.session.status = 'completed'; f.store.finishRun(f.session, run);
const checks = new CheckRunner(f.store); await (await checks.start(f.session, (await checks.plan(f.session)).id)).promise;
}
await assert.rejects(delivery.plan(f.session), /过滤|合并|命名分支|归属明确/);
assert.equal(fs.existsSync(path.join(f.projectRoot, 'filter-was-run.txt')), false);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '1');
}
});
test('local snapshot commits disable repository hooks and signing and isolate the Git child environment', async t => {
const f = fixture(t), { delivery } = await checkedDelivery(f);
const hook = path.join(f.projectRoot, '.git/hooks/reference-transaction');
fs.writeFileSync(hook, '#!/bin/sh\nprintf unexpected > hook-was-run.txt\n', { mode: 0o755 });
git(f, ['config', 'commit.gpgSign', 'true']); git(f, ['config', 'user.signingKey', 'a-key-that-is-not-present']);
const previous = process.env.EXTRA_MODEL_CONNECTION_SECRET;
process.env.EXTRA_MODEL_CONNECTION_SECRET = 'synthetic-runtime-only-secret';
try {
assert.equal(gitEnvironment().EXTRA_MODEL_CONNECTION_SECRET, undefined); assert.equal(gitEnvironment().NODE_TEST_CONTEXT, undefined);
const job = await (await delivery.start(f.session, (await delivery.plan(f.session, { authorName: 'Local Reviewer', authorEmail: 'reviewer@local.invalid', message: 'fix: reviewed delivery' })).id)).promise;
assert.equal(job.status, 'completed', job.summary); assert.equal(fs.existsSync(path.join(f.projectRoot, 'hook-was-run.txt')), false);
assert.match(git(f, ['cat-file', 'commit', 'HEAD']), /author Local Reviewer <reviewer@local.invalid>/);
assert.ok(!git(f, ['cat-file', 'commit', 'HEAD']).includes('gpgsig'));
assert.ok(!fs.readFileSync(path.join(f.data, 'workbench.sqlite')).includes(Buffer.from('synthetic-runtime-only-secret')));
} finally { if (previous === undefined) delete process.env.EXTRA_MODEL_CONNECTION_SECRET; else process.env.EXTRA_MODEL_CONNECTION_SECRET = previous; }
});
test('stopping before publication cancels safely; stopping after publication preserves the actual completed commit', async t => {
for (const phase of ['prepared', 'committed']) {
const f = fixture(t); await checkedDelivery(f);
let reached = false, release;
const gate = new Promise(resolve => { release = resolve; });
const delivery = new GitDelivery(f.store, { faultInjector: async current => { if (current === phase) { reached = true; await gate; } } });
f.onCleanup(() => { release(); return delivery.stopAll(); });
const runtime = new Runtime(f.store, new Provider()); runtime.deliveries = delivery;
delivery.projectBusy = projectId => [...runtime.active.values()].some(run => run.project.id === projectId);
const plan = await delivery.plan(f.session), started = await delivery.start(f.session, plan.id);
await until(() => reached, 12000); assert.equal(runtime.hasActivity(), true); assert.equal(runtime.projectHasActivity(f.project.id), true);
await assert.rejects(delivery.start(f.session, plan.id), { status: 409 });
assert.throws(() => runtime.start(f.store.createSession(f.project.id).id, 'Concurrent work'), { status: 409 });
const stop = runtime.stop(f.session.id); release(); await stop; const job = await started.promise;
assert.equal(job.status, phase === 'prepared' ? 'cancelled' : 'completed', job.summary);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), phase === 'prepared' ? '1' : '2');
assert.equal(job.stopRequested, true); assert.equal(fs.existsSync(path.join(f.projectRoot, '.git/index.lock')), false);
assert.equal(runtime.hasActivity(), false);
}
});
test('delivery API confirms exactly one owned plan, serves verified reports and preserves results across restart', async t => {
const f = fixture(t); await checkedDelivery(f); f.store.close(); const app = await launchWorkbench(t, f.data);
const planned = await app.api('/sessions/' + f.session.id + '/delivery-plans', { message: 'API reviewed delivery' }), plan = planned.execution.deliveryPlans.at(-1);
assert.deepEqual(plan.paths, ['src/add.mjs', 'src/math.mjs']); assert.equal(plan.snapshot, undefined); assert.match(plan.baseline.head, /^[a-f0-9]{40}$/);
const other = await app.api('/sessions', { projectId: f.project.id });
await assert.rejects(app.api('/sessions/' + other.id + '/delivery-jobs', { planId: plan.id }), { status: 409 });
const responses = await Promise.allSettled([app.api('/sessions/' + f.session.id + '/delivery-jobs', { planId: plan.id }), app.api('/sessions/' + f.session.id + '/delivery-jobs', { planId: plan.id })]);
assert.equal(responses.filter(value => value.status === 'fulfilled').length, 1);
const accepted = responses.find(value => value.status === 'fulfilled').value, jobId = accepted.execution.deliveries.at(-1).id;
let job; await until(async () => { job = await app.api('/sessions/' + f.session.id + '/delivery-jobs/' + jobId); return !['queued', 'preparing', 'prepared', 'publishing', 'committed', 'reporting'].includes(job.status); }, 20000);
assert.equal(job.status, 'completed', job.summary);
await assert.rejects(app.api('/sessions/' + other.id + '/delivery-jobs/' + jobId), { status: 404 });
const preview = await app.api('/artifacts/' + job.reportArtifactId + '?session=' + f.session.id + '&limit=64000');
assert.equal(JSON.parse(preview.content).status, 'verified_committed');
await assert.rejects(app.api('/artifacts/' + job.reportArtifactId + '?session=' + other.id), { status: 404 });
const download = await fetch(app.url + '/api/artifacts/' + job.reportMarkdownArtifactId + '?session=' + f.session.id + '&download=1', { headers: { 'X-Workbench-Token': app.bootstrap.token } });
assert.equal(download.status, 200); assert.match(download.headers.get('content-disposition'), /\.md/); assert.match(await download.text(), /检查通过并已提交/);
const unknown = await fetch(app.url + '/api/artifacts/' + job.reportArtifactId + '?session=' + f.session.id + '&download=1');
assert.equal(unknown.status, 403);
await app.close(); const resumed = await launchWorkbench(t, f.data);
assert.equal((await resumed.api('/sessions/' + f.session.id + '/delivery-jobs/' + jobId)).commit, job.commit);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
});
test('the model can prepare an owned Git review and evidence report while publishing remains a separate user action', async t => {
const f = fixture(t); await checkedDelivery(f);
f.session.status = 'paused'; f.store.save(f.session);
const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let round = 0;
provider.complete = async () => ({ message: { role: 'assistant', content: round > 1 ? 'Git预览已准备,等待用户确认。' : '',
tool_calls: round++ === 0 ? [call('model-git-plan', 'plan_delivery', { message: 'fix: model-prepared reviewed changes' })]
: round === 2 ? [call('model-evidence-report', 'delivery_report', {})] : [] }, usage: { prompt_tokens: 10, completion_tokens: 5 } });
const head = git(f, ['rev-parse', 'HEAD']);
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
const result = f.store.publicSession(f.session);
assert.equal(result.execution.deliveryPlans.length, 1); assert.equal(result.execution.deliveryPlans[0].preparedBy, 'agent');
assert.equal(result.execution.deliveries.length, 0); assert.equal(git(f, ['rev-parse', 'HEAD']), head);
assert.equal(result.events.find(event => event.tool === 'plan_delivery').phase, 'completed');
assert.equal(result.events.some(event => event.tool === 'delivery_report'), false);
assert.equal(round, 1); assert.equal(result.status, 'needs_review'); assert.equal(result.execution.run.stopReason, 'git_review');
assert.throws(() => runtime.resume(f.session.id), { status: 409 });
assert.throws(() => runtime.continueReviewed(f.session.id), { status: 409 });
assertToolPairs(f.session.providerMessages);
assert.equal((await runtime.gitDelivery().report(f.session)).report.status, 'needs_review');
const accepted = await runtime.gitDelivery().start(f.session, result.execution.deliveryPlans[0].id), job = await accepted.promise;
assert.equal(job.status, 'completed', job.summary); assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
assert.equal(f.session.status, 'completed'); assert.equal(f.store.core.run.get(f.session.runId).review.status, 'committed');
assert.equal(JSON.parse(f.store.artifactStore.read(f.store.core.artifact.get(job.reportArtifactId))).status, 'verified_committed');
assert.equal(round, 1);
});
test('HTTP Git review pairs deferred calls, survives reload and never requests more work until the saved preview is resolved', async t => {
const f = fixture(t); await checkedDelivery(f); f.session.status = 'paused'; f.store.save(f.session);
const model = await compatibleFixture(t, () => completion([
call('sealed-http-git', 'plan_delivery', { message: 'fix: HTTP review boundary' }),
call('deferred-after-preview', 'run_command', { command: 'node -e "require(\'node:fs\').writeFileSync(\'should-not-run.txt\',\'no\')"', reason: 'This must wait after Git preview' }),
]));
f.store.saveConfig({ mode: 'compatible', baseUrl: model.baseUrl, model: 'review-fixture' });
const provider = new Provider(); provider.setKey('synthetic-review-key');
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
assert.equal(model.requests.length, 1); assert.equal(f.session.status, 'needs_review');
assert.equal(f.session.approvals.length, 0); assert.equal(fs.existsSync(path.join(f.projectRoot, 'should-not-run.txt')), false);
const invocation = [...f.store.core.tool.values()].find(tool => tool.callId === 'deferred-after-preview');
assert.equal(invocation.state, 'interrupted'); assert.equal(invocation.requiresVerification, false);
assert.equal(JSON.parse(f.store.artifactStore.read(f.store.core.artifact.get(invocation.resultArtifactId))).reason, 'git_review');
assertToolPairs(f.session.providerMessages);
const task = f.session.taskId, plan = f.store.executionSnapshot(f.session).deliveryPlans.at(-1);
f.store.close(); const restored = new Store(f.data); f.onCleanup(() => restored.close());
const resumed = new Runtime(restored, provider); f.onCleanup(() => resumed.stopAll());
assert.throws(() => resumed.resume(f.session.id), { status: 409 }); assert.equal(model.requests.length, 1);
const original = restored.session(f.session.id);
const job = await (await resumed.gitDelivery().start(original, plan.id)).promise;
assert.equal(job.status, 'completed', job.summary); assert.equal(original.taskId, task);
assert.equal(original.status, 'completed'); assert.equal(model.requests.length, 1);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
});
test('a reviewed Git preview can be revised or withdrawn without publishing or losing the current Task', async t => {
const f = fixture(t); await checkedDelivery(f); f.session.status = 'paused'; f.store.save(f.session);
const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let requests = 0;
provider.complete = async () => ({ message: { role: 'assistant', content: '', tool_calls: requests++ === 0
? [call('withdraw-git-plan', 'plan_delivery', { message: 'First preview' })] : [] }, usage: null });
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
const task = f.session.taskId, run = f.session.runId, delivery = runtime.gitDelivery(), original = f.store.executionSnapshot(f.session).deliveryPlans.at(-1);
const revised = await delivery.plan(f.session, { message: 'Revised explicit review' });
assert.equal(f.store.core.delivery_plan.get(original.id).status, 'superseded');
assert.equal(f.store.core.run.get(run).review.planId, revised.id);
const other = f.store.createSession(f.project.id);
assert.throws(() => delivery.discardPlan(other, revised.id), { status: 404 });
const before = git(f, ['status', '--porcelain=v1']);
delivery.discardPlan(f.session, revised.id);
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(run).stopReason, 'git_review_declined');
assert.equal(f.session.taskId, task); assert.equal(git(f, ['status', '--porcelain=v1']), before);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '1');
assert.equal(f.store.executionSnapshot(f.session).deliveries.length, 0);
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.taskId, task); assert.equal(f.session.status, 'completed'); assert.equal(requests, 2);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '1');
});