Files
Agent-Workbench/tests/terminal-delivery.test.mjs

221 lines
18 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { randomUUID } from 'node:crypto';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { DeliveryReporter } from '../src/core/delivery-report.mjs';
import { CheckRunner } from '../src/core/check-runner.mjs';
import { fixture, call, compatibleFixture, completion, launchWorkbench, until, workspace } from './helpers.mjs';
import { createDeliveryProject, applyDeliveryFix, deliveryGit as git, writeDeliveryFile as write } from './fixtures/delivery-project.mjs';
const paths = ['src/add.mjs', 'src/math.mjs'];
const commit = message => 'git -c core.hooksPath= commit --no-gpg-sign --only -m "' + message + '" -- ' + paths.join(' ');
function engineering(t, { apply = true } = {}) {
const f = fixture(t, { modernDefaults: true }); f.store.saveConfig({ approvalMode: 'full' });
createDeliveryProject(f);
const record = f.store.beginRun(f.session, 'Continue the original repair and deliver its exact source files.', true, f.store.config);
f.session.status = 'running';
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
const run = { id: record.id, taskId: record.taskId, record, config: f.store.config, project: f.project, controller: new AbortController() };
if (apply) applyDeliveryFix(f);
return Object.assign(f, { runtime, run, command: text => runtime.executeTool(f.session,
call(randomUUID(), 'run_command', { command: text, reason: 'Owned terminal delivery fixture.' }), run) });
}
function read(f, id) { return JSON.parse(f.store.artifactStore.read(f.store.core.artifact.get(id))); }
function reports(f) { return [...f.store.core.artifact.values()].filter(row => row.kind === 'delivery-report'); }
test('split terminal add/commit is independently verified without inventing a delivery job or altering owner index', { timeout: 45000 }, async t => {
const f = engineering(t), parent = git(f, ['rev-parse', 'HEAD']).trim();
const ownerIndex = git(f, ['show', ':owner-staged.txt']);
const owner = ['owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(f.projectRoot, name))]);
const checks = new CheckRunner(f.store); f.onCleanup(() => checks.stopAll());
const job = await (await checks.start(f.session, (await checks.plan(f.session)).id)).promise;
assert.equal(job.status, 'passed');
assert.equal((await f.command('git add -- ' + paths.join(' '))).status, 'completed');
const result = await f.command(commit('actual terminal repair'));
assert.equal(result.status, 'completed', result.stderr); assert.equal(result.terminal_git.status, 'verified');
const head = git(f, ['rev-parse', 'HEAD']).trim(); assert.notEqual(head, parent);
assert.equal(result.terminal_git.commit, head); assert.equal(f.store.core.delivery_job.size, 0);
assert.equal(git(f, ['show', ':owner-staged.txt']), ownerIndex);
for (const [name, bytes] of owner) assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, name)), bytes);
f.session.status = 'completed'; f.store.finishRun(f.session, f.run.record);
const report = (await new DeliveryReporter(f.store).create(f.session)).report;
assert.equal(report.status, 'verified_committed'); assert.equal(report.gitOutcome.status, 'committed');
assert.deepEqual(report.gitOutcome.sources, ['terminal']); assert.equal(report.deliveries.length, 0);
assert.equal(report.terminalGit[0].currentVerified, true); assert.equal(report.terminalGit[0].processAttribution, 'unproven');
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
});
test('terminal commands still execute when a real commit includes an unrelated file, and the report preserves that mismatch', { timeout: 35000 }, async t => {
const f = engineering(t);
const result = await f.command('git add -- ' + paths.join(' ') + '\ngit -c core.hooksPath= commit --no-gpg-sign --only -m "mixed scope" -- ' + paths.join(' ') + ' owner-unstaged.txt');
assert.equal(result.status, 'completed', result.stderr); assert.equal(result.terminal_git.status, 'unverified');
const report = await new DeliveryReporter(f.store).inspect(f.session);
assert.equal(report.gitOutcome.status, 'observed_unverified'); assert.equal(report.terminalGit[0].evidenceVerified, true);
assert.equal(report.terminalGit[0].currentVerified, false); assert.match(report.terminalGit[0].error, /路径/);
assert.ok(report.terminalGit[0].paths.includes('owner-unstaged.txt')); assert.equal(f.store.core.delivery_job.size, 0);
});
test('spoofed commit output cannot become terminal Git evidence', { timeout: 25000 }, async t => {
const f = engineering(t), head = git(f, ['rev-parse', 'HEAD']).trim();
const command = process.platform === 'win32' ? 'Write-Output "[main abc1234] committed successfully"' : 'printf "[main abc1234] committed successfully\\n"';
assert.equal((await f.command(command)).status, 'completed');
assert.equal(git(f, ['rev-parse', 'HEAD']).trim(), head); assert.equal(f.run.record.terminalGitEvidence, undefined);
assert.equal((await new DeliveryReporter(f.store).inspect(f.session)).gitOutcome.status, 'uncommitted');
});
test('a changed non-target index and external HEAD drift remain unverified without blocking or replaying the real command', { timeout: 50000 }, async t => {
for (const external of [false, true]) {
const f = engineering(t);
await f.command('git add -- ' + paths.join(' '));
if (external) git(f, ['-c', 'core.hooksPath=', 'commit', '--no-gpg-sign', '--only', '-m', 'outside terminal window', '--', 'owner-unstaged.txt']);
const result = await f.command((external ? '' : 'git add -- owner-unstaged.txt\n') + commit('scope under observation'));
assert.equal(result.status, 'completed', result.stderr); assert.equal(result.terminal_git.status, 'unverified');
const evidence = read(f, result.terminal_git.artifact_id);
assert.equal(evidence.externalDrift, external);
assert.match(evidence.errors.join(' '), external ? /外部漂移/ : /非目标Git索引/);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), external ? '3' : '2');
}
});
test('intermediate commits cannot hide an unrelated file by reverting it before the final net diff', { timeout: 35000 }, async t => {
const f = engineering(t);
const add = process.platform === 'win32' ? 'Set-Content -LiteralPath transient.txt -Value temporary' : 'printf temporary > transient.txt';
const remove = process.platform === 'win32' ? 'Remove-Item -LiteralPath transient.txt' : 'rm transient.txt';
const result = await f.command([add, 'git add -- ' + paths.join(' ') + ' transient.txt', 'git -c core.hooksPath= commit --no-gpg-sign --only -m "transient outside file" -- transient.txt',
remove, 'git -c core.hooksPath= commit --no-gpg-sign --only -m "final repair" -- ' + paths.join(' ') + ' transient.txt'].join('\n'));
assert.equal(result.status, 'completed', result.stderr); assert.equal(result.terminal_git.status, 'unverified');
const evidence = read(f, result.terminal_git.artifact_id);
assert.deepEqual(evidence.paths, paths); assert.equal(evidence.commits.length, 2); assert.match(evidence.errors.join(' '), /中间提交/);
});
test('later source changes and damaged original receipts cannot certify the current terminal commit', { timeout: 35000 }, async t => {
const f = engineering(t); const result = await f.command('git add -- ' + paths.join(' ') + '\n' + commit('receipt verification'));
assert.equal(result.status, 'completed', result.stderr);
assert.equal(result.terminal_git.status, 'verified');
const reporter = new DeliveryReporter(f.store);
assert.equal((await reporter.inspect(f.session)).terminalGit[0].currentVerified, true);
const original = fs.readFileSync(path.join(f.projectRoot, paths[0])); write(f, paths[0], 'export const modifiedLater = true;\n');
assert.equal((await reporter.inspect(f.session)).terminalGit[0].status, 'stale');
write(f, paths[0], original);
const artifact = f.store.core.artifact.get(result.terminal_git.artifact_id);
fs.writeFileSync(path.join(f.store.artifactStore.root, artifact.relative), '{}');
const damaged = await reporter.inspect(f.session);
assert.equal(damaged.terminalGit[0].evidenceVerified, false); assert.equal(damaged.status, 'needs_verification');
assert.match(damaged.terminalGit[0].error, /校验失败/); assert.equal(f.store.core.delivery_job.size, 0);
});
test('final reports append to immutable in-progress reports and do not appear for greetings, diagnostics, workers or a different task', { timeout: 35000 }, async t => {
const f = engineering(t);
const saved = await f.runtime.gitDelivery().report(f.session, { source: 'agent' });
const original = f.store.artifactStore.read(f.store.core.artifact.get(saved.artifactId));
assert.equal(saved.report.status, 'in_progress'); assert.equal(saved.report.lifecycle.phase, 'snapshot');
f.session.status = 'completed'; f.store.finishRun(f.session, f.run.record);
await f.runtime.finalizeDeliveryReport(f.session, f.run);
const final = read(f, f.run.record.finalDeliveryReport.artifactId);
assert.notEqual(final.status, 'in_progress'); assert.equal(final.lifecycle.phase, 'final'); assert.equal(final.lifecycle.runStatus, 'completed');
assert.ok(final.lifecycle.finishedAt); assert.equal(f.store.artifactStore.read(f.store.core.artifact.get(saved.artifactId)), original);
await f.runtime.finalizeDeliveryReport(f.session, f.run); assert.equal(reports(f).length, 2);
for (const kind of ['greeting', 'diagnostic', 'worker', 'different-task']) {
const other = engineering(t, { apply: false });
if (kind !== 'greeting') await other.runtime.gitDelivery().report(other.session, { source: 'agent' });
const count = reports(other).length;
if (kind === 'diagnostic') other.run.record.purpose = 'diagnostic';
if (kind === 'worker') other.run.worker = { id: randomUUID() };
if (kind === 'different-task') other.store.beginRun(other.session, 'A separate new task.', false, other.store.config, { forceNewTask: true });
await other.runtime.finalizeDeliveryReport(other.session, other.run);
assert.equal(reports(other).length, count, kind); assert.equal(other.run.record.finalDeliveryReport, undefined, kind);
}
});
test('report failures preserve the completed run and never retry a terminal side effect', { timeout: 25000 }, async t => {
const f = engineering(t); const result = await f.command('git add -- ' + paths.join(' ') + '\n' + commit('keep completed command'));
assert.equal(result.status, 'completed', result.stderr);
f.session.status = 'completed'; f.store.finishRun(f.session, f.run.record);
f.runtime.gitDelivery().report = async () => { throw new Error('Owned report failure'); };
await f.runtime.finalizeDeliveryReport(f.session, f.run);
assert.equal(f.session.status, 'completed'); assert.equal(f.run.record.status, 'completed');
assert.match(f.run.record.finalDeliveryReportError, /Owned report failure/); assert.equal(f.run.record.finalDeliveryReport, undefined);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
});
test('a real commit remains observable when a later command fails and its exit code is preserved', { timeout: 25000 }, async t => {
const f = engineering(t);
const result = await f.command('git add -- ' + paths.join(' ') + '\n' + commit('commit before command failure') + '\nexit 7');
assert.equal(result.status, 'failed'); assert.equal(result.exit_code, 7); assert.equal(result.terminal_git.status, 'verified');
f.session.status = 'failed'; f.store.finishRun(f.session, f.run.record);
const report = await new DeliveryReporter(f.store).inspect(f.session);
assert.equal(report.gitOutcome.status, 'committed'); assert.equal(report.terminalGit[0].command.exitCode, 7);
assert.equal(report.lifecycle.runStatus, 'failed'); assert.equal(f.store.core.delivery_job.size, 0);
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
});
test('observer initialization and post-observation failures never suppress a real terminal command', { timeout: 25000 }, async t => {
for (const phase of ['initialization', 'after']) {
const f = engineering(t, { apply: false });
if (phase === 'initialization') f.runtime.gitDelivery = () => { throw new Error('Owned observer initialization failure'); };
else f.runtime.terminalGit = { before: async () => ({}), after: async () => { throw new Error('Owned post-observation failure'); } };
const result = await f.command('git --version');
assert.equal(result.status, 'completed'); assert.equal(result.exit_code, 0); assert.match(result.stdout, /git version/);
assert.equal(result.terminal_git.status, 'unverified'); assert.match(result.terminal_git.message, /Owned/);
assert.match(f.run.record.terminalGitObservationError, /Owned/); assert.equal(f.store.core.delivery_job.size, 0);
}
});
test('isolated HTTP model tools register a real terminal commit and append a final report under saved full access', { timeout: 60000 }, async t => {
const f = engineering(t, { apply: false }); f.store.close(); let requests = 0, oldArtifact;
const model = await compatibleFixture(t, (_url, body) => {
requests++;
if (requests === 1) return completion([
call('http-add', 'propose_file', { path: paths[0], content: "import { sum } from './math.mjs';\nexport const add = sum;\n", reason: 'Owned HTTP repair.' }),
call('http-math', 'propose_file', { path: paths[1], content: 'export const sum = (a, b) => a + b;\n', reason: 'Owned HTTP repair.' }),
]);
if (requests === 2) return completion([call('http-stage', 'run_command', { command: 'git add -- ' + paths.join(' '), reason: 'Stage only owned task files.' })]);
if (requests === 3) {
const staged = JSON.parse(body.messages.findLast(row => row.role === 'tool').content);
assert.equal(staged.status, 'completed', JSON.stringify(staged));
return completion([call('http-commit', 'run_command', { command: commit('HTTP terminal delivery'), reason: 'Commit owned task files while preserving owner staging.' })]);
}
if (requests === 4) {
const committed = JSON.parse(body.messages.findLast(row => row.role === 'tool').content);
assert.equal(committed.status, 'completed', JSON.stringify(committed)); assert.equal(committed.terminal_git.status, 'verified');
return completion([call('http-report', 'delivery_report', {})]);
}
assert.equal(requests, 5); oldArtifact = JSON.parse(body.messages.findLast(row => row.role === 'tool').content).artifact_id;
return completion([], 'The owned terminal repair is committed.');
});
const appRoot = process.env.AGENT_WORKBENCH_TEST_APP_ROOT ?? workspace;
const host = await launchWorkbench(t, f.data, {}, { appRoot });
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'owned-terminal-delivery', apiKey: 'owned-local-fixture-only' });
const route = '/sessions/' + f.session.id; await host.api(route + '/messages', { content: 'Complete the addition repair and make a local commit; preserve owner files and index.' });
let state; await until(async () => { state = await host.api(route); return state.execution.run.finalDeliveryReport || state.execution.run.finalDeliveryReportError; }, 45000);
assert.equal(state.status, 'completed'); assert.equal(requests, 5); assert.ok(state.execution.run.finalDeliveryReport, state.execution.run.finalDeliveryReportError);
assert.equal(state.execution.deliveries.length, 0); assert.ok(state.approvals.every(row => row.status === 'approved' && row.approvalMode === 'full'));
const final = JSON.parse((await host.api('/artifacts/' + state.execution.run.finalDeliveryReport.artifactId + '?session=' + f.session.id + '&limit=1000000')).content);
const old = JSON.parse((await host.api('/artifacts/' + oldArtifact + '?session=' + f.session.id + '&limit=1000000')).content);
assert.equal(final.gitOutcome.status, 'committed'); assert.equal(final.terminalGit[0].currentVerified, true);
assert.equal(final.lifecycle.phase, 'final'); assert.equal(old.status, 'in_progress'); assert.equal(old.lifecycle.phase, 'snapshot');
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
});
test('a real HTTP upstream failure remains a failed run while its applied files get a final evidence report', { timeout: 40000 }, async t => {
const f = engineering(t, { apply: false }); f.store.close(); let requests = 0;
const model = await compatibleFixture(t, () => {
requests++;
if (requests === 1) return completion([call('http-failure-proposal', 'propose_file', { path: paths[0], content: 'export const add = (a, b) => a + b;\n', reason: 'Owned failure fixture repair.' })]);
return { status: 400, body: { error: { message: 'owned upstream failure is preserved', code: 'owned_fixture_error' } } };
});
const appRoot = process.env.AGENT_WORKBENCH_TEST_APP_ROOT ?? workspace;
const host = await launchWorkbench(t, f.data, {}, { appRoot });
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'owned-failure-report', apiKey: 'owned-local-fixture-only' });
const route = '/sessions/' + f.session.id; await host.api(route + '/messages', { content: 'Repair addition and report the real result.' });
let state; await until(async () => { state = await host.api(route); return state.execution.run.finalDeliveryReport || state.execution.run.finalDeliveryReportError; }, 30000);
assert.equal(state.status, 'failed'); assert.equal(requests, 2); assert.ok(state.execution.run.finalDeliveryReport, state.execution.run.finalDeliveryReportError);
assert.match(JSON.stringify(state.lastFailure), /owned upstream failure is preserved/);
const final = JSON.parse((await host.api('/artifacts/' + state.execution.run.finalDeliveryReport.artifactId + '?session=' + f.session.id + '&limit=1000000')).content);
assert.equal(final.lifecycle.runStatus, 'failed'); assert.equal(final.lifecycle.phase, 'final'); assert.notEqual(final.status, 'in_progress');
assert.equal(git(f, ['rev-list', '--count', 'HEAD']).trim(), '1');
});