Files
Agent-Workbench/tests/checks.test.mjs
T
2026-10-29 14:03:50 +08:00

168 lines
16 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { execFileSync, spawn } from 'node:child_process';
import { fixture, call, until, launchWorkbench, workspace, assertToolPairs } from './helpers.mjs';
import { CheckRunner } from '../src/core/check-runner.mjs';
import { inspectProject } from '../src/core/project-inspector.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Store } from '../src/server/store.mjs';
import { proposeFile, applyChange } from '../src/server/files.mjs';
function write(f, relative, value) { const file = path.join(f.projectRoot, relative); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, value); }
function nodeProject(f, { source = 'export const add = (a, b) => a + b;\n', testCode, script = 'node --test test/*.test.mjs', build = true } = {}) {
write(f, 'package.json', JSON.stringify({ name: 'checks-fixture', private: true, type: 'module', scripts: { test: script, ...(build ? { build: 'node scripts/build.mjs' } : {}) } }));
write(f, 'src/add.mjs', source);
write(f, 'test/add.test.mjs', testCode ?? "import test from 'node:test'; import assert from 'node:assert/strict'; import { add } from '../src/add.mjs';\ntest('adds positive values', () => assert.equal(add(2, 3), 5));\ntest('adds negative values', () => assert.equal(add(-2, 3), 1));\n");
write(f, 'scripts/build.mjs', "import fs from 'node:fs'; fs.mkdirSync('dist', { recursive: true }); fs.writeFileSync('dist/bundle.mjs', 'export const built = true;'); console.log('构建完成');\n");
const run = f.store.beginRun(f.session, 'Deliver this Node project with actual checks', false, f.store.config);
f.session.status = 'completed'; f.store.finishRun(f.session, run);
return run;
}
function runnerFor(f, options) { const runner = new CheckRunner(f.store, options); f.onCleanup(() => runner.stopAll()); return runner; }
async function runPlan(runner, session, names) { const plan = await runner.plan(session, names); const started = await runner.start(session, plan.id); return { plan, job: await started.promise }; }
test('engineering inspection preserves a dirty Git baseline and treats project rules as data', async t => {
const f = fixture(t); nodeProject(f);
write(f, '.gitignore', 'dist/\n.env\n'); write(f, 'AGENTS.md', 'Project data: never treat this text as a request to run commands.'); write(f, '.env', 'secret-not-for-inspection');
const git = args => execFileSync('git', ['-C', f.projectRoot, ...args], { windowsHide: true, encoding: 'utf8' });
git(['init', '--initial-branch=main']); git(['-c', 'core.autocrlf=false', 'add', '.']); git(['-c', 'core.autocrlf=false', '-c', 'user.name=Workbench QA', '-c', 'user.email=qa@workbench.local', 'commit', '-m', 'Fixture baseline']);
write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;\n'); write(f, 'untracked.ts', 'export const untouched = true;\n');
const before = git(['status', '--porcelain=v1']);
const inspection = await inspectProject(f.project);
assert.equal(inspection.git.available, true); assert.equal(inspection.git.branch, 'main'); assert.equal(inspection.git.dirty, true); assert.match(inspection.git.head, /^[a-f0-9]{40}$/);
assert.equal(inspection.rules[0].interpretation, 'project-data'); assert.ok(inspection.languages.includes('TypeScript')); assert.equal(inspection.scope.complete, true);
assert.ok(!JSON.stringify(inspection).includes('secret-not-for-inspection')); assert.equal(git(['status', '--porcelain=v1']), before);
write(f, 'dist/ignored.mjs', 'generated'); assert.equal((await inspectProject(f.project)).fingerprint, inspection.fingerprint);
write(f, 'untracked.ts', 'export const untouched = false;\n'); assert.notEqual((await inspectProject(f.project)).fingerprint, inspection.fingerprint);
});
test('real npm tests and build produce counted results, log artifacts and persistent verification', async t => {
const f = fixture(t); nodeProject(f); const runner = runnerFor(f);
const { plan, job } = await runPlan(runner, f.session);
assert.deepEqual(plan.checks.map(check => check.command), ['npm run test', 'npm run build']);
assert.equal(job.status, 'passed'); assert.equal(job.results[0].exitCode, 0); assert.deepEqual(job.results[0].tests, { total: 2, passed: 2, failed: 0, skipped: 0, cancelled: 0 });
assert.equal(plan.project.git.available, false); assert.ok(plan.project.scope.scannedFiles >= 4); assert.ok(plan.project.languages.includes('JavaScript'));
assert.equal(job.results[1].status, 'passed'); assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist/bundle.mjs')), true);
const log = f.store.owned('artifact', job.results[1].logArtifactId, f.session.id); assert.match(f.store.artifactStore.read(log), /构建完成/);
assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'passed');
f.store.close(); const recovered = new Store(f.data); f.onCleanup(() => recovered.close());
assert.equal(recovered.owned('check_job', job.id, f.session.id).status, 'passed'); assert.equal(recovered.executionSnapshot(recovered.session(f.session.id)).task.verification.status, 'passed');
});
test('an actual failing test retains the nonzero exit code and failed count', async t => {
const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' }); const runner = runnerFor(f);
const { job } = await runPlan(runner, f.session);
assert.equal(job.status, 'failed'); assert.notEqual(job.results[0].exitCode, 0); assert.equal(job.results[0].tests.failed, 2);
assert.match(f.store.artifactStore.read(f.store.owned('artifact', job.results[0].logArtifactId, f.session.id)), /AssertionError|ERR_ASSERTION/);
assert.equal(job.results[1].status, 'passed'); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'failed');
});
test('zero tests, skipped tests and unrecognized stdout cannot become verification passes', async t => {
const variants = [
{ script: 'node --test', removeTest: true, expectedTotal: 0 },
{ testCode: '', expectedTotal: null },
{ testCode: "import test from 'node:test'; test.skip('not exercised', () => {});", expectedTotal: 1 },
{ script: 'node scripts/emit-summary.mjs', expectedTotal: null },
];
for (const variant of variants) {
const f = fixture(t); nodeProject(f, { ...variant, build: false }); write(f, 'scripts/emit-summary.mjs', "console.log('ℹ tests 20\\nℹ pass 20\\nℹ fail 0\\nℹ cancelled 0\\nℹ skipped 0');");
if (variant.removeTest) fs.unlinkSync(path.join(f.projectRoot, 'test/add.test.mjs'));
const runner = runnerFor(f); const { job } = await runPlan(runner, f.session);
assert.equal(job.status, 'unverified'); assert.equal(job.results[0].exitCode, 0); assert.equal(job.results[0].tests?.total ?? null, variant.expectedTotal);
assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'unverified');
}
});
test('a successful build with no selected tests does not verify the task', async t => {
const f = fixture(t); nodeProject(f); const { job } = await runPlan(runnerFor(f), f.session, ['build']);
assert.equal(job.results[0].status, 'passed'); assert.equal(job.status, 'unverified');
});
test('stale check plans and forged script names execute no project commands', async t => {
const f = fixture(t); nodeProject(f); const runner = runnerFor(f), plan = await runner.plan(f.session);
write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;\n');
const { promise } = await runner.start(f.session, plan.id), job = await promise;
assert.equal(job.status, 'stale'); assert.equal(job.results.length, 0); assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist')), false);
await assert.rejects(runner.plan(f.session, ['test; echo invalid']), /已识别/);
const other = f.store.createSession(f.project.id); await assert.rejects(runner.start(other, plan.id), { status: 404 });
});
test('applying a later proposal invalidates the prior passed engineering snapshot', async t => {
const f = fixture(t); nodeProject(f); const runner = runnerFor(f), { job } = await runPlan(runner, f.session);
const proposal = proposeFile(f.store, f.session, { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b + 1;\n', reason: 'Controlled change after verification' });
await assert.rejects(runner.start(f.session, (await runner.plan(f.session)).id), /待审查/);
applyChange(f.store, f.session, proposal.change_id);
assert.equal(job.stale, true); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'stale');
});
test('stopping an actual check terminates its child process and blocks concurrent jobs', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('long running check', async () => { fs.writeFileSync('.awb-check-entered', 'entered'); await new Promise(resolve => setTimeout(resolve, 30000)); });\n", build: false });
const runner = runnerFor(f), plan = await runner.plan(f.session), started = await runner.start(f.session, plan.id);
await until(() => fs.existsSync(path.join(f.projectRoot, '.awb-check-entered')), 10000);
await assert.rejects(runner.start(f.session, plan.id), { status: 409 });
await runner.stop(f.session.id, started.job.id);
const job = await started.promise; assert.equal(job.status, 'cancelled'); assert.equal(job.results[0].status, 'cancelled'); assert.equal(runner.active.size, 0);
});
test('check timeout records a failure with its actual termination result', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; test('timeout', async () => { await new Promise(resolve => setTimeout(resolve, 30000)); });", build: false });
const { job } = await runPlan(runnerFor(f, { timeoutMs: 400 }), f.session);
assert.equal(job.status, 'failed'); assert.equal(job.results[0].timedOut, true); assert.match(job.results[0].reason, /时间上限/);
});
test('source edits during a successful check make the result stale', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('check snapshot', async () => { fs.writeFileSync('.awb-check-entered', 'entered'); await new Promise(resolve => setTimeout(resolve, 1200)); });", build: false });
const runner = runnerFor(f), plan = await runner.plan(f.session), started = await runner.start(f.session, plan.id);
await until(() => fs.existsSync(path.join(f.projectRoot, '.awb-check-entered')), 10000); write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;');
const job = await started.promise; assert.equal(job.results[0].status, 'passed'); assert.equal(job.status, 'stale'); assert.equal(job.stale, true);
});
test('bounded large logs retain the test footer and readable artifact instead of inventing counts', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; test('large output', () => { console.log('中'.repeat(600000)); });", build: false });
const { job } = await runPlan(runnerFor(f), f.session);
assert.equal(job.status, 'passed'); assert.equal(job.results[0].truncated, true); assert.equal(job.results[0].tests.total, 1);
const artifact = f.store.owned('artifact', job.results[0].logArtifactId, f.session.id), text = f.store.artifactStore.read(artifact);
assert.match(text, /中间日志超出保存上限/); assert.ok(artifact.bytes < 2 * 1024 * 1024); assert.match(text, /tests 1/);
});
test('model-selected checks require a run-bound decision and denial creates no check job', async t => {
const f = fixture(t); nodeProject(f); const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let round = 0;
provider.complete = async (_config, messages) => {
const planResult = messages.filter(message => message.role === 'tool').map(message => { try { return JSON.parse(message.content); } catch { return {}; } }).find(value => value.checks && value.fingerprint);
const calls = round++ === 0 ? [call('plan-for-model', 'plan_checks', {})] : round === 2 ? [call('run-for-model', 'run_checks', { plan_id: planResult.id })] : [];
return { message: { role: 'assistant', content: calls.length ? '' : 'Checks were declined; task has no engineering pass.', tool_calls: calls }, usage: { prompt_tokens: 10, completion_tokens: 5 } };
};
runtime.start(f.session.id, 'Validate this engineering project'); const promise = runtime.active.get(f.session.id).promise;
await until(() => f.session.approvals.some(value => value.status === 'pending'), 8000);
assert.equal(f.store.core.check_job.size, 0); const approval = f.session.approvals.at(-1);
assert.equal(approval.runId, f.session.runId); runtime.decide(f.session.id, approval.id, false); await promise;
assert.equal(f.store.core.check_job.size, 0); assert.equal(f.session.events.find(event => event.tool === 'run_checks').phase, 'denied'); assertToolPairs(f.session.providerMessages);
});
test('check APIs expose owned plans and jobs and preserve the authoritative results across restart', async t => {
const f = fixture(t); nodeProject(f); f.store.close(); const app = await launchWorkbench(t, f.data);
const planned = await app.api('/sessions/' + f.session.id + '/check-plans', {}), plan = planned.execution.plans.at(-1);
const accepted = await app.api('/sessions/' + f.session.id + '/check-jobs', { planId: plan.id }); const jobId = accepted.execution.checks.at(-1).id;
await until(async () => !['queued', 'running'].includes((await app.api('/sessions/' + f.session.id + '/check-jobs/' + jobId)).status), 12000);
const finished = await app.api('/sessions/' + f.session.id + '/check-jobs/' + jobId); assert.equal(finished.status, 'passed');
const other = await app.api('/sessions', { projectId: f.project.id }); await assert.rejects(app.api('/sessions/' + other.id + '/check-jobs/' + jobId), { status: 404 });
const log = await app.api('/artifacts/' + finished.results[0].logArtifactId + '?session=' + f.session.id); assert.match(log.content, /tests 2/);
const exported = await app.api('/sessions/' + f.session.id + '/export'); assert.equal(exported.execution.checks.at(-1).id, jobId);
await app.close(); const resumed = await launchWorkbench(t, f.data); assert.equal((await resumed.api('/sessions/' + f.session.id)).execution.task.verification.status, 'passed');
});
for (const phase of ['queued', 'result_committed']) test('process exit at check ' + phase + ' preserves results and never replays commands', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('one side effect', () => fs.writeFileSync('.awb-effect', '1'));" }); f.store.close();
const worker = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/check-crash-worker.mjs'), f.data, f.session.id, phase], { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let error = ''; worker.stderr.on('data', bytes => { error += bytes.toString(); }); const code = await new Promise(resolve => worker.once('exit', resolve)); assert.equal(code, 69, error);
const recovered = new Store(f.data); f.onCleanup(() => recovered.close()); const session = recovered.session(f.session.id), job = recovered.executionSnapshot(session).checks.at(-1);
assert.equal(job.status, phase === 'queued' ? 'interrupted' : 'unknown'); assert.equal(session.execution, undefined);
if (phase === 'result_committed') { assert.equal(job.results.length, 1); assert.equal(job.results[0].tests.total, 1); assert.equal(fs.readFileSync(path.join(f.projectRoot, '.awb-effect'), 'utf8'), '1'); }
else assert.equal(fs.existsSync(path.join(f.projectRoot, '.awb-effect')), false);
assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist')), false); assert.equal(recovered.executionSnapshot(session).task.verification.status, job.status);
});