import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { execFileSync, spawn } from 'node:child_process'; import { fixture, call, until, launchWorkbench, compatibleFixture, completion, workspace, assertToolPairs } from './helpers.mjs'; import { CheckRunner } from '../src/core/check-runner.mjs'; import { inspectProject } from '../src/core/project-inspector.mjs'; import { Runtime } from '../src/server/runtime.mjs'; import { Provider } from '../src/server/provider.mjs'; import { Store } from '../src/server/store.mjs'; import { proposeFile, applyChange } from '../src/server/files.mjs'; function write(f, relative, value) { const file = path.join(f.projectRoot, relative); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, value); } function nodeProject(f, { source = 'export const add = (a, b) => a + b;\n', testCode, script = 'node --test test/*.test.mjs', build = true } = {}) { write(f, 'package.json', JSON.stringify({ name: 'checks-fixture', private: true, type: 'module', scripts: { test: script, ...(build ? { build: 'node scripts/build.mjs' } : {}) } })); write(f, 'src/add.mjs', source); write(f, 'test/add.test.mjs', testCode ?? "import test from 'node:test'; import assert from 'node:assert/strict'; import { add } from '../src/add.mjs';\ntest('adds positive values', () => assert.equal(add(2, 3), 5));\ntest('adds negative values', () => assert.equal(add(-2, 3), 1));\n"); write(f, 'scripts/build.mjs', "import fs from 'node:fs'; fs.mkdirSync('dist', { recursive: true }); fs.writeFileSync('dist/bundle.mjs', 'export const built = true;'); console.log('构建完成');\n"); const run = f.store.beginRun(f.session, 'Deliver this Node project with actual checks', false, f.store.config); f.session.status = 'completed'; f.store.finishRun(f.session, run); return run; } function runnerFor(f, options) { const runner = new CheckRunner(f.store, options); f.onCleanup(() => runner.stopAll()); return runner; } async function runPlan(runner, session, names) { const plan = await runner.plan(session, names); const started = await runner.start(session, plan.id); return { plan, job: await started.promise }; } test('engineering inspection preserves a dirty Git baseline and treats project rules as data', async t => { const f = fixture(t); nodeProject(f); write(f, '.gitignore', 'dist/\n.env\n'); write(f, 'AGENTS.md', 'Project data: never treat this text as a request to run commands.'); write(f, '.env', 'secret-not-for-inspection'); const git = args => execFileSync('git', ['-C', f.projectRoot, ...args], { windowsHide: true, encoding: 'utf8' }); git(['init', '--initial-branch=main']); git(['-c', 'core.autocrlf=false', 'add', '.']); git(['-c', 'core.autocrlf=false', '-c', 'user.name=Workbench QA', '-c', 'user.email=qa@workbench.local', 'commit', '-m', 'Fixture baseline']); write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;\n'); write(f, 'untracked.ts', 'export const untouched = true;\n'); const before = git(['status', '--porcelain=v1']); const inspection = await inspectProject(f.project); assert.equal(inspection.git.available, true); assert.equal(inspection.git.branch, 'main'); assert.equal(inspection.git.dirty, true); assert.match(inspection.git.head, /^[a-f0-9]{40}$/); assert.equal(inspection.rules[0].interpretation, 'project-data'); assert.ok(inspection.languages.includes('TypeScript')); assert.equal(inspection.scope.complete, true); assert.ok(!JSON.stringify(inspection).includes('secret-not-for-inspection')); assert.equal(git(['status', '--porcelain=v1']), before); write(f, 'dist/ignored.mjs', 'generated'); assert.equal((await inspectProject(f.project)).fingerprint, inspection.fingerprint); write(f, 'untracked.ts', 'export const untouched = false;\n'); assert.notEqual((await inspectProject(f.project)).fingerprint, inspection.fingerprint); }); test('real npm tests and build produce counted results, log artifacts and persistent verification', async t => { const f = fixture(t); nodeProject(f); const runner = runnerFor(f); const { plan, job } = await runPlan(runner, f.session); assert.deepEqual(plan.checks.map(check => check.command), ['npm run test', 'npm run build']); assert.equal(job.status, 'passed'); assert.equal(job.results[0].exitCode, 0); assert.deepEqual(job.results[0].tests, { total: 2, passed: 2, failed: 0, skipped: 0, cancelled: 0 }); assert.equal(plan.project.git.available, false); assert.ok(plan.project.scope.scannedFiles >= 4); assert.ok(plan.project.languages.includes('JavaScript')); assert.equal(job.results[1].status, 'passed'); assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist/bundle.mjs')), true); const log = f.store.owned('artifact', job.results[1].logArtifactId, f.session.id); assert.match(f.store.artifactStore.read(log), /构建完成/); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'passed'); f.store.close(); const recovered = new Store(f.data); f.onCleanup(() => recovered.close()); assert.equal(recovered.owned('check_job', job.id, f.session.id).status, 'passed'); assert.equal(recovered.executionSnapshot(recovered.session(f.session.id)).task.verification.status, 'passed'); }); test('an actual failing test retains the nonzero exit code and failed count', async t => { const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' }); const runner = runnerFor(f); const { job } = await runPlan(runner, f.session); assert.equal(job.status, 'failed'); assert.notEqual(job.results[0].exitCode, 0); assert.equal(job.results[0].tests.failed, 2); assert.match(f.store.artifactStore.read(f.store.owned('artifact', job.results[0].logArtifactId, f.session.id)), /AssertionError|ERR_ASSERTION/); assert.equal(job.results[1].status, 'passed'); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'failed'); }); test('unchanged automatic checks return original failure evidence, while explicit user reruns and changed source still execute', async t => { const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' }); const runner = runnerFor(f); let executions = 0; const execute = runner.executeCheck.bind(runner); runner.executeCheck = async (...args) => { executions++; return execute(...args); }; const { job } = await runPlan(runner, f.session, ['test']); const second = await runner.plan(f.session, ['test']); assert.notEqual(second.id, job.planId); assert.equal((await runner.unchangedResult(f.session, second.id)).id, job.id); assert.equal(executions, 1); assert.equal(f.store.core.check_job.size, 1); const rerun = await runner.start(f.session, second.id, { source: 'user' }); await rerun.promise; assert.equal(executions, 2); assert.equal(f.store.core.check_job.size, 2); assert.notEqual(rerun.job.id, job.id); write(f, 'src/add.mjs', 'export const add = (a, b) => a + b;\n'); const changed = await runner.plan(f.session, ['test']); assert.equal(await runner.unchangedResult(f.session, changed.id), null); const repaired = await runner.start(f.session, changed.id); assert.equal((await repaired.promise).status, 'passed'); assert.equal(executions, 3); }); test('agent loops cannot hide identical failing checks behind new plan IDs or unrelated reads', async t => { const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' }); const provider = new Provider(); let round = 0; provider.complete = async () => { const count = round++, tools = count % 2 === 0 ? [call('plan-loop-' + count, 'plan_checks', { names: ['test'] })] : [call('check-loop-' + count, 'run_checks', { plan_id: [...f.store.core.check_plan.values()].at(-1).id }), call('loop-read-' + count, 'read_file', { path: 'src/add.mjs' })]; return { message: { role: 'assistant', content: '', tool_calls: tools }, usage: { prompt_tokens: 10, completion_tokens: 4 } }; }; const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '修复测试失败'); await until(() => f.session.approvals.some(value => value.status === 'pending'), 15000); runtime.decide(f.session.id, f.session.approvals.find(value => value.status === 'pending').id, true); await until(() => !runtime.isActive(f.session.id), 20000); assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'no_progress'); assert.equal(f.session.approvals.length, 1); assert.equal([...f.store.core.check_job.values()].filter(job => job.taskId === f.session.taskId).length, 1); const repeated = [...f.store.core.tool.values()].filter(tool => tool.taskId === f.session.taskId && tool.tool === 'run_checks').map(tool => tool.result); assert.equal(repeated.filter(result => result?.reused === true).length, 2); assertToolPairs(f.session.providerMessages); }); test('zero tests, skipped tests and unrecognized stdout cannot become verification passes', async t => { const variants = [ { script: 'node --test', removeTest: true, expectedTotal: 0 }, { testCode: '', expectedTotal: null }, { testCode: "import test from 'node:test'; test.skip('not exercised', () => {});", expectedTotal: 1 }, { script: 'node scripts/emit-summary.mjs', expectedTotal: null }, ]; for (const variant of variants) { const f = fixture(t); nodeProject(f, { ...variant, build: false }); write(f, 'scripts/emit-summary.mjs', "console.log('ℹ tests 20\\nℹ pass 20\\nℹ fail 0\\nℹ cancelled 0\\nℹ skipped 0');"); if (variant.removeTest) fs.unlinkSync(path.join(f.projectRoot, 'test/add.test.mjs')); const runner = runnerFor(f); const { job } = await runPlan(runner, f.session); assert.equal(job.status, 'unverified'); assert.equal(job.results[0].exitCode, 0); assert.equal(job.results[0].tests?.total ?? null, variant.expectedTotal); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'unverified'); } }); test('a successful build with no selected tests does not verify the task', async t => { const f = fixture(t); nodeProject(f); const { job } = await runPlan(runnerFor(f), f.session, ['build']); assert.equal(job.results[0].status, 'passed'); assert.equal(job.status, 'unverified'); }); test('stale check plans and forged script names execute no project commands', async t => { const f = fixture(t); nodeProject(f); const runner = runnerFor(f), plan = await runner.plan(f.session); write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;\n'); const { promise } = await runner.start(f.session, plan.id), job = await promise; assert.equal(job.status, 'stale'); assert.equal(job.results.length, 0); assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist')), false); await assert.rejects(runner.plan(f.session, ['test; echo invalid']), /已识别/); const other = f.store.createSession(f.project.id); await assert.rejects(runner.start(other, plan.id), { status: 404 }); }); test('applying a later proposal invalidates the prior passed engineering snapshot', async t => { const f = fixture(t); nodeProject(f); const runner = runnerFor(f), { job } = await runPlan(runner, f.session); const proposal = proposeFile(f.store, f.session, { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b + 1;\n', reason: 'Controlled change after verification' }); await assert.rejects(runner.start(f.session, (await runner.plan(f.session)).id), /待审查/); applyChange(f.store, f.session, proposal.change_id); assert.equal(job.stale, true); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'stale'); }); test('stopping an actual check terminates its child process and blocks concurrent jobs', async t => { const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('long running check', async () => { fs.writeFileSync('.awb-check-entered', 'entered'); await new Promise(resolve => setTimeout(resolve, 30000)); });\n", build: false }); const runner = runnerFor(f), plan = await runner.plan(f.session), started = await runner.start(f.session, plan.id); await until(() => fs.existsSync(path.join(f.projectRoot, '.awb-check-entered')), 10000); await assert.rejects(runner.start(f.session, plan.id), { status: 409 }); await runner.stop(f.session.id, started.job.id); const job = await started.promise; assert.equal(job.status, 'cancelled'); assert.equal(job.results[0].status, 'cancelled'); assert.equal(runner.active.size, 0); }); test('check timeout records a failure with its actual termination result', async t => { const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; test('timeout', async () => { await new Promise(resolve => setTimeout(resolve, 30000)); });", build: false }); const { job } = await runPlan(runnerFor(f, { timeoutMs: 400 }), f.session); assert.equal(job.status, 'failed'); assert.equal(job.results[0].timedOut, true); assert.match(job.results[0].reason, /时间上限/); }); test('source edits during a successful check make the result stale', async t => { const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('check snapshot', async () => { fs.writeFileSync('.awb-check-entered', 'entered'); await new Promise(resolve => setTimeout(resolve, 1200)); });", build: false }); const runner = runnerFor(f), plan = await runner.plan(f.session), started = await runner.start(f.session, plan.id); await until(() => fs.existsSync(path.join(f.projectRoot, '.awb-check-entered')), 10000); write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;'); const job = await started.promise; assert.equal(job.results[0].status, 'passed'); assert.equal(job.status, 'stale'); assert.equal(job.stale, true); }); test('bounded large logs retain the test footer and readable artifact instead of inventing counts', async t => { const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; test('large output', () => { console.log('中'.repeat(600000)); });", build: false }); const { job } = await runPlan(runnerFor(f), f.session); assert.equal(job.status, 'passed'); assert.equal(job.results[0].truncated, true); assert.equal(job.results[0].tests.total, 1); const artifact = f.store.owned('artifact', job.results[0].logArtifactId, f.session.id), text = f.store.artifactStore.read(artifact); assert.match(text, /中间日志超出保存上限/); assert.ok(artifact.bytes < 2 * 1024 * 1024); assert.match(text, /tests 1/); }); test('model-selected checks require a run-bound decision and denial creates no check job', async t => { const f = fixture(t); nodeProject(f); const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); let round = 0; provider.complete = async (_config, messages) => { const planResult = messages.filter(message => message.role === 'tool').map(message => { try { return JSON.parse(message.content); } catch { return {}; } }).find(value => value.checks && value.fingerprint); const calls = round++ === 0 ? [call('plan-for-model', 'plan_checks', {})] : round === 2 ? [call('run-for-model', 'run_checks', { plan_id: planResult.id })] : []; return { message: { role: 'assistant', content: calls.length ? '' : 'Checks were declined; task has no engineering pass.', tool_calls: calls }, usage: { prompt_tokens: 10, completion_tokens: 5 } }; }; runtime.start(f.session.id, 'Validate this engineering project'); const promise = runtime.active.get(f.session.id).promise; await until(() => f.session.approvals.some(value => value.status === 'pending'), 8000); assert.equal(f.store.core.check_job.size, 0); const approval = f.session.approvals.at(-1); assert.equal(approval.runId, f.session.runId); runtime.decide(f.session.id, approval.id, false); await promise; assert.equal(f.store.core.check_job.size, 0); assert.equal(f.session.events.find(event => event.tool === 'run_checks').phase, 'denied'); assertToolPairs(f.session.providerMessages); }); test('check APIs expose owned plans and jobs and preserve the authoritative results across restart', async t => { const f = fixture(t); nodeProject(f); f.store.close(); const app = await launchWorkbench(t, f.data); const planned = await app.api('/sessions/' + f.session.id + '/check-plans', {}), plan = planned.execution.plans.at(-1); const accepted = await app.api('/sessions/' + f.session.id + '/check-jobs', { planId: plan.id }); const jobId = accepted.execution.checks.at(-1).id; await until(async () => !['queued', 'running'].includes((await app.api('/sessions/' + f.session.id + '/check-jobs/' + jobId)).status), 12000); const finished = await app.api('/sessions/' + f.session.id + '/check-jobs/' + jobId); assert.equal(finished.status, 'passed'); const other = await app.api('/sessions', { projectId: f.project.id }); await assert.rejects(app.api('/sessions/' + other.id + '/check-jobs/' + jobId), { status: 404 }); const log = await app.api('/artifacts/' + finished.results[0].logArtifactId + '?session=' + f.session.id); assert.match(log.content, /tests 2/); const exported = await app.api('/sessions/' + f.session.id + '/export'); assert.equal(exported.execution.checks.at(-1).id, jobId); await app.close(); const resumed = await launchWorkbench(t, f.data); assert.equal((await resumed.api('/sessions/' + f.session.id)).execution.task.verification.status, 'passed'); }); test('reviewed changes continue the original task without reopening a generic completed run', async t => { const f = fixture(t); nodeProject(f); const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); let round = 0; provider.complete = async () => ({ message: { role: 'assistant', content: round++ === 0 ? '' : 'Reviewed changes need actual verification.', ...(round === 1 ? { tool_calls: [call('reviewed-proposal', 'propose_file', { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b + 0;\n', reason: 'Verify the reviewed-task continuation' })] } : {}) }, usage: { prompt_tokens: 5, completion_tokens: 5 } }); runtime.start(f.session.id, 'Keep this original engineering goal'); await runtime.active.get(f.session.id).promise; assert.equal(f.session.status, 'needs_review'); const taskId = f.session.taskId, firstRun = f.session.runId; assert.throws(() => runtime.continueReviewed(f.session.id), { status: 409 }); applyChange(f.store, f.session, f.session.changes.at(-1).id); assert.equal(f.session.status, 'completed'); assert.throws(() => runtime.resume(f.session.id), { status: 409 }); const before = JSON.stringify(f.store.publicSession(f.session)), config = f.store.config; f.store.config = { ...config, mode: 'compatible', model: '' }; assert.throws(() => runtime.continueReviewed(f.session.id), /模型名称/); assert.equal(JSON.stringify(f.store.publicSession(f.session)), before, 'rejected continuation must not append an instruction or run'); f.store.config = config; runtime.continueReviewed(f.session.id); assert.throws(() => runtime.continueReviewed(f.session.id), { status: 409 }); await runtime.active.get(f.session.id).promise; assert.equal(f.session.taskId, taskId); assert.equal(f.store.core.task.get(taskId).goal, 'Keep this original engineering goal'); assert.equal(f.store.core.run.get(f.session.runId).resumeOf, firstRun); assert.equal(f.store.core.task.get(taskId).instructions.length, 0); assert.equal(f.store.core.run.get(f.session.runId).source, 'review-action'); assert.throws(() => runtime.continueReviewed(f.session.id), { status: 409 }); assertToolPairs(f.session.providerMessages); }); test('concurrent review-continuation API requests accept one run and retain the goal after restart', async t => { const f = fixture(t); nodeProject(f); let requests = 0; const model = await compatibleFixture(t, async () => { if (requests++ === 0) return completion([call('api-review-proposal', 'propose_file', { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b + 0;\n', reason: 'A reviewable API fixture change' })]); await new Promise(resolve => setTimeout(resolve, 120)); return completion([], 'Continue the same delivery goal with the saved results.'); }); const app = await launchWorkbench(t, f.data); await app.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'review-fixture' }); await app.api('/sessions/' + f.session.id + '/messages', { content: 'Retain this delivery goal' }); let reviewed; await until(async () => { reviewed = await app.api('/sessions/' + f.session.id); return reviewed.status === 'needs_review'; }); await app.api('/sessions/' + f.session.id + '/changes/' + reviewed.changes.at(-1).id + '/apply', {}); const responses = await Promise.allSettled([app.api('/sessions/' + f.session.id + '/continue-reviewed', {}), app.api('/sessions/' + f.session.id + '/continue-reviewed', {})]); assert.equal(responses.filter(value => value.status === 'fulfilled').length, 1); assert.equal(responses.find(value => value.status === 'rejected').reason.status, 409); let delivered; await until(async () => { delivered = await app.api('/sessions/' + f.session.id); return delivered.status === 'completed'; }); assert.equal(delivered.execution.task.id, reviewed.execution.task.id); assert.equal(delivered.execution.task.goal, 'Retain this delivery goal'); assert.equal(delivered.execution.run.resumeOf, reviewed.execution.run.id); assert.equal(delivered.execution.task.instructions.filter(value => value.source === 'review-action').length, 0); assert.equal(delivered.execution.run.source, 'review-action'); await app.close(); const restarted = await launchWorkbench(t, f.data); const saved = await restarted.api('/sessions/' + f.session.id); assert.equal(saved.execution.task.id, delivered.execution.task.id); assert.equal(saved.execution.run.id, delivered.execution.run.id); assert.equal(saved.execution.task.instructions.filter(value => value.source === 'review-action').length, 0); assert.equal(saved.execution.run.source, 'review-action'); assertToolPairs((await restarted.api('/sessions/' + f.session.id + '/export')).session.providerMessages); }); test('the model reads owned failure logs without running the check again', async t => { const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' }); const { job } = await runPlan(runnerFor(f), f.session); const artifactId = job.results[0].logArtifactId; assert.match(job.results[0].logExcerpt, /AssertionError|ERR_ASSERTION/); const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll()); let round = 0; provider.complete = async () => ({ message: round++ === 0 ? { role: 'assistant', content: '', tool_calls: [call('read-check-log', 'read_artifact', { artifact_id: artifactId, offset: 0, limit: 16000 })] } : { role: 'assistant', content: 'Failure log read without replay.' }, usage: { prompt_tokens: 5, completion_tokens: 5 } }); runtime.start(f.session.id, 'Explain the saved check failure'); await runtime.active.get(f.session.id).promise; assert.equal(f.store.core.check_job.size, 1); const result = f.store.savedToolResult(f.session.id, 'read-check-log'); assert.match(result.content, /AssertionError|ERR_ASSERTION/); assert.equal(result.artifact_id, artifactId); const other = f.store.createSession(f.project.id); assert.throws(() => f.store.owned('artifact', artifactId, other.id), { status: 404 }); }); for (const phase of ['queued', 'result_committed']) test('process exit at check ' + phase + ' preserves results and never replays commands', async t => { const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('one side effect', () => fs.writeFileSync('.awb-effect', '1'));" }); f.store.close(); const worker = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/check-crash-worker.mjs'), f.data, f.session.id, phase], { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); let error = ''; worker.stderr.on('data', bytes => { error += bytes.toString(); }); const code = await new Promise(resolve => worker.once('exit', resolve)); assert.equal(code, 69, error); const recovered = new Store(f.data); f.onCleanup(() => recovered.close()); const session = recovered.session(f.session.id), job = recovered.executionSnapshot(session).checks.at(-1); assert.equal(job.status, phase === 'queued' ? 'interrupted' : 'unknown'); assert.equal(session.execution, undefined); if (phase === 'result_committed') { assert.equal(job.results.length, 1); assert.equal(job.results[0].tests.total, 1); assert.equal(fs.readFileSync(path.join(f.projectRoot, '.awb-effect'), 'utf8'), '1'); } else assert.equal(fs.existsSync(path.join(f.projectRoot, '.awb-effect')), false); assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist')), false); assert.equal(recovered.executionSnapshot(session).task.verification.status, job.status); });