import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash, randomUUID } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; // Operates only on a fresh synthetic project. The configured model remains in // the running host; bootstrap contains an API session token, never its model key. const workspace = process.cwd(); const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318'; const origin = new URL(url); assert.equal(origin.hostname, '127.0.0.1'); assert.equal(origin.protocol, 'http:'); const bootstrap = await (await fetch(origin.origin + '/api/bootstrap')).json(); assert.equal(bootstrap.version, '1.1.0'); assert.equal(bootstrap.config.mode, 'compatible'); assert.ok(bootstrap.config.model, 'Configure a model on the running Workbench first.'); const api = async (route, input) => { const response = await fetch(origin.origin + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token }, ...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(15000) }); const result = await response.json(); if (!response.ok) throw Object.assign(new Error(result.error ?? 'Workbench HTTP ' + response.status), { status: response.status }); return result; }; const caseRoot = path.resolve('.qa', 'live-delivery-' + randomUUID()); const projectRoot = path.join(caseRoot, '交付报告工程'); const allowed = ['src/normalize.mjs', 'src/summary.mjs', 'src/render.mjs', 'src/cli.mjs']; const scripts = { test: 'node --test test/*.test.mjs', build: 'node scripts/build.mjs' }; fs.mkdirSync(caseRoot, { recursive: true }); fs.cpSync(path.join(workspace, 'benchmarks/node-delivery-v1/baseline'), projectRoot, { recursive: true }); fs.writeFileSync(path.join(projectRoot, '.gitignore'), 'dist/\n'); fs.writeFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), '# Owner notes\nBaseline owner content.\n'); const git = args => execFileSync('git', ['-c', 'core.autocrlf=false', '-c', 'user.name=Agent Workbench QA', '-c', 'user.email=qa@workbench.local', '-C', projectRoot, ...args], { encoding: 'utf8', windowsHide: true }); git(['init', '--initial-branch=main']); git(['add', '.']); git(['commit', '-m', 'test: seal delivery-report acceptance baseline']); const baselineCommit = git(['rev-parse', 'HEAD']).trim(); fs.appendFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), 'Existing owner edit: ' + randomUUID() + '\n'); fs.writeFileSync(path.join(projectRoot, 'UNTRACKED-OWNER.txt'), 'Existing untracked owner file: ' + randomUUID() + '\n'); const sha = file => createHash('sha256').update(fs.readFileSync(path.join(projectRoot, file))).digest('hex'); const protectedPaths = ['README.md', 'package.json', 'test/delivery.test.mjs', 'scripts/build.mjs', 'fixtures/tasks.json', 'fixtures/invalid.json', '.gitignore', 'OWNER-NOTES.md', 'UNTRACKED-OWNER.txt']; const protectedHashes = Object.fromEntries(protectedPaths.map(file => [file, sha(file)])); const beforeSources = Object.fromEntries(allowed.map(file => [file, sha(file)])); const ownerDiff = git(['diff', '--', 'OWNER-NOTES.md']); const assertProtected = () => { for (const [file, hash] of Object.entries(protectedHashes)) assert.equal(sha(file), hash, 'Protected input changed: ' + file); assert.equal(git(['diff', '--', 'OWNER-NOTES.md']), ownerDiff, 'Existing owner edits changed.'); }; const project = await api('/projects', { root: projectRoot }); const session = await api('/sessions', { projectId: project.id }); const started = Date.now(), applied = [], decisions = [], reviewLimit = 4; let reviewedRounds = 0, state, succeeded = false, lastProgress = '', originalTaskId; const saveEvidence = result => { fs.writeFileSync(path.join(caseRoot, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync(path.join(workspace, '.qa/live-delivery-validation.json'), JSON.stringify(result, null, 2)); }; try { await api('/sessions/' + session.id + '/messages', { content: '完成当前 Delivery Report 工程,严格实现 README.md 的七项要求。先用 inspect_project 识别工程,' + '再用 plan_checks 选择 test/build,并用 run_checks 实际运行封存输入,取得失败的测试和日志。' + '依据实际失败读取有关文件,使用 read_artifact 进一步核对失败日志。只能用 propose_file 修改 src/normalize.mjs、src/summary.mjs、src/render.mjs、src/cli.mjs;' + '测试、fixtures、构建脚本、package.json、README 和 OWNER 文件必须保留。四模块都需修复,不能修改验收来通过。' + '本工程没有第三方依赖,不用安装。不要使用 run_command、不要自己提交Git;本轮终端仅通过已识别的npm检查计划运行。' + '提案等待审查应用,随后继续同一目标并实际重新运行 test/build,依据日志修复直至通过;最后用中文报告实际测试数量、构建结果和改动。' }); while (Date.now() - started < 720000) { state = await api('/sessions/' + session.id); originalTaskId ??= state.execution.task.id; assert.equal(state.execution.task.id, originalTaskId, 'Engineering delivery must retain the original Task.'); assertProtected(); const progress = state.status + '|' + state.events.length + '|' + state.execution.checks.map(job => job.status).join(','); if (progress !== lastProgress) { console.log(JSON.stringify({ status: state.status, elapsedSeconds: Math.round((Date.now() - started) / 1000), tools: state.events.filter(event => event.kind === 'tool').slice(-5).map(event => ({ tool: event.tool, phase: event.phase })), checks: state.execution.checks.map(job => ({ status: job.status, results: job.results.map(result => ({ name: result.name, status: result.status, tests: result.tests })) })), appliedFiles: applied.map(item => item.path) })); lastProgress = progress; } for (const proposal of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(proposal.path), 'Out-of-scope proposal: ' + proposal.path); for (const approval of state.approvals.filter(item => item.status === 'pending')) { const plan = state.execution.plans?.find(item => item.id === approval.checkPlanId); const allow = !!plan && plan.taskId === originalTaskId && plan.checks.length > 0 && plan.checks.length <= 2 && plan.checks.every(check => scripts[check.name] === check.script && check.command === 'npm run ' + check.name && check.lifecycle.length === 0) && approval.command === plan.checks.map(check => check.command).join('\n'); decisions.push({ approvalId: approval.id, planId: plan?.id ?? null, command: approval.command, allow }); await api('/sessions/' + session.id + '/approvals/' + approval.id, { allow }); assert.ok(allow, 'Model requested a command outside the sealed test/build contract.'); } if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await new Promise(resolve => setTimeout(resolve, 1000)); continue; } if (state.status === 'failed') throw new Error(state.failure?.message ?? 'The real model task failed.'); if (state.status === 'paused') { assert.equal(state.execution.run.stopReason, 'step_limit', 'Budget/cancellation must be inspected before continuing.'); assert.ok(state.execution.runs.length < 5, 'Bounded continuation count exceeded.'); await api('/sessions/' + session.id + '/resume', {}); continue; } const pending = state.changes.filter(change => change.status === 'pending'); if (pending.length) { assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(result => result.tests?.failed > 0)), 'Capture a real baseline test failure before applying repairs.'); assert.ok(++reviewedRounds <= reviewLimit, 'Too many repair/review rounds.'); for (const proposal of pending) { assert.equal(sha(proposal.path), proposal.baseSha256, 'Proposal baseline changed.'); assert.notEqual(proposal.content.trim(), '', 'Empty source proposal.'); const beforeApply = await api('/sessions/' + session.id + '/changes/' + proposal.id + '/apply', {}); assert.equal(sha(proposal.path), proposal.sha256, 'Applied bytes do not match the reviewed proposal.'); applied.push({ id: proposal.id, path: proposal.path, sha256: proposal.sha256, taskId: originalTaskId }); assert.ok(beforeApply.session ?? beforeApply, 'No apply response.'); assertProtected(); } await api('/sessions/' + session.id + '/continue-reviewed', {}); continue; } assert.equal(state.status, 'completed'); break; } assert.equal(state?.status, 'completed', 'Engineering delivery timed out.'); assert.equal(state.execution.task.verification.status, 'passed', 'No authoritative passing engineering check.'); const passedJob = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passedJob); assert.ok(passedJob.results.some(result => result.name === 'test' && result.exitCode === 0 && result.tests?.total === 14 && result.tests.passed === 14)); assert.ok(passedJob.results.some(result => result.name === 'build' && result.exitCode === 0 && result.status === 'passed')); assert.deepEqual([...new Set(applied.map(item => item.path))].sort(), [...allowed].sort()); for (const file of allowed) assert.notEqual(sha(file), beforeSources[file], 'Module was not repaired: ' + file); assertProtected(); const built = spawnSync(process.execPath, [path.join(projectRoot, 'dist/cli.mjs'), path.join(projectRoot, 'fixtures/tasks.json'), '--json'], { cwd: projectRoot, encoding: 'utf8', windowsHide: true }); assert.equal(built.status, 0); assert.equal(built.stderr, ''); assert.deepEqual(JSON.parse(built.stdout), { total: 4, counts: { todo: 0, doing: 1, done: 2, blocked: 1 }, completedPercent: 50, remainingMinutes: 50 }); const diff = git(['diff', '--', ...allowed]); assert.ok(diff.length > 0); fs.writeFileSync(path.join(caseRoot, 'reviewed-changes.diff'), diff); git(['add', '--', ...allowed]); git(['commit', '--only', '-m', 'feat: repair delivery report with 14 verified acceptance cases', '--', ...allowed]); const deliveredCommit = git(['rev-parse', 'HEAD']).trim(); assert.notEqual(deliveredCommit, baselineCommit); assertProtected(); assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), [...allowed].sort()); const exported = await api('/sessions/' + session.id + '/export'); fs.writeFileSync(path.join(caseRoot, 'session.json'), JSON.stringify(exported, null, 2)); for (const job of state.execution.checks) for (const result of job.results) { if (!result.logArtifactId) continue; let offset = 0, text = '', log; do { log = await api('/artifacts/' + result.logArtifactId + '?session=' + session.id + '&offset=' + offset + '&limit=16000'); text += log.content; offset += log.content.length; } while (log.truncated && log.content.length); fs.writeFileSync(path.join(caseRoot, job.id + '-' + result.name + '.log'), text); } const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), model: bootstrap.config.model, projectRoot, projectId: project.id, sessionId: session.id, taskId: originalTaskId, baselineCommit, deliveredCommit, elapsedSeconds: Math.round((Date.now() - started) / 1000), runs: state.execution.runs.length, reviewRounds: reviewedRounds, applied, protectedHashes, ownerEditsPreserved: true, buildOutput: JSON.parse(built.stdout), usage: state.execution.task.usage, checks: state.execution.checks, decisions, diffPath: path.join(caseRoot, 'reviewed-changes.diff'), sessionPath: path.join(caseRoot, 'session.json'), credentialsHandling: 'Calls use the running host model configuration; no model key was read, copied or saved.', intervention: 'Verifier approved only sealed npm scripts, checked four source proposals and applied them; implementation came from the configured model. The verifier saved a source-only local Git commit.', scope: 'One synthetic Node engineering sample; does not establish M2 or M6 delivery success rates.' }; saveEvidence(result); succeeded = true; console.log(JSON.stringify({ success: true, model: result.model, testsPassed: 14, buildPassed: true, deliveredCommit, elapsedSeconds: result.elapsedSeconds, reviewRounds: reviewedRounds, projectRoot, sessionId: session.id }, null, 2)); } catch (error) { saveEvidence({ success: false, version: bootstrap.version, testedAt: new Date().toISOString(), model: bootstrap.config.model, projectRoot, projectId: project.id, sessionId: session.id, taskId: originalTaskId, error: error.message, elapsedSeconds: Math.round((Date.now() - started) / 1000), applied, decisions, checks: state?.execution?.checks ?? [] }); throw error; } finally { if (!succeeded) await api('/sessions/' + session.id + '/stop', {}).catch(() => {}); }