Files
Agent-Workbench/scripts/verify-live-delivery.mjs
T

186 lines
16 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { execFileSync, spawnSync } from 'node:child_process';
import { APP_VERSION } from '../src/server/diagnostics.mjs';
// Operates only on a fresh synthetic project. The configured model remains in
// the running host; bootstrap contains an API session token, never its model key.
const workspace = process.cwd();
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318';
const origin = new URL(url);
assert.equal(origin.hostname, '127.0.0.1'); assert.equal(origin.protocol, 'http:');
const bootstrap = await (await fetch(origin.origin + '/api/bootstrap')).json();
assert.equal(bootstrap.version, APP_VERSION); assert.equal(bootstrap.config.mode, 'compatible');
assert.ok(bootstrap.config.model, 'Configure a model on the running Workbench first.');
const api = async (route, input) => {
const response = await fetch(origin.origin + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token },
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(15000) });
const result = await response.json();
if (!response.ok) throw Object.assign(new Error(result.error ?? 'Workbench HTTP ' + response.status), { status: response.status });
return result;
};
const caseRoot = path.resolve('.qa', 'live-delivery-' + randomUUID());
const projectRoot = path.join(caseRoot, '交付报告工程');
const allowed = ['src/normalize.mjs', 'src/summary.mjs', 'src/render.mjs', 'src/cli.mjs'];
const scripts = { test: 'node --test test/*.test.mjs', build: 'node scripts/build.mjs' };
fs.mkdirSync(caseRoot, { recursive: true });
fs.cpSync(path.join(workspace, 'benchmarks/node-delivery-v1/baseline'), projectRoot, { recursive: true });
fs.writeFileSync(path.join(projectRoot, '.gitignore'), 'dist/\n');
fs.writeFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), '# Owner notes\nBaseline owner content.\n');
fs.writeFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Baseline staged owner content.\n');
const git = args => execFileSync('git', ['-c', 'core.autocrlf=false', '-c', 'user.name=Agent Workbench QA', '-c', 'user.email=qa@workbench.local', '-C', projectRoot, ...args], { encoding: 'utf8', windowsHide: true });
git(['init', '--initial-branch=main']); git(['add', '.']); git(['commit', '-m', 'test: seal delivery-report acceptance baseline']);
const baselineCommit = git(['rev-parse', 'HEAD']).trim();
fs.appendFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), 'Existing owner edit: ' + randomUUID() + '\n');
fs.writeFileSync(path.join(projectRoot, 'UNTRACKED-OWNER.txt'), 'Existing untracked owner file: ' + randomUUID() + '\n');
fs.appendFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Existing staged owner edit: ' + randomUUID() + '\n');
git(['add', '--', 'OWNER-STAGED.txt']);
const sha = file => createHash('sha256').update(fs.readFileSync(path.join(projectRoot, file))).digest('hex');
const protectedPaths = ['README.md', 'package.json', 'test/delivery.test.mjs', 'scripts/build.mjs', 'fixtures/tasks.json', 'fixtures/invalid.json', '.gitignore', 'OWNER-NOTES.md', 'OWNER-STAGED.txt', 'UNTRACKED-OWNER.txt'];
const protectedHashes = Object.fromEntries(protectedPaths.map(file => [file, sha(file)]));
const beforeSources = Object.fromEntries(allowed.map(file => [file, sha(file)]));
const ownerDiff = git(['diff', '--', 'OWNER-NOTES.md']);
const ownerStagedDiff = git(['diff', '--cached', '--', 'OWNER-STAGED.txt']);
const assertProtected = () => {
for (const [file, hash] of Object.entries(protectedHashes)) assert.equal(sha(file), hash, 'Protected input changed: ' + file);
assert.equal(git(['diff', '--', 'OWNER-NOTES.md']), ownerDiff, 'Existing owner edits changed.');
assert.equal(git(['diff', '--cached', '--', 'OWNER-STAGED.txt']), ownerStagedDiff, 'Existing owner stage changed.');
};
const project = await api('/projects', { root: projectRoot });
const session = await api('/sessions', { projectId: project.id });
const started = Date.now(), applied = [], decisions = [], reviewLimit = 4;
let reviewedRounds = 0, state, succeeded = false, lastProgress = '', originalTaskId;
const saveEvidence = result => {
fs.writeFileSync(path.join(caseRoot, 'result.json'), JSON.stringify(result, null, 2));
fs.writeFileSync(path.join(workspace, '.qa/live-delivery-validation.json'), JSON.stringify(result, null, 2));
};
try {
await api('/sessions/' + session.id + '/messages', { content:
'完成当前 Delivery Report 工程,严格实现 README.md 的七项要求。先用 inspect_project 识别工程,' +
'再用 plan_checks 选择 test/build,并用 run_checks 实际运行封存输入,取得失败的测试和日志。' +
'依据实际失败读取有关文件,使用 read_artifact 进一步核对失败日志。只能用 propose_file 修改 src/normalize.mjs、src/summary.mjs、src/render.mjs、src/cli.mjs;' +
'测试、fixtures、构建脚本、package.json、README 和 OWNER 文件必须保留。四模块都需修复,不能修改验收来通过。' +
'本工程没有第三方依赖,不用安装。不要使用 run_command、不要自己提交Git;本轮终端仅通过已识别的npm检查计划运行。' +
'提案等待审查应用,随后继续同一目标并实际重新运行 test/build,依据日志修复直至通过。检查通过后用 plan_delivery 准备本Task的四文件本地Git预览,等待用户独立确认;最后用中文报告实际测试数量、构建结果和改动。' });
while (Date.now() - started < 720000) {
state = await api('/sessions/' + session.id); originalTaskId ??= state.execution.task.id;
assert.equal(state.execution.task.id, originalTaskId, 'Engineering delivery must retain the original Task.');
assertProtected();
const progress = state.status + '|' + state.events.length + '|' + state.execution.checks.map(job => job.status).join(',');
if (progress !== lastProgress) {
console.log(JSON.stringify({ status: state.status, elapsedSeconds: Math.round((Date.now() - started) / 1000),
tools: state.events.filter(event => event.kind === 'tool').slice(-5).map(event => ({ tool: event.tool, phase: event.phase })),
checks: state.execution.checks.map(job => ({ status: job.status, results: job.results.map(result => ({ name: result.name, status: result.status, tests: result.tests })) })),
appliedFiles: applied.map(item => item.path) }));
lastProgress = progress;
}
for (const proposal of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(proposal.path), 'Out-of-scope proposal: ' + proposal.path);
for (const approval of state.approvals.filter(item => item.status === 'pending')) {
const plan = state.execution.plans?.find(item => item.id === approval.checkPlanId);
const allow = !!plan && plan.taskId === originalTaskId && plan.checks.length > 0 && plan.checks.length <= 2
&& plan.checks.every(check => scripts[check.name] === check.script && check.command === 'npm run ' + check.name && check.lifecycle.length === 0)
&& approval.command === plan.checks.map(check => check.command).join('\n');
decisions.push({ approvalId: approval.id, planId: plan?.id ?? null, command: approval.command, allow });
await api('/sessions/' + session.id + '/approvals/' + approval.id, { allow });
assert.ok(allow, 'Model requested a command outside the sealed test/build contract.');
}
if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await new Promise(resolve => setTimeout(resolve, 1000)); continue; }
if (state.status === 'failed') throw new Error(state.failure?.message ?? 'The real model task failed.');
if (state.status === 'paused') {
assert.equal(state.execution.run.stopReason, 'step_limit', 'Budget/cancellation must be inspected before continuing.');
assert.ok(state.execution.runs.length < 5, 'Bounded continuation count exceeded.');
await api('/sessions/' + session.id + '/resume', {}); continue;
}
const pending = state.changes.filter(change => change.status === 'pending');
if (pending.length) {
assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(result => result.tests?.failed > 0)), 'Capture a real baseline test failure before applying repairs.');
assert.ok(++reviewedRounds <= reviewLimit, 'Too many repair/review rounds.');
for (const proposal of pending) {
assert.equal(sha(proposal.path), proposal.baseSha256, 'Proposal baseline changed.');
assert.notEqual(proposal.content.trim(), '', 'Empty source proposal.');
const beforeApply = await api('/sessions/' + session.id + '/changes/' + proposal.id + '/apply', {});
assert.equal(sha(proposal.path), proposal.sha256, 'Applied bytes do not match the reviewed proposal.');
applied.push({ id: proposal.id, path: proposal.path, sha256: proposal.sha256, taskId: originalTaskId });
assert.ok(beforeApply.session ?? beforeApply, 'No apply response.'); assertProtected();
}
await api('/sessions/' + session.id + '/continue-reviewed', {}); continue;
}
assert.equal(state.status, 'completed'); break;
}
assert.equal(state?.status, 'completed', 'Engineering delivery timed out.');
assert.equal(state.execution.task.verification.status, 'passed', 'No authoritative passing engineering check.');
const passedJob = state.execution.checks.filter(job => job.status === 'passed').at(-1);
assert.ok(passedJob); assert.ok(passedJob.results.some(result => result.name === 'test' && result.exitCode === 0 && result.tests?.total === 14 && result.tests.passed === 14));
assert.ok(passedJob.results.some(result => result.name === 'build' && result.exitCode === 0 && result.status === 'passed'));
assert.deepEqual([...new Set(applied.map(item => item.path))].sort(), [...allowed].sort());
for (const file of allowed) assert.notEqual(sha(file), beforeSources[file], 'Module was not repaired: ' + file);
assertProtected();
const built = spawnSync(process.execPath, [path.join(projectRoot, 'dist/cli.mjs'), path.join(projectRoot, 'fixtures/tasks.json'), '--json'], { cwd: projectRoot, encoding: 'utf8', windowsHide: true });
assert.equal(built.status, 0); assert.equal(built.stderr, '');
assert.deepEqual(JSON.parse(built.stdout), { total: 4, counts: { todo: 0, doing: 1, done: 2, blocked: 1 }, completedPercent: 50, remainingMinutes: 50 });
const diff = git(['diff', '--', ...allowed]); assert.ok(diff.length > 0); fs.writeFileSync(path.join(caseRoot, 'reviewed-changes.diff'), diff);
let deliveryPlan = state.execution.deliveryPlans?.filter(plan => plan.status === 'planned' && plan.taskId === originalTaskId && plan.runId === state.runId).at(-1);
if (!deliveryPlan) {
state = await api('/sessions/' + session.id + '/delivery-plans', { message: 'feat: repair delivery report with 14 verified acceptance cases' });
deliveryPlan = state.execution.deliveryPlans.at(-1);
}
assert.deepEqual(deliveryPlan.paths.slice().sort(), allowed.slice().sort());
assert.equal(deliveryPlan.checkJobId, passedJob.id); assert.equal(deliveryPlan.sourceFingerprint, passedJob.sourceFingerprint);
assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineCommit); assertProtected();
const queued = await api('/sessions/' + session.id + '/delivery-jobs', { planId: deliveryPlan.id });
const deliveryJobId = queued.execution.deliveries.at(-1).id;
let deliveryJob;
for (let attempt = 0; attempt < 120; attempt++) {
deliveryJob = await api('/sessions/' + session.id + '/delivery-jobs/' + deliveryJobId);
if (!['queued', 'preparing', 'prepared', 'publishing', 'committed', 'reporting'].includes(deliveryJob.status)) break;
await new Promise(resolve => setTimeout(resolve, 500));
}
assert.equal(deliveryJob.status, 'completed', deliveryJob.summary); assert.ok(deliveryJob.reportArtifactId); assert.ok(deliveryJob.reportMarkdownArtifactId);
const deliveredCommit = git(['rev-parse', 'HEAD']).trim(); assert.equal(deliveredCommit, deliveryJob.commit);
assert.notEqual(deliveredCommit, baselineCommit); assertProtected();
assert.equal(git(['rev-list', '--count', 'HEAD']).trim(), '2');
const readArtifact = async id => {
let offset = 0, content = '', artifact;
do { artifact = await api('/artifacts/' + id + '?session=' + session.id + '&offset=' + offset + '&limit=16000'); content += artifact.content; offset += artifact.content.length; } while (artifact.truncated && artifact.content.length);
assert.equal(createHash('sha256').update(content).digest('hex'), artifact.sha256);
return content;
};
const deliveryJson = await readArtifact(deliveryJob.reportArtifactId), deliveryMarkdown = await readArtifact(deliveryJob.reportMarkdownArtifactId);
const report = JSON.parse(deliveryJson); assert.equal(report.status, 'verified_committed'); assert.equal(report.task.id, originalTaskId);
assert.equal(report.verifiedCheckJobId, passedJob.id); assert.equal(report.deliveries.at(-1).commit, deliveredCommit);
assert.equal(report.task.requirementsAcceptance, 'not_recorded'); assert.match(deliveryMarkdown, /14\/14/);
fs.writeFileSync(path.join(caseRoot, 'delivery-report.json'), deliveryJson); fs.writeFileSync(path.join(caseRoot, 'delivery-report.md'), deliveryMarkdown);
state = await api('/sessions/' + session.id);
assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), [...allowed].sort());
const exported = await api('/sessions/' + session.id + '/export');
fs.writeFileSync(path.join(caseRoot, 'session.json'), JSON.stringify(exported, null, 2));
for (const job of state.execution.checks) for (const result of job.results) {
if (!result.logArtifactId) continue;
let offset = 0, text = '', log;
do { log = await api('/artifacts/' + result.logArtifactId + '?session=' + session.id + '&offset=' + offset + '&limit=16000'); text += log.content; offset += log.content.length; } while (log.truncated && log.content.length);
fs.writeFileSync(path.join(caseRoot, job.id + '-' + result.name + '.log'), text);
}
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), model: bootstrap.config.model,
projectRoot, projectId: project.id, sessionId: session.id, taskId: originalTaskId, baselineCommit, deliveredCommit, deliveryPlan, deliveryJob,
elapsedSeconds: Math.round((Date.now() - started) / 1000), runs: state.execution.runs.length, reviewRounds: reviewedRounds, applied,
protectedHashes, ownerEditsPreserved: true, ownerStagePreserved: true, reportStatus: report.status, reportPath: path.join(caseRoot, 'delivery-report.md'), buildOutput: JSON.parse(built.stdout), usage: state.execution.task.usage,
checks: state.execution.checks, decisions, diffPath: path.join(caseRoot, 'reviewed-changes.diff'), sessionPath: path.join(caseRoot, 'session.json'),
credentialsHandling: 'Calls use the running host model configuration; no model key was read, copied or saved.',
intervention: 'Verifier approved only sealed npm scripts, reviewed and applied four source proposals, and confirmed an owned Git preview through the Workbench API. Implementation came from the configured model; the Workbench generated and verified the commit and evidence reports.',
scope: 'One synthetic Node engineering sample; does not establish M2 or M6 delivery success rates.' };
saveEvidence(result); succeeded = true;
console.log(JSON.stringify({ success: true, model: result.model, testsPassed: 14, buildPassed: true, deliveredCommit, elapsedSeconds: result.elapsedSeconds, reviewRounds: reviewedRounds, projectRoot, sessionId: session.id }, null, 2));
} catch (error) {
saveEvidence({ success: false, version: bootstrap.version, testedAt: new Date().toISOString(), model: bootstrap.config.model, projectRoot,
projectId: project.id, sessionId: session.id, taskId: originalTaskId, error: error.message, elapsedSeconds: Math.round((Date.now() - started) / 1000), applied, decisions,
checks: state?.execution?.checks ?? [] });
throw error;
} finally {
if (!succeeded) await api('/sessions/' + session.id + '/stop', {}).catch(() => {});
}