Files
Agent-Workbench/scripts/verify-live-engineering.mjs

179 lines
18 KiB
JavaScript

import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { createHash, randomUUID } from 'node:crypto';
import { Store } from '../src/server/store.mjs';
import { APP_VERSION } from '../src/server/diagnostics.mjs';
import { gitEnvironment } from '../src/core/git-process.mjs';
import { createPythonDeliveryProject, pythonProtected } from '../tests/fixtures/python-project.mjs';
import { deliveryGit, writeDeliveryFile } from '../tests/fixtures/delivery-project.mjs';
import { captureWorkspaceSource, assertWorkspaceSource } from '../tests/fixtures/workspace-project.mjs';
import { verifyLiveReactClient } from './live-react-client.mjs';
import { observedTaskModels } from './live-model-observation.mjs';
if (process.argv.includes('--help')) {
console.log('node scripts/verify-live-engineering.mjs --case react|python [--url http://127.0.0.1:4318] — fresh owned Git source/workspace, configured model and bounded review/check/delivery acceptance'); process.exit(0);
}
const index = process.argv.indexOf('--case'), kind = index >= 0 ? process.argv[index + 1] : null;
assert.ok(['react', 'python'].includes(kind), '--case react or --case python is required');
const urlIndex = process.argv.indexOf('--url'), origin = new URL(urlIndex >= 0 ? process.argv[urlIndex + 1] : 'http://127.0.0.1:4318');
assert.equal(origin.hostname, '127.0.0.1'); assert.equal(origin.protocol, 'http:');
const bootstrap = await (await fetch(origin.origin + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
assert.equal(bootstrap.version, APP_VERSION); assert.equal(bootstrap.config.mode, 'compatible'); assert.ok(bootstrap.config.model);
const api = async (route, input) => {
const response = await fetch(origin.origin + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
const value = await response.json(); if (!response.ok) throw Object.assign(new Error(value.error ?? 'Workbench HTTP ' + response.status), { status: response.status }); return value;
};
const root = path.resolve('.qa', 'live-' + kind + '-' + randomUUID()), sourceRoot = path.join(root, kind + '-source');
fs.mkdirSync(sourceRoot, { recursive: true }); fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# Owned ' + kind + ' engineering acceptance\n');
const sourceStore = new Store(path.join(root, 'fixture-data')), sourceProject = sourceStore.addProject(sourceRoot), fixtureSession = sourceStore.createSession(sourceProject.id);
const sourceF = { projectRoot: sourceRoot, store: sourceStore, session: fixtureSession };
const allowed = kind === 'react' ? ['src/query.ts', 'src/summary.ts', 'src/transition.ts', 'src/App.tsx']
: ['src/workflow/normalize.py', 'src/workflow/summary.py', 'src/workflow/render.py'];
const checkNames = kind === 'react' ? ['test', 'typecheck', 'build'] : ['python:test', 'python:compile'];
try {
if (kind === 'python') createPythonDeliveryProject(sourceF);
else {
fs.cpSync(path.resolve('benchmarks/react-delivery-v1/baseline'), sourceRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
writeDeliveryFile(sourceF, '.gitignore', 'node_modules/\ndist/\n.env\n.awb-*\n');
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(sourceF, name, 'Owner baseline bytes\n');
deliveryGit(sourceF, ['init', '--initial-branch=main']); deliveryGit(sourceF, ['config', 'core.autocrlf', 'false']);
deliveryGit(sourceF, ['config', 'user.name', 'Workbench Engineering QA']); deliveryGit(sourceF, ['config', 'user.email', 'qa@workbench.invalid']);
deliveryGit(sourceF, ['add', '--', '.']); deliveryGit(sourceF, ['commit', '--no-gpg-sign', '-m', 'Seal four-file React failure for actual-model acceptance']);
writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner staged change\n'); deliveryGit(sourceF, ['add', '--', 'owner-staged.txt']);
writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner working copy after staging\n'); writeDeliveryFile(sourceF, 'owner-unstaged.txt', 'Owner unstaged change\n');
writeDeliveryFile(sourceF, 'owner-untracked.txt', 'Owner untracked change\n');
}
} finally { sourceStore.close(); }
const originalSource = captureWorkspaceSource(sourceF), protectedNames = kind === 'react'
? ['README.md', 'package.json', 'package-lock.json', 'tsconfig.json', 'vite.config.mjs', 'index.html', 'test/domain.test.mjs', 'src/types.ts', 'src/main.tsx', 'src/styles.css']
: ['README.md', ...pythonProtected, 'package.json', 'test/add.test.mjs', 'scripts/build.mjs', 'src/add.mjs'];
const parentProject = await api('/projects', { root: sourceRoot }), parentSession = await api('/sessions', { projectId: parentProject.id });
const parentRoute = '/sessions/' + parentSession.id;
let state, workspace, childProject, childSession, route, taskId, result, succeeded = false, completedDelivery = null, lastProgress = '';
const decisions = [], applications = [], continuations = [], startedAt = Date.now(), maxModelResponses = 30, maxReportedTokens = 180000, maxDurationMs = 360000;
const sha = value => createHash('sha256').update(value).digest('hex');
const wait = async (predicate, timeout = 30000) => {
const deadline = Date.now() + timeout; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Acceptance wait timed out'); await new Promise(resolve => setTimeout(resolve, 250)); }
};
let protectedBytes, childF, ownerStage;
const assertProtected = () => {
assertWorkspaceSource(assert, sourceF, originalSource, workspace.branch);
for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(childProject.root, name)), bytes, 'Protected input changed: ' + name);
assert.equal(deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']), ownerStage);
};
const readArtifact = async id => {
let text = '', offset = 0, page;
do { page = await api('/artifacts/' + id + '?session=' + childSession.id + '&offset=' + offset + '&limit=16000');
assert.ok(page.content.length || !page.truncated); text += page.content; offset += page.content.length; } while (page.truncated);
assert.equal(sha(text), page.sha256); return text;
};
try {
const prepared = await api(parentRoute + '/workspaces', { label: kind + ' actual engineering' }); workspace = prepared;
await api(parentRoute + '/workspaces/' + workspace.id + '/create', { fingerprint: workspace.fingerprint });
await wait(async () => { workspace = await api(parentRoute + '/workspaces/' + workspace.id); return workspace.status === 'ready'; }, 30000);
const opened = await api(parentRoute + '/workspaces/' + workspace.id + '/open', {}); childProject = opened.project; childSession = opened.session; route = '/sessions/' + childSession.id;
childF = { projectRoot: childProject.root };
if (kind === 'react') {
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
const installed = execFileSync(process.execPath, [npmCli, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: childProject.root, env: gitEnvironment(), windowsHide: true, encoding: 'utf8' });
fs.writeFileSync(path.join(root, 'offline-install.log'), installed);
}
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(childF, name, 'Child owner baseline\n');
deliveryGit(childF, ['add', '--', 'owner-staged.txt']); writeDeliveryFile(childF, 'owner-staged.txt', 'Child owner working copy after stage\n');
writeDeliveryFile(childF, 'owner-untracked.txt', 'Child owner untracked change\n');
protectedBytes = Object.fromEntries([...protectedNames, 'owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(childProject.root, name))]));
ownerStage = deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']); assertProtected();
const requirements = kind === 'react'
? '修复交付看板的query、summary、transition和App四个模块。状态筛选与trim后忽略大小写的搜索取交集;统计针对全部任务,完成数量及待办工时准确;toggle只切换指定id并保持其他对象和原数组,App绑定筛选/搜索和函数式状态更新,切换一条任务后界面与统计同步。必须通过9项测试、类型检查、构建,以及真正浏览器的筛选/搜索/单任务切换。'
: '修复normalize、summary和render三个Python模块:normalize把数字或数字字符串转换为数值并把负数归零;summary返回count和数值合计total,空列表为0;render返回“共 N 项 · 合计 X”,整数合计没有多余小数。必须通过六项unittest及源码编译。';
const goal = requirements + '先用inspect_project核对工程与工作区,再用plan_checks明确选择' + JSON.stringify(checkNames)
+ '并用run_checks实际执行,保留基线失败与日志。根据真实失败读取有关源码;只能修改' + JSON.stringify(allowed)
+ ',测试、配置、README、类型声明、样式、OWNER文件和已有暂存保持。不能改测试来通过,不使用run_command、不能直接提交Git。'
+ '用propose_patch或propose_file准备全部允许模块的提案,再用prepare_change_set一次封存完整三/四文件批次,等待用户审查应用;未审查的提案不是已写入。应用后继续同一Task,用相同检查范围重新执行并根据日志修复。全部检查通过后用plan_delivery准备这些文件的本地Git预览,等待确认;停止继续查询或请求模型。最后以实际结果说明测试、构建和交付。';
await api(route + '/messages', { content: goal });
while (Date.now() - startedAt < maxDurationMs) {
state = await api(route); taskId ??= state.execution.task.id; assert.equal(state.execution.task.id, taskId); assertProtected();
if (state.execution.task.usage.requests > maxModelResponses || state.execution.task.usage.totalTokens > maxReportedTokens) throw new Error('Actual-model acceptance reached its bounded response/token scope');
const progress = state.status + '|' + state.events.length + '|' + state.execution.checks.map(job => job.status).join(',');
if (progress !== lastProgress) { console.log(JSON.stringify({ kind, status: state.status, elapsedSeconds: Math.round((Date.now() - startedAt) / 1000),
responses: state.execution.task.usage.requests, reportedTokens: state.execution.task.usage.totalTokens,
latestTools: state.events.filter(event => event.kind === 'tool').slice(-3).map(event => ({ tool: event.tool, phase: event.phase })) })); lastProgress = progress; }
for (const proposal of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(proposal.path), 'Out-of-scope model proposal: ' + proposal.path);
for (const approval of state.approvals.filter(item => item.status === 'pending')) {
const plan = state.execution.plans.find(value => value.id === approval.checkPlanId);
const allow = !!plan && plan.taskId === taskId && plan.checks.length > 0 && plan.checks.every(check => checkNames.includes(check.name) && !check.lifecycle?.length)
&& approval.cwd === childProject.root && approval.command === plan.checks.map(check => check.command).join('\n');
decisions.push({ approvalId: approval.id, checkPlanId: plan?.id, command: approval.command, allow });
await api(route + '/approvals/' + approval.id, { allow }); assert.ok(allow, 'Requested command does not match the owned engineering contract');
}
if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await new Promise(resolve => setTimeout(resolve, 750)); continue; }
if (state.status === 'failed') throw new Error(state.failure?.message ?? 'Actual-model Task failed');
const deliveryPlan = state.execution.deliveryPlans?.findLast(plan => plan.status === 'planned' && plan.taskId === taskId);
if (deliveryPlan) {
assert.deepEqual(deliveryPlan.paths.slice().sort(), allowed.slice().sort()); const requestsBefore = state.execution.task.usage.requests;
assert.ok(state.events.some(event => event.tool === 'plan_delivery' && event.phase === 'completed'), 'The model must prepare the Git review');
await api(route + '/delivery-jobs', { planId: deliveryPlan.id });
await wait(async () => { state = await api(route); completedDelivery = state.execution.deliveries.at(-1); return completedDelivery?.reportArtifactId && completedDelivery?.reportMarkdownArtifactId; }, 30000);
assert.equal(completedDelivery.status, 'completed'); assert.equal(state.execution.task.usage.requests, requestsBefore); break;
}
const pending = state.changes.filter(change => change.status === 'pending');
const batch = state.execution.changeSets?.findLast(set => set.taskId === taskId && set.status === 'prepared');
if (state.status === 'paused' && !batch) {
assert.equal(state.execution.run.stopReason, 'step_limit', 'Other stopped outcomes are recorded, not automatically replayed');
assert.ok(continuations.length < 3); continuations.push({ runId: state.runId, reason: 'step_limit', taskId }); await api(route + '/resume', {}); continue;
}
if (pending.length) {
assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(value => value.tests?.failed > 0)), 'Record actual initial test failure before repairs');
assert.ok(batch && batch.source === 'agent', 'The model must seal the reviewed batch');
assert.deepEqual(batch.files.map(file => file.changeId).sort(), pending.map(change => change.id).sort());
if (!applications.length) assert.deepEqual(batch.files.map(file => file.path).sort(), allowed.slice().sort());
assert.ok(applications.length < 3); await api(route + '/change-sets/' + batch.id + '/apply', { fingerprint: batch.fingerprint });
await wait(async () => (await api(route + '/change-sets/' + batch.id)).status === 'completed');
for (const proposal of pending) assert.equal(sha(fs.readFileSync(path.join(childProject.root, proposal.path))), proposal.sha256);
applications.push({ id: batch.id, fingerprint: batch.fingerprint, paths: batch.files.map(file => file.path), source: batch.source }); assertProtected();
await api(route + '/continue-reviewed', {}); continue;
}
throw new Error('The model stopped before the complete check/review/Git flow: ' + state.status + '/' + state.execution.run.stopReason);
}
assert.ok(completedDelivery, 'Actual-model engineering acceptance timed out'); assert.equal(state.status, 'completed');
const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passing);
assert.deepEqual(passing.results.map(value => value.name).sort(), checkNames.slice().sort());
const tests = passing.results.find(value => value.name === (kind === 'react' ? 'test' : 'python:test')).tests;
assert.equal(tests.total, kind === 'react' ? 9 : 6); assert.equal(tests.passed, tests.total); assert.equal(tests.failed, 0);
assert.deepEqual(deliveryGit(childF, ['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), allowed.slice().sort()); assertProtected();
const reportText = await readArtifact(completedDelivery.reportArtifactId), markdown = await readArtifact(completedDelivery.reportMarkdownArtifactId), report = JSON.parse(reportText);
assert.equal(report.status, 'verified_committed'); assert.equal(report.task.id, taskId); assert.equal(report.verifiedCheckJobId, passing.id);
fs.writeFileSync(path.join(root, 'delivery-report.json'), reportText); fs.writeFileSync(path.join(root, 'delivery-report.md'), markdown);
const browser = kind === 'react' ? await verifyLiveReactClient(childProject.root, path.join(root, 'screenshots')) : null;
fs.writeFileSync(path.join(root, 'session.json'), JSON.stringify(await api(route + '/export'), null, 2) + '\n');
result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, root,
taskId, sessionId: childSession.id, projectId: childProject.id, sourceProjectId: parentProject.id, workspaceId: workspace.id, workspaceRoot: childProject.root,
sourceCommit: workspace.source.head, commit: completedDelivery.commit, tests, checks: passing.results, usage: state.execution.task.usage,
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), applications, decisions, continuations, browser,
reportStatus: report.status, reportSha256: sha(reportText), sourcePreserved: true, protectedInputs: Object.fromEntries(Object.entries(protectedBytes).map(([name, bytes]) => [name, sha(bytes)])),
scope: 'One actual configured-model engineering sample. Verifier confirms bounded checks, reviews model-written batches and confirms the model-prepared Git preview; no implementation supplied by the verifier. Not an autonomous success-rate measurement.',
credentialsHandling: 'The original Provider executes requests in its own process; no model key is read, copied or written.' };
succeeded = true;
} catch (error) {
result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root,
taskId, sessionId: childSession?.id, projectId: childProject?.id, workspaceId: workspace?.id, error: error.stack,
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), usage: state?.execution?.task?.usage, state: state?.status,
stopReason: state?.execution?.run?.stopReason, applications, decisions, continuations, checks: state?.execution?.checks }; process.exitCode = 1;
} finally {
if (!succeeded && route) await api(route + '/stop', {}).catch(() => {});
let refreshError=null;
if(route){try{state=await api(route);taskId??=state.execution?.task?.id;}catch(error){refreshError=error.message;}}
const observation=observedTaskModels(state,taskId);
if(refreshError){observation.status='incomplete';observation.model=null;observation.issues.push({kind:'final_state_refresh_failed',message:refreshError});}
result.model=observation.model;result.modelObservation=observation;result.plannedModel=bootstrap.config.model;
if(state?.execution?.task?.id===taskId)result.usage=state.execution.task.usage;
}
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n');
fs.writeFileSync(path.resolve('docs', 'live-' + kind + '-engineering-validation.json'), JSON.stringify(result, null, 2) + '\n');
console.log(JSON.stringify({ success: result.success, kind, root, taskId, model:result.model,modelObservation:result.modelObservation.status,
commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));