import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; import { createHash, randomUUID } from 'node:crypto'; import { Store } from '../src/server/store.mjs'; import { APP_VERSION } from '../src/server/diagnostics.mjs'; import { gitEnvironment } from '../src/core/git-process.mjs'; import { createPythonDeliveryProject, pythonProtected } from '../tests/fixtures/python-project.mjs'; import { deliveryGit, writeDeliveryFile } from '../tests/fixtures/delivery-project.mjs'; import { captureWorkspaceSource, assertWorkspaceSource } from '../tests/fixtures/workspace-project.mjs'; import { verifyLiveReactClient } from './live-react-client.mjs'; import { observedTaskModels } from './live-model-observation.mjs'; if (process.argv.includes('--help')) { console.log('node scripts/verify-live-engineering.mjs --case react|python [--url http://127.0.0.1:4318] — fresh owned Git source/workspace, configured model and bounded review/check/delivery acceptance'); process.exit(0); } const index = process.argv.indexOf('--case'), kind = index >= 0 ? process.argv[index + 1] : null; assert.ok(['react', 'python'].includes(kind), '--case react or --case python is required'); const urlIndex = process.argv.indexOf('--url'), origin = new URL(urlIndex >= 0 ? process.argv[urlIndex + 1] : 'http://127.0.0.1:4318'); assert.equal(origin.hostname, '127.0.0.1'); assert.equal(origin.protocol, 'http:'); const bootstrap = await (await fetch(origin.origin + '/api/bootstrap', { headers: { Connection: 'close' } })).json(); assert.equal(bootstrap.version, APP_VERSION); assert.equal(bootstrap.config.mode, 'compatible'); assert.ok(bootstrap.config.model); const api = async (route, input) => { const response = await fetch(origin.origin + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, ...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) }); const value = await response.json(); if (!response.ok) throw Object.assign(new Error(value.error ?? 'Workbench HTTP ' + response.status), { status: response.status }); return value; }; const root = path.resolve('.qa', 'live-' + kind + '-' + randomUUID()), sourceRoot = path.join(root, kind + '-source'); fs.mkdirSync(sourceRoot, { recursive: true }); fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# Owned ' + kind + ' engineering acceptance\n'); const sourceStore = new Store(path.join(root, 'fixture-data')), sourceProject = sourceStore.addProject(sourceRoot), fixtureSession = sourceStore.createSession(sourceProject.id); const sourceF = { projectRoot: sourceRoot, store: sourceStore, session: fixtureSession }; const allowed = kind === 'react' ? ['src/query.ts', 'src/summary.ts', 'src/transition.ts', 'src/App.tsx'] : ['src/workflow/normalize.py', 'src/workflow/summary.py', 'src/workflow/render.py']; const checkNames = kind === 'react' ? ['test', 'typecheck', 'build'] : ['python:test', 'python:compile']; try { if (kind === 'python') createPythonDeliveryProject(sourceF); else { fs.cpSync(path.resolve('benchmarks/react-delivery-v1/baseline'), sourceRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') }); writeDeliveryFile(sourceF, '.gitignore', 'node_modules/\ndist/\n.env\n.awb-*\n'); for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(sourceF, name, 'Owner baseline bytes\n'); deliveryGit(sourceF, ['init', '--initial-branch=main']); deliveryGit(sourceF, ['config', 'core.autocrlf', 'false']); deliveryGit(sourceF, ['config', 'user.name', 'Workbench Engineering QA']); deliveryGit(sourceF, ['config', 'user.email', 'qa@workbench.invalid']); deliveryGit(sourceF, ['add', '--', '.']); deliveryGit(sourceF, ['commit', '--no-gpg-sign', '-m', 'Seal four-file React failure for actual-model acceptance']); writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner staged change\n'); deliveryGit(sourceF, ['add', '--', 'owner-staged.txt']); writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner working copy after staging\n'); writeDeliveryFile(sourceF, 'owner-unstaged.txt', 'Owner unstaged change\n'); writeDeliveryFile(sourceF, 'owner-untracked.txt', 'Owner untracked change\n'); } } finally { sourceStore.close(); } const originalSource = captureWorkspaceSource(sourceF), protectedNames = kind === 'react' ? ['README.md', 'package.json', 'package-lock.json', 'tsconfig.json', 'vite.config.mjs', 'index.html', 'test/domain.test.mjs', 'src/types.ts', 'src/main.tsx', 'src/styles.css'] : ['README.md', ...pythonProtected, 'package.json', 'test/add.test.mjs', 'scripts/build.mjs', 'src/add.mjs']; const parentProject = await api('/projects', { root: sourceRoot }), parentSession = await api('/sessions', { projectId: parentProject.id }); const parentRoute = '/sessions/' + parentSession.id; let state, workspace, childProject, childSession, route, taskId, result, succeeded = false, completedDelivery = null, lastProgress = ''; const decisions = [], applications = [], continuations = [], startedAt = Date.now(), maxModelResponses = 30, maxReportedTokens = 180000, maxDurationMs = 360000; const sha = value => createHash('sha256').update(value).digest('hex'); const wait = async (predicate, timeout = 30000) => { const deadline = Date.now() + timeout; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Acceptance wait timed out'); await new Promise(resolve => setTimeout(resolve, 250)); } }; let protectedBytes, childF, ownerStage; const assertProtected = () => { assertWorkspaceSource(assert, sourceF, originalSource, workspace.branch); for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(childProject.root, name)), bytes, 'Protected input changed: ' + name); assert.equal(deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']), ownerStage); }; const readArtifact = async id => { let text = '', offset = 0, page; do { page = await api('/artifacts/' + id + '?session=' + childSession.id + '&offset=' + offset + '&limit=16000'); assert.ok(page.content.length || !page.truncated); text += page.content; offset += page.content.length; } while (page.truncated); assert.equal(sha(text), page.sha256); return text; }; try { const prepared = await api(parentRoute + '/workspaces', { label: kind + ' actual engineering' }); workspace = prepared; await api(parentRoute + '/workspaces/' + workspace.id + '/create', { fingerprint: workspace.fingerprint }); await wait(async () => { workspace = await api(parentRoute + '/workspaces/' + workspace.id); return workspace.status === 'ready'; }, 30000); const opened = await api(parentRoute + '/workspaces/' + workspace.id + '/open', {}); childProject = opened.project; childSession = opened.session; route = '/sessions/' + childSession.id; childF = { projectRoot: childProject.root }; if (kind === 'react') { const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js'); const installed = execFileSync(process.execPath, [npmCli, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: childProject.root, env: gitEnvironment(), windowsHide: true, encoding: 'utf8' }); fs.writeFileSync(path.join(root, 'offline-install.log'), installed); } for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(childF, name, 'Child owner baseline\n'); deliveryGit(childF, ['add', '--', 'owner-staged.txt']); writeDeliveryFile(childF, 'owner-staged.txt', 'Child owner working copy after stage\n'); writeDeliveryFile(childF, 'owner-untracked.txt', 'Child owner untracked change\n'); protectedBytes = Object.fromEntries([...protectedNames, 'owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(childProject.root, name))])); ownerStage = deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']); assertProtected(); const requirements = kind === 'react' ? '修复交付看板的query、summary、transition和App四个模块。状态筛选与trim后忽略大小写的搜索取交集;统计针对全部任务,完成数量及待办工时准确;toggle只切换指定id并保持其他对象和原数组,App绑定筛选/搜索和函数式状态更新,切换一条任务后界面与统计同步。必须通过9项测试、类型检查、构建,以及真正浏览器的筛选/搜索/单任务切换。' : '修复normalize、summary和render三个Python模块:normalize把数字或数字字符串转换为数值并把负数归零;summary返回count和数值合计total,空列表为0;render返回“共 N 项 · 合计 X”,整数合计没有多余小数。必须通过六项unittest及源码编译。'; const goal = requirements + '先用inspect_project核对工程与工作区,再用plan_checks明确选择' + JSON.stringify(checkNames) + '并用run_checks实际执行,保留基线失败与日志。根据真实失败读取有关源码;只能修改' + JSON.stringify(allowed) + ',测试、配置、README、类型声明、样式、OWNER文件和已有暂存保持。不能改测试来通过,不使用run_command、不能直接提交Git。' + '用propose_patch或propose_file准备全部允许模块的提案,再用prepare_change_set一次封存完整三/四文件批次,等待用户审查应用;未审查的提案不是已写入。应用后继续同一Task,用相同检查范围重新执行并根据日志修复。全部检查通过后用plan_delivery准备这些文件的本地Git预览,等待确认;停止继续查询或请求模型。最后以实际结果说明测试、构建和交付。'; await api(route + '/messages', { content: goal }); while (Date.now() - startedAt < maxDurationMs) { state = await api(route); taskId ??= state.execution.task.id; assert.equal(state.execution.task.id, taskId); assertProtected(); if (state.execution.task.usage.requests > maxModelResponses || state.execution.task.usage.totalTokens > maxReportedTokens) throw new Error('Actual-model acceptance reached its bounded response/token scope'); const progress = state.status + '|' + state.events.length + '|' + state.execution.checks.map(job => job.status).join(','); if (progress !== lastProgress) { console.log(JSON.stringify({ kind, status: state.status, elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), responses: state.execution.task.usage.requests, reportedTokens: state.execution.task.usage.totalTokens, latestTools: state.events.filter(event => event.kind === 'tool').slice(-3).map(event => ({ tool: event.tool, phase: event.phase })) })); lastProgress = progress; } for (const proposal of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(proposal.path), 'Out-of-scope model proposal: ' + proposal.path); for (const approval of state.approvals.filter(item => item.status === 'pending')) { const plan = state.execution.plans.find(value => value.id === approval.checkPlanId); const allow = !!plan && plan.taskId === taskId && plan.checks.length > 0 && plan.checks.every(check => checkNames.includes(check.name) && !check.lifecycle?.length) && approval.cwd === childProject.root && approval.command === plan.checks.map(check => check.command).join('\n'); decisions.push({ approvalId: approval.id, checkPlanId: plan?.id, command: approval.command, allow }); await api(route + '/approvals/' + approval.id, { allow }); assert.ok(allow, 'Requested command does not match the owned engineering contract'); } if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await new Promise(resolve => setTimeout(resolve, 750)); continue; } if (state.status === 'failed') throw new Error(state.failure?.message ?? 'Actual-model Task failed'); const deliveryPlan = state.execution.deliveryPlans?.findLast(plan => plan.status === 'planned' && plan.taskId === taskId); if (deliveryPlan) { assert.deepEqual(deliveryPlan.paths.slice().sort(), allowed.slice().sort()); const requestsBefore = state.execution.task.usage.requests; assert.ok(state.events.some(event => event.tool === 'plan_delivery' && event.phase === 'completed'), 'The model must prepare the Git review'); await api(route + '/delivery-jobs', { planId: deliveryPlan.id }); await wait(async () => { state = await api(route); completedDelivery = state.execution.deliveries.at(-1); return completedDelivery?.reportArtifactId && completedDelivery?.reportMarkdownArtifactId; }, 30000); assert.equal(completedDelivery.status, 'completed'); assert.equal(state.execution.task.usage.requests, requestsBefore); break; } const pending = state.changes.filter(change => change.status === 'pending'); const batch = state.execution.changeSets?.findLast(set => set.taskId === taskId && set.status === 'prepared'); if (state.status === 'paused' && !batch) { assert.equal(state.execution.run.stopReason, 'step_limit', 'Other stopped outcomes are recorded, not automatically replayed'); assert.ok(continuations.length < 3); continuations.push({ runId: state.runId, reason: 'step_limit', taskId }); await api(route + '/resume', {}); continue; } if (pending.length) { assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(value => value.tests?.failed > 0)), 'Record actual initial test failure before repairs'); assert.ok(batch && batch.source === 'agent', 'The model must seal the reviewed batch'); assert.deepEqual(batch.files.map(file => file.changeId).sort(), pending.map(change => change.id).sort()); if (!applications.length) assert.deepEqual(batch.files.map(file => file.path).sort(), allowed.slice().sort()); assert.ok(applications.length < 3); await api(route + '/change-sets/' + batch.id + '/apply', { fingerprint: batch.fingerprint }); await wait(async () => (await api(route + '/change-sets/' + batch.id)).status === 'completed'); for (const proposal of pending) assert.equal(sha(fs.readFileSync(path.join(childProject.root, proposal.path))), proposal.sha256); applications.push({ id: batch.id, fingerprint: batch.fingerprint, paths: batch.files.map(file => file.path), source: batch.source }); assertProtected(); await api(route + '/continue-reviewed', {}); continue; } throw new Error('The model stopped before the complete check/review/Git flow: ' + state.status + '/' + state.execution.run.stopReason); } assert.ok(completedDelivery, 'Actual-model engineering acceptance timed out'); assert.equal(state.status, 'completed'); const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passing); assert.deepEqual(passing.results.map(value => value.name).sort(), checkNames.slice().sort()); const tests = passing.results.find(value => value.name === (kind === 'react' ? 'test' : 'python:test')).tests; assert.equal(tests.total, kind === 'react' ? 9 : 6); assert.equal(tests.passed, tests.total); assert.equal(tests.failed, 0); assert.deepEqual(deliveryGit(childF, ['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), allowed.slice().sort()); assertProtected(); const reportText = await readArtifact(completedDelivery.reportArtifactId), markdown = await readArtifact(completedDelivery.reportMarkdownArtifactId), report = JSON.parse(reportText); assert.equal(report.status, 'verified_committed'); assert.equal(report.task.id, taskId); assert.equal(report.verifiedCheckJobId, passing.id); fs.writeFileSync(path.join(root, 'delivery-report.json'), reportText); fs.writeFileSync(path.join(root, 'delivery-report.md'), markdown); const browser = kind === 'react' ? await verifyLiveReactClient(childProject.root, path.join(root, 'screenshots')) : null; fs.writeFileSync(path.join(root, 'session.json'), JSON.stringify(await api(route + '/export'), null, 2) + '\n'); result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, root, taskId, sessionId: childSession.id, projectId: childProject.id, sourceProjectId: parentProject.id, workspaceId: workspace.id, workspaceRoot: childProject.root, sourceCommit: workspace.source.head, commit: completedDelivery.commit, tests, checks: passing.results, usage: state.execution.task.usage, elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), applications, decisions, continuations, browser, reportStatus: report.status, reportSha256: sha(reportText), sourcePreserved: true, protectedInputs: Object.fromEntries(Object.entries(protectedBytes).map(([name, bytes]) => [name, sha(bytes)])), scope: 'One actual configured-model engineering sample. Verifier confirms bounded checks, reviews model-written batches and confirms the model-prepared Git preview; no implementation supplied by the verifier. Not an autonomous success-rate measurement.', credentialsHandling: 'The original Provider executes requests in its own process; no model key is read, copied or written.' }; succeeded = true; } catch (error) { result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root, taskId, sessionId: childSession?.id, projectId: childProject?.id, workspaceId: workspace?.id, error: error.stack, elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), usage: state?.execution?.task?.usage, state: state?.status, stopReason: state?.execution?.run?.stopReason, applications, decisions, continuations, checks: state?.execution?.checks }; process.exitCode = 1; } finally { if (!succeeded && route) await api(route + '/stop', {}).catch(() => {}); let refreshError=null; if(route){try{state=await api(route);taskId??=state.execution?.task?.id;}catch(error){refreshError=error.message;}} const observation=observedTaskModels(state,taskId); if(refreshError){observation.status='incomplete';observation.model=null;observation.issues.push({kind:'final_state_refresh_failed',message:refreshError});} result.model=observation.model;result.modelObservation=observation;result.plannedModel=bootstrap.config.model; if(state?.execution?.task?.id===taskId)result.usage=state.execution.task.usage; } fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n'); fs.writeFileSync(path.resolve('docs', 'live-' + kind + '-engineering-validation.json'), JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify({ success: result.success, kind, root, taskId, model:result.model,modelObservation:result.modelObservation.status, commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));