179 lines
18 KiB
JavaScript
179 lines
18 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { Store } from '../src/server/store.mjs';
|
|
import { APP_VERSION } from '../src/server/diagnostics.mjs';
|
|
import { gitEnvironment } from '../src/core/git-process.mjs';
|
|
import { createPythonDeliveryProject, pythonProtected } from '../tests/fixtures/python-project.mjs';
|
|
import { deliveryGit, writeDeliveryFile } from '../tests/fixtures/delivery-project.mjs';
|
|
import { captureWorkspaceSource, assertWorkspaceSource } from '../tests/fixtures/workspace-project.mjs';
|
|
import { verifyLiveReactClient } from './live-react-client.mjs';
|
|
import { observedTaskModels } from './live-model-observation.mjs';
|
|
|
|
if (process.argv.includes('--help')) {
|
|
console.log('node scripts/verify-live-engineering.mjs --case react|python [--url http://127.0.0.1:4318] — fresh owned Git source/workspace, configured model and bounded review/check/delivery acceptance'); process.exit(0);
|
|
}
|
|
const index = process.argv.indexOf('--case'), kind = index >= 0 ? process.argv[index + 1] : null;
|
|
assert.ok(['react', 'python'].includes(kind), '--case react or --case python is required');
|
|
const urlIndex = process.argv.indexOf('--url'), origin = new URL(urlIndex >= 0 ? process.argv[urlIndex + 1] : 'http://127.0.0.1:4318');
|
|
assert.equal(origin.hostname, '127.0.0.1'); assert.equal(origin.protocol, 'http:');
|
|
const bootstrap = await (await fetch(origin.origin + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
|
|
assert.equal(bootstrap.version, APP_VERSION); assert.equal(bootstrap.config.mode, 'compatible'); assert.ok(bootstrap.config.model);
|
|
const api = async (route, input) => {
|
|
const response = await fetch(origin.origin + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
|
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
|
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
|
|
const value = await response.json(); if (!response.ok) throw Object.assign(new Error(value.error ?? 'Workbench HTTP ' + response.status), { status: response.status }); return value;
|
|
};
|
|
const root = path.resolve('.qa', 'live-' + kind + '-' + randomUUID()), sourceRoot = path.join(root, kind + '-source');
|
|
fs.mkdirSync(sourceRoot, { recursive: true }); fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# Owned ' + kind + ' engineering acceptance\n');
|
|
const sourceStore = new Store(path.join(root, 'fixture-data')), sourceProject = sourceStore.addProject(sourceRoot), fixtureSession = sourceStore.createSession(sourceProject.id);
|
|
const sourceF = { projectRoot: sourceRoot, store: sourceStore, session: fixtureSession };
|
|
const allowed = kind === 'react' ? ['src/query.ts', 'src/summary.ts', 'src/transition.ts', 'src/App.tsx']
|
|
: ['src/workflow/normalize.py', 'src/workflow/summary.py', 'src/workflow/render.py'];
|
|
const checkNames = kind === 'react' ? ['test', 'typecheck', 'build'] : ['python:test', 'python:compile'];
|
|
try {
|
|
if (kind === 'python') createPythonDeliveryProject(sourceF);
|
|
else {
|
|
fs.cpSync(path.resolve('benchmarks/react-delivery-v1/baseline'), sourceRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
|
writeDeliveryFile(sourceF, '.gitignore', 'node_modules/\ndist/\n.env\n.awb-*\n');
|
|
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(sourceF, name, 'Owner baseline bytes\n');
|
|
deliveryGit(sourceF, ['init', '--initial-branch=main']); deliveryGit(sourceF, ['config', 'core.autocrlf', 'false']);
|
|
deliveryGit(sourceF, ['config', 'user.name', 'Workbench Engineering QA']); deliveryGit(sourceF, ['config', 'user.email', 'qa@workbench.invalid']);
|
|
deliveryGit(sourceF, ['add', '--', '.']); deliveryGit(sourceF, ['commit', '--no-gpg-sign', '-m', 'Seal four-file React failure for actual-model acceptance']);
|
|
writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner staged change\n'); deliveryGit(sourceF, ['add', '--', 'owner-staged.txt']);
|
|
writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner working copy after staging\n'); writeDeliveryFile(sourceF, 'owner-unstaged.txt', 'Owner unstaged change\n');
|
|
writeDeliveryFile(sourceF, 'owner-untracked.txt', 'Owner untracked change\n');
|
|
}
|
|
} finally { sourceStore.close(); }
|
|
const originalSource = captureWorkspaceSource(sourceF), protectedNames = kind === 'react'
|
|
? ['README.md', 'package.json', 'package-lock.json', 'tsconfig.json', 'vite.config.mjs', 'index.html', 'test/domain.test.mjs', 'src/types.ts', 'src/main.tsx', 'src/styles.css']
|
|
: ['README.md', ...pythonProtected, 'package.json', 'test/add.test.mjs', 'scripts/build.mjs', 'src/add.mjs'];
|
|
const parentProject = await api('/projects', { root: sourceRoot }), parentSession = await api('/sessions', { projectId: parentProject.id });
|
|
const parentRoute = '/sessions/' + parentSession.id;
|
|
let state, workspace, childProject, childSession, route, taskId, result, succeeded = false, completedDelivery = null, lastProgress = '';
|
|
const decisions = [], applications = [], continuations = [], startedAt = Date.now(), maxModelResponses = 30, maxReportedTokens = 180000, maxDurationMs = 360000;
|
|
const sha = value => createHash('sha256').update(value).digest('hex');
|
|
const wait = async (predicate, timeout = 30000) => {
|
|
const deadline = Date.now() + timeout; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Acceptance wait timed out'); await new Promise(resolve => setTimeout(resolve, 250)); }
|
|
};
|
|
let protectedBytes, childF, ownerStage;
|
|
const assertProtected = () => {
|
|
assertWorkspaceSource(assert, sourceF, originalSource, workspace.branch);
|
|
for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(childProject.root, name)), bytes, 'Protected input changed: ' + name);
|
|
assert.equal(deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']), ownerStage);
|
|
};
|
|
const readArtifact = async id => {
|
|
let text = '', offset = 0, page;
|
|
do { page = await api('/artifacts/' + id + '?session=' + childSession.id + '&offset=' + offset + '&limit=16000');
|
|
assert.ok(page.content.length || !page.truncated); text += page.content; offset += page.content.length; } while (page.truncated);
|
|
assert.equal(sha(text), page.sha256); return text;
|
|
};
|
|
try {
|
|
const prepared = await api(parentRoute + '/workspaces', { label: kind + ' actual engineering' }); workspace = prepared;
|
|
await api(parentRoute + '/workspaces/' + workspace.id + '/create', { fingerprint: workspace.fingerprint });
|
|
await wait(async () => { workspace = await api(parentRoute + '/workspaces/' + workspace.id); return workspace.status === 'ready'; }, 30000);
|
|
const opened = await api(parentRoute + '/workspaces/' + workspace.id + '/open', {}); childProject = opened.project; childSession = opened.session; route = '/sessions/' + childSession.id;
|
|
childF = { projectRoot: childProject.root };
|
|
if (kind === 'react') {
|
|
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
|
const installed = execFileSync(process.execPath, [npmCli, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: childProject.root, env: gitEnvironment(), windowsHide: true, encoding: 'utf8' });
|
|
fs.writeFileSync(path.join(root, 'offline-install.log'), installed);
|
|
}
|
|
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(childF, name, 'Child owner baseline\n');
|
|
deliveryGit(childF, ['add', '--', 'owner-staged.txt']); writeDeliveryFile(childF, 'owner-staged.txt', 'Child owner working copy after stage\n');
|
|
writeDeliveryFile(childF, 'owner-untracked.txt', 'Child owner untracked change\n');
|
|
protectedBytes = Object.fromEntries([...protectedNames, 'owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(childProject.root, name))]));
|
|
ownerStage = deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']); assertProtected();
|
|
const requirements = kind === 'react'
|
|
? '修复交付看板的query、summary、transition和App四个模块。状态筛选与trim后忽略大小写的搜索取交集;统计针对全部任务,完成数量及待办工时准确;toggle只切换指定id并保持其他对象和原数组,App绑定筛选/搜索和函数式状态更新,切换一条任务后界面与统计同步。必须通过9项测试、类型检查、构建,以及真正浏览器的筛选/搜索/单任务切换。'
|
|
: '修复normalize、summary和render三个Python模块:normalize把数字或数字字符串转换为数值并把负数归零;summary返回count和数值合计total,空列表为0;render返回“共 N 项 · 合计 X”,整数合计没有多余小数。必须通过六项unittest及源码编译。';
|
|
const goal = requirements + '先用inspect_project核对工程与工作区,再用plan_checks明确选择' + JSON.stringify(checkNames)
|
|
+ '并用run_checks实际执行,保留基线失败与日志。根据真实失败读取有关源码;只能修改' + JSON.stringify(allowed)
|
|
+ ',测试、配置、README、类型声明、样式、OWNER文件和已有暂存保持。不能改测试来通过,不使用run_command、不能直接提交Git。'
|
|
+ '用propose_patch或propose_file准备全部允许模块的提案,再用prepare_change_set一次封存完整三/四文件批次,等待用户审查应用;未审查的提案不是已写入。应用后继续同一Task,用相同检查范围重新执行并根据日志修复。全部检查通过后用plan_delivery准备这些文件的本地Git预览,等待确认;停止继续查询或请求模型。最后以实际结果说明测试、构建和交付。';
|
|
await api(route + '/messages', { content: goal });
|
|
while (Date.now() - startedAt < maxDurationMs) {
|
|
state = await api(route); taskId ??= state.execution.task.id; assert.equal(state.execution.task.id, taskId); assertProtected();
|
|
if (state.execution.task.usage.requests > maxModelResponses || state.execution.task.usage.totalTokens > maxReportedTokens) throw new Error('Actual-model acceptance reached its bounded response/token scope');
|
|
const progress = state.status + '|' + state.events.length + '|' + state.execution.checks.map(job => job.status).join(',');
|
|
if (progress !== lastProgress) { console.log(JSON.stringify({ kind, status: state.status, elapsedSeconds: Math.round((Date.now() - startedAt) / 1000),
|
|
responses: state.execution.task.usage.requests, reportedTokens: state.execution.task.usage.totalTokens,
|
|
latestTools: state.events.filter(event => event.kind === 'tool').slice(-3).map(event => ({ tool: event.tool, phase: event.phase })) })); lastProgress = progress; }
|
|
for (const proposal of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(proposal.path), 'Out-of-scope model proposal: ' + proposal.path);
|
|
for (const approval of state.approvals.filter(item => item.status === 'pending')) {
|
|
const plan = state.execution.plans.find(value => value.id === approval.checkPlanId);
|
|
const allow = !!plan && plan.taskId === taskId && plan.checks.length > 0 && plan.checks.every(check => checkNames.includes(check.name) && !check.lifecycle?.length)
|
|
&& approval.cwd === childProject.root && approval.command === plan.checks.map(check => check.command).join('\n');
|
|
decisions.push({ approvalId: approval.id, checkPlanId: plan?.id, command: approval.command, allow });
|
|
await api(route + '/approvals/' + approval.id, { allow }); assert.ok(allow, 'Requested command does not match the owned engineering contract');
|
|
}
|
|
if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await new Promise(resolve => setTimeout(resolve, 750)); continue; }
|
|
if (state.status === 'failed') throw new Error(state.failure?.message ?? 'Actual-model Task failed');
|
|
const deliveryPlan = state.execution.deliveryPlans?.findLast(plan => plan.status === 'planned' && plan.taskId === taskId);
|
|
if (deliveryPlan) {
|
|
assert.deepEqual(deliveryPlan.paths.slice().sort(), allowed.slice().sort()); const requestsBefore = state.execution.task.usage.requests;
|
|
assert.ok(state.events.some(event => event.tool === 'plan_delivery' && event.phase === 'completed'), 'The model must prepare the Git review');
|
|
await api(route + '/delivery-jobs', { planId: deliveryPlan.id });
|
|
await wait(async () => { state = await api(route); completedDelivery = state.execution.deliveries.at(-1); return completedDelivery?.reportArtifactId && completedDelivery?.reportMarkdownArtifactId; }, 30000);
|
|
assert.equal(completedDelivery.status, 'completed'); assert.equal(state.execution.task.usage.requests, requestsBefore); break;
|
|
}
|
|
const pending = state.changes.filter(change => change.status === 'pending');
|
|
const batch = state.execution.changeSets?.findLast(set => set.taskId === taskId && set.status === 'prepared');
|
|
if (state.status === 'paused' && !batch) {
|
|
assert.equal(state.execution.run.stopReason, 'step_limit', 'Other stopped outcomes are recorded, not automatically replayed');
|
|
assert.ok(continuations.length < 3); continuations.push({ runId: state.runId, reason: 'step_limit', taskId }); await api(route + '/resume', {}); continue;
|
|
}
|
|
if (pending.length) {
|
|
assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(value => value.tests?.failed > 0)), 'Record actual initial test failure before repairs');
|
|
assert.ok(batch && batch.source === 'agent', 'The model must seal the reviewed batch');
|
|
assert.deepEqual(batch.files.map(file => file.changeId).sort(), pending.map(change => change.id).sort());
|
|
if (!applications.length) assert.deepEqual(batch.files.map(file => file.path).sort(), allowed.slice().sort());
|
|
assert.ok(applications.length < 3); await api(route + '/change-sets/' + batch.id + '/apply', { fingerprint: batch.fingerprint });
|
|
await wait(async () => (await api(route + '/change-sets/' + batch.id)).status === 'completed');
|
|
for (const proposal of pending) assert.equal(sha(fs.readFileSync(path.join(childProject.root, proposal.path))), proposal.sha256);
|
|
applications.push({ id: batch.id, fingerprint: batch.fingerprint, paths: batch.files.map(file => file.path), source: batch.source }); assertProtected();
|
|
await api(route + '/continue-reviewed', {}); continue;
|
|
}
|
|
throw new Error('The model stopped before the complete check/review/Git flow: ' + state.status + '/' + state.execution.run.stopReason);
|
|
}
|
|
assert.ok(completedDelivery, 'Actual-model engineering acceptance timed out'); assert.equal(state.status, 'completed');
|
|
const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passing);
|
|
assert.deepEqual(passing.results.map(value => value.name).sort(), checkNames.slice().sort());
|
|
const tests = passing.results.find(value => value.name === (kind === 'react' ? 'test' : 'python:test')).tests;
|
|
assert.equal(tests.total, kind === 'react' ? 9 : 6); assert.equal(tests.passed, tests.total); assert.equal(tests.failed, 0);
|
|
assert.deepEqual(deliveryGit(childF, ['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), allowed.slice().sort()); assertProtected();
|
|
const reportText = await readArtifact(completedDelivery.reportArtifactId), markdown = await readArtifact(completedDelivery.reportMarkdownArtifactId), report = JSON.parse(reportText);
|
|
assert.equal(report.status, 'verified_committed'); assert.equal(report.task.id, taskId); assert.equal(report.verifiedCheckJobId, passing.id);
|
|
fs.writeFileSync(path.join(root, 'delivery-report.json'), reportText); fs.writeFileSync(path.join(root, 'delivery-report.md'), markdown);
|
|
const browser = kind === 'react' ? await verifyLiveReactClient(childProject.root, path.join(root, 'screenshots')) : null;
|
|
fs.writeFileSync(path.join(root, 'session.json'), JSON.stringify(await api(route + '/export'), null, 2) + '\n');
|
|
result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, root,
|
|
taskId, sessionId: childSession.id, projectId: childProject.id, sourceProjectId: parentProject.id, workspaceId: workspace.id, workspaceRoot: childProject.root,
|
|
sourceCommit: workspace.source.head, commit: completedDelivery.commit, tests, checks: passing.results, usage: state.execution.task.usage,
|
|
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), applications, decisions, continuations, browser,
|
|
reportStatus: report.status, reportSha256: sha(reportText), sourcePreserved: true, protectedInputs: Object.fromEntries(Object.entries(protectedBytes).map(([name, bytes]) => [name, sha(bytes)])),
|
|
scope: 'One actual configured-model engineering sample. Verifier confirms bounded checks, reviews model-written batches and confirms the model-prepared Git preview; no implementation supplied by the verifier. Not an autonomous success-rate measurement.',
|
|
credentialsHandling: 'The original Provider executes requests in its own process; no model key is read, copied or written.' };
|
|
succeeded = true;
|
|
} catch (error) {
|
|
result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root,
|
|
taskId, sessionId: childSession?.id, projectId: childProject?.id, workspaceId: workspace?.id, error: error.stack,
|
|
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), usage: state?.execution?.task?.usage, state: state?.status,
|
|
stopReason: state?.execution?.run?.stopReason, applications, decisions, continuations, checks: state?.execution?.checks }; process.exitCode = 1;
|
|
} finally {
|
|
if (!succeeded && route) await api(route + '/stop', {}).catch(() => {});
|
|
let refreshError=null;
|
|
if(route){try{state=await api(route);taskId??=state.execution?.task?.id;}catch(error){refreshError=error.message;}}
|
|
const observation=observedTaskModels(state,taskId);
|
|
if(refreshError){observation.status='incomplete';observation.model=null;observation.issues.push({kind:'final_state_refresh_failed',message:refreshError});}
|
|
result.model=observation.model;result.modelObservation=observation;result.plannedModel=bootstrap.config.model;
|
|
if(state?.execution?.task?.id===taskId)result.usage=state.execution.task.usage;
|
|
}
|
|
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
|
fs.writeFileSync(path.resolve('docs', 'live-' + kind + '-engineering-validation.json'), JSON.stringify(result, null, 2) + '\n');
|
|
console.log(JSON.stringify({ success: result.success, kind, root, taskId, model:result.model,modelObservation:result.modelObservation.status,
|
|
commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));
|