Files
Agent-Workbench/scripts/verify-evaluations-projects.mjs

110 lines
17 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { execFileSync, spawnSync } from 'node:child_process';
import { fixture, compatibleFixture, call, completion, assertToolPairs } from '../tests/helpers.mjs';
import { specification } from '../tests/fixtures/engineering-specification.mjs';
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
import { ownerBytes, applyWorkerProposals, approveWorkerChecks } from '../tests/fixtures/workers-project.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
import { evaluationSummary } from '../src/shared/contracts/evaluations.mjs';
if(process.argv.includes('--help')){console.log('node scripts/verify-evaluations-projects.mjs — actual fixed HTTP evaluation of Node8/React9/Python10 and a source-bound Workbench summary module; original tests, isolated candidate, review, checks, Edge, local Git. Set AGENT_WORKBENCH_TEST_PYTHON. No real inference claim.');process.exit(0);}
assert.equal(process.argv.length,2);
const directory=path.resolve('.qa','engineering-evals-projects-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
const version=JSON.parse(fs.readFileSync('package.json','utf8')).version,hash=value=>createHash('sha256').update(value).digest('hex');
const sourceFixture=JSON.parse(fs.readFileSync(new URL('../tests/fixtures/evaluation-summary-source.json',import.meta.url),'utf8'));
const sourceCommit=sourceFixture.commit;assert.equal(hash(sourceFixture.source),sourceFixture.moduleSha256,'Original committed module fixture bytes');
const python=process.env.AGENT_WORKBENCH_TEST_PYTHON;assert.ok(python&&path.isAbsolute(python)&&fs.existsSync(python),'Set the isolated QA Python runtime.');
const assertions=[],results=[];const mark=value=>{assertions.push(value);console.log('PASS '+value);};
function selfSpecification(){
const source=sourceFixture.source;
const baseline=source.slice(source.indexOf('export function evaluationSummary(batch)'));
assert.ok(baseline.startsWith('export function evaluationSummary(batch)'));
const before="candidate: ['failed','stopped','unknown'].includes(sample.status) ? sample.status : 'pending'";
const after="candidate: ['failed','stopped','unknown'].includes(sample.status) ? sample.status : ['candidate_running','candidate_complete','awaiting_review','paused','collecting','adopting','sealed'].includes(sample.status) ? 'unknown' : 'pending'";
assert.ok(baseline.includes(before),'Self pilot must use the original committed summary implementation, without an invented defect.');
const tests="import test from 'node:test';import assert from 'node:assert/strict';import {evaluationSummary} from '../src/summary.mjs';\nconst summary=(status,report=null)=>evaluationSummary({evidenceMode:'local-fixture',samples:[{status,report}]});\n"+
"test('pending sample keeps its original denominator',()=>{const value=summary('pending');assert.equal(value.samples,1);assert.equal(value.candidate.pending,1);});\n"+
"test('fixed fixture never proves inference',()=>assert.equal(summary('pending').inferenceValidated,false));\n"+
"test('original failure and candidate pass remain paired',()=>{const value=summary('sealed',{summary:{original:'failed',candidate:'passed'}});assert.equal(value.original.failed,1);assert.equal(value.candidate.passed,1);});\n"+
"test('stopped attempt remains stopped',()=>assert.equal(summary('stopped').candidate.stopped,1));\n"+
"test('failed attempt remains failed',()=>assert.equal(summary('failed').candidate.failed,1));\n"+
"test('finished candidate without report is unknown, never unexecuted',()=>assert.equal(summary('candidate_complete').candidate.unknown,1));\n"+
"test('paused candidate without report is unknown, never unexecuted',()=>assert.equal(summary('paused').candidate.unknown,1));\n"+
"test('reviewed pending evidence is unknown, never unexecuted',()=>assert.equal(summary('awaiting_review').candidate.unknown,1));\n"+
"test('running candidate without report is unknown, never unexecuted',()=>assert.equal(summary('candidate_running').candidate.unknown,1));\n";
return {tests:9,checks:['test','build'],source:{commit:sourceCommit,path:'src/shared/contracts/evaluations.mjs',moduleSha256:hash(source),extractedFunctionSha256:hash(baseline),scope:'Exact committed function extracted into an isolated module project; this is not an entire-repository autonomous upgrade.'},
initial:{'src/summary.mjs':baseline,'test/summary.test.mjs':tests,'package.json':JSON.stringify({name:'workbench-summary-pilot',private:true,type:'module',scripts:{test:'node --test test/*.test.mjs',build:'node scripts/build.mjs'}}),'scripts/build.mjs':"import fs from 'node:fs';fs.mkdirSync('dist',{recursive:true});fs.copyFileSync('src/summary.mjs','dist/summary.mjs');console.log('source-bound summary built');\n"},fixes:{'src/summary.mjs':baseline.replace(before,after)}};
}
async function sample(kind){
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),saved=path.join(directory,kind);fs.mkdirSync(saved);let runtime,result;
try{
if(kind!=='self')fs.cpSync(path.resolve(kind==='react'?'benchmarks/react-delivery-v1/baseline':'benchmarks/workers-delivery-v1/'+kind),f.projectRoot,{recursive:true,filter:file=>!file.split(path.sep).includes('node_modules')});
const spec=kind==='self'?selfSpecification():specification(kind,f.projectRoot);
for(const [file,content]of Object.entries(spec.initial??{}))write(f,file,content);
write(f,'.gitignore','node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n');
write(f,'owner-staged.txt','Owner baseline\n');write(f,'owner-unstaged.txt','Owner baseline\n');
git(f,['init','--initial-branch=main']);git(f,['config','core.autocrlf','false']);git(f,['config','user.name','Workbench QA']);git(f,['config','user.email','qa@workbench.invalid']);git(f,['add','--','.']);git(f,['commit','--no-gpg-sign','-m','Seal fixed '+kind+' evaluation source']);
write(f,'owner-staged.txt','Owner staged change\n');git(f,['add','--','owner-staged.txt']);write(f,'owner-staged.txt','Owner working copy after staging\n');write(f,'owner-unstaged.txt','Owner unstaged change\n');write(f,'owner-untracked.txt','Owner untracked file\n');
const original=ownerBytes(f),writable=Object.keys(spec.fixes).sort();
const protectedFiles=Object.fromEntries(git(f,['ls-files','-z']).split('\0').filter(file=>file&&!writable.includes(file)).map(file=>[file,hash(fs.readFileSync(path.join(f.projectRoot,file)))]));
const verify=()=>{for(const[file,sha]of Object.entries(protectedFiles))assert.equal(hash(fs.readFileSync(path.join(f.projectRoot,file))),sha,'Protected '+file);assert.deepEqual(ownerBytes(f).files,original.files);};
let phase='new',candidateSession;
const model=await compatibleFixture(t,(_route,input)=>{
assertToolPairs(input.messages);let calls=[];
const owner=candidateSession&&f.store.session(candidateSession);
if(phase==='new'){phase='indexed';calls=[call(randomUUID(),'index_project',{})];}
else if(phase==='indexed'){phase='read';calls=writable.map(file=>call(randomUUID(),'read_file',{path:file}));}
else if(phase==='read'){phase='proposed';calls=Object.entries(spec.fixes).map(([file,content])=>call(randomUUID(),'propose_file',{path:file,content,reason:'本地固定HTTP候选;保留原测试与运行声明'}));}
else if(!owner.changes.some(change=>change.status==='pending')){
const plan=[...f.store.core.check_plan.values()].find(plan=>plan.taskId===owner.taskId),job=[...f.store.core.check_job.values()].find(job=>job.taskId===owner.taskId);
if(!plan)calls=[call(randomUUID(),'plan_checks',{names:spec.checks})];else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
}
return completion(calls,calls.length?'':'固定HTTP返回结束,宿主按实际成果核验');
});
f.store.saveConfig({mode:'compatible',baseUrl:model.baseUrl,model:'owned-fixed-engineering-http',maxSteps:48});
const provider=new Provider();provider.setKey('self-owned-engineering-fixture');runtime=new Runtime(f.store,provider);runtime.checkRunner().pythonExecutable=python;
const manager=runtime.evaluationManager(),batch=await manager.plan(f.session,{title:kind+' 同源工程评测',evidenceMode:'local-fixture',samples:[{label:kind+' 原版与候选',goal:kind==='self'?'修复工作台评测汇总:已经执行但尚未封存报告的候选应显示未知,不能显示未执行。':'修复实际工程边界并保持原行为;原测试与构建必须通过。',criteria:spec.tests+'项固定原测试及完整检查通过;不改测试、脚本、运行声明,保留父原暂存和未提交内容。',paths:writable,checks:spec.checks}]});
const item=batch.samples[0];await manager.prepare(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(item.status,'ready',batch.message);
const group=f.store.core.collaboration.get(item.groupId);candidateSession=group.children.find(child=>child.evaluation.role==='candidate').sessionId;
const dependencyActions=[];
if(kind==='react')for(const project of [f.project,...group.children.map(child=>f.store.project(child.projectId))]){
const cli=path.join(path.dirname(process.execPath),'node_modules/npm/bin/npm-cli.js'),action=spawnSync(process.execPath,[cli,'ci','--offline','--no-audit','--no-fund'],{cwd:project.root,windowsHide:true,timeout:120000,maxBuffer:8*1024*1024});
const file=path.join(saved,'npm-ci-'+project.id+'.log');fs.writeFileSync(file,Buffer.concat([action.stdout??Buffer.alloc(0),action.stderr??Buffer.alloc(0)]));assert.equal(action.status,0,fs.readFileSync(file,'utf8'));dependencyActions.push({projectId:project.id,operation:'QA explicitly installed dependencies from the fixed original lockfile',log:file,sha256:hash(fs.readFileSync(file))});
}
await manager.baseline(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(item.status,'baseline_complete',batch.message);assert.equal(model.requests.length,0);verify();
const baseline=f.store.core.check_job.get(item.baselineJobId);assert.equal(baseline.status,'failed',baseline.summary);assert.equal(baseline.results.find(result=>result.kind==='test').tests.total,spec.tests);
mark(kind+': fixed same-source commands run the original failing '+spec.tests+' tests with no original model request; original tests/config and owner bytes stay exact');
await manager.start(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(item.status,'awaiting_review',batch.message);await applyWorkerProposals(f,runtime,group);
await approveWorkerChecks(f,runtime,group,manager.start(f.session,batch.id,batch.fingerprint,item.id));assert.equal(item.status,'candidate_complete',batch.message);verify();
const candidate=f.store.session(candidateSession),plans=[...f.store.core.context_plan.values()].filter(plan=>plan.sessionId===candidateSession);assert.ok(plans.length>0);for(const plan of plans)verifyPlanEvidence(f.store,candidate,plan);assertToolPairs(candidate.providerMessages);
await manager.seal(f.session,batch.id,batch.fingerprint,item.id).promise;const report=manager.readReport(f.session,batch,item);assert.deepEqual(report.summary,{original:'failed',candidate:'passed'});assert.equal(report.candidate.attempts.at(-1).results.find(result=>result.tests).tests.passed,spec.tests);
assert.equal(report.inferenceValidated,false);assert.equal(report.evidenceMode,'local-fixture');assert.ok(report.candidate.approvals.some(approval=>approval.status==='approved'));
mark(kind+': actual HTTP coding tools and approved original checks pass '+spec.tests+' tests; full original failure/pass report and sealed request provenance are verified');
await manager.adopt(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(batch.status,'awaiting_delivery',batch.message);
const set=f.store.core.change_set.get(group.merge.changeSetId);await (await runtime.changeSetManager().start(f.session,set.id,set.fingerprint)).promise;assert.equal(set.status,'completed',set.summary);
const parentPlan=await runtime.checkRunner().plan(f.session,spec.checks),parentCheck=await (await runtime.checkRunner().start(f.session,parentPlan.id)).promise;assert.equal(parentCheck.status,'passed',parentCheck.summary);
const browserJobs=[];
if(kind==='react')for(const viewport of [{width:1280,height:900},{width:390,height:844}]){
const steps=[{action:'assert_count',selector:'.task-list article',count:3},{action:'select',selector:'select[aria-label="任务状态"]',value:'pending'},{action:'assert_count',selector:'.task-list article',count:2},{action:'fill',selector:'input[aria-label="任务搜索"]',value:' 编写 '},{action:'assert_count',selector:'.task-list article',count:1},{action:'fill',selector:'input[aria-label="任务搜索"]',value:''},{action:'select',selector:'select[aria-label="任务状态"]',value:'all'},{action:'click',selector:'.task-list article[data-task="api"] button'},{action:'assert_text',selector:'.task-list article[data-task="api"] .tag',text:'已完成'},{action:'assert_text',selector:'.task-list article[data-task="tests"] .tag',text:'待完成'},{action:'assert_text',selector:'.summary > div:nth-child(2) strong',text:'2'},{action:'assert_text',selector:'.summary > div:nth-child(3) strong',text:viewport.width>600?'5小时':'5\n小时'}];
const plan=await runtime.browserManager().plan(f.session,{viewport,steps}),job=await runtime.browserManager().start(f.session,plan.id).promise;assert.equal(job.status,'passed',job.summary);assert.equal(job.cleanup.exited,true);browserJobs.push(job);
for(const image of job.screenshots)fs.writeFileSync(path.join(saved,'page-'+viewport.width+'-'+image.artifactId+'.png'),f.store.artifactStore.readBytes(f.store.core.artifact.get(image.artifactId)));
mark('react: actual Edge '+viewport.width+'px interaction passes 12 fixed assertions after parent review');
}
const deliveryPlan=await runtime.gitDelivery().plan(f.session,{message:'Reviewed fixed '+kind+' evaluation candidate'}),delivery=await (await runtime.gitDelivery().start(f.session,deliveryPlan.id)).promise;assert.equal(delivery.status,'completed',delivery.summary);
await manager.finalize(f.session,batch.id,batch.fingerprint);assert.equal(batch.adoption.status,'completed');assert.deepEqual(git(f,['show','--format=','--name-only',delivery.commit]).trim().split(/\r?\n/).sort(),writable);verify();assert.equal(git(f,['show',':owner-staged.txt']),'Owner staged change\n');
mark(kind+': candidate adoption, fresh parent checks and exact-scope local Git complete; original staged/unstaged/untracked bytes remain untouched');
fs.writeFileSync(path.join(saved,'report.json'),JSON.stringify(report,null,2));fs.writeFileSync(path.join(saved,'fixed-http-requests.json'),JSON.stringify(model.requests.map(request=>({url:request.url,input:request.input,sha256:hash(JSON.stringify(request.input))})),null,2));
result={success:true,version,kind,source:spec.source??null,batch,summary:evaluationSummary(batch),report,baseline,parentCheck,dependencyActions,browserJobs,commit:delivery.commit,protectedFiles,fixtureRequests:model.requests.length,realModelRequests:0};return result;
}catch(error){result={success:false,kind,error:error.stack,checks:[...f.store.core.check_job.values()],evaluations:[...f.store.core.evaluation.values()]};throw error;}
finally{await runtime?.stopAll();f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');fs.cpSync(f.root,path.join(saved,'original-fixture'),{recursive:true,filter:file=>!file.split(path.sep).includes('node_modules')});fs.writeFileSync(path.join(saved,'result.json'),JSON.stringify(result,null,2));for(const dispose of disposers.reverse())await dispose();}
}
let report;
try{for(const kind of ['node','react','python','self'])results.push(await sample(kind));report={success:true,version,directory,assertions,results,sourceCommit,realModelRequests:0,scope:'Fixed own HTTP responses validate actual engineering execution and controlled module improvement; they do not measure autonomous inference or whole-repository self-upgrade.'};}
catch(error){report={success:false,version,directory,assertions,results,sourceCommit,error:error.stack,realModelRequests:0};process.exitCode=1;}
finally{fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));fs.writeFileSync('.qa/engineering-evals-projects-location.json',JSON.stringify({directory,result:path.join(directory,'result.json')}));console.log('EVIDENCE '+directory);console.log(JSON.stringify({success:report.success,assertions:assertions.length,error:report.error??null}));}