Files
Agent-Workbench/tests/evaluations.test.mjs

158 lines
18 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { randomUUID } from 'node:crypto';
import { fixture, call, until } from './helpers.mjs';
import { createDeliveryProject, deliveryGit } from './fixtures/delivery-project.mjs';
import { ownerBytes, applyWorkerProposals, approveWorkerChecks } from './fixtures/workers-project.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Store } from '../src/server/store.mjs';
import { proposeFile } from '../src/server/files.mjs';
import { evaluationInput, evaluationSourcePath, evaluationSummary } from '../src/shared/contracts/evaluations.mjs';
import { verifyEvaluationSources } from '../src/core/evaluation/provenance.mjs';
import { workerHttp } from './fixtures/workers-http.mjs';
export const definition = (count=1) => ({title:'修复求和并核对完整原测试',evidenceMode:'local-fixture',samples:Array.from({length:count},(_,index)=>({label:'加法样本'+index,goal:'修复加法运算,保留已有功能',criteria:'原3项加法测试及构建通过;不修改原测试',paths:['src/add.mjs'],checks:['test','build']}))});
test('both evaluation worktrees can approve fixed dependency preparation and different actual dependencies reject candidate execution',{timeout:90000},async t=>{
const f=fixture(t);createDeliveryProject(f,{ownerChanges:false});
const manifest=JSON.parse(fs.readFileSync(path.join(f.projectRoot,'package.json'),'utf8'));
fs.writeFileSync(path.join(f.projectRoot,'package-lock.json'),JSON.stringify({name:manifest.name,version:manifest.version,lockfileVersion:3,requires:true,packages:{'':{name:manifest.name,version:manifest.version}}}));
deliveryGit(f,['add','--','package-lock.json']);deliveryGit(f,['-c','user.name=Workbench QA','-c','user.email=qa@workbench.local','commit','-m','Seal owned lockfile']);
const provider=new Provider();let requests=0;provider.complete=async()=>{requests++;throw new Error('Unexpected model call');};
const runtime=new Runtime(f.store,provider),manager=runtime.evaluationManager();f.onCleanup(()=>runtime.stopAll());
const batch=await manager.plan(f.session,definition()),sample=batch.samples[0];await manager.prepare(f.session,batch.id,batch.fingerprint,sample.id).promise;
assert.equal(sample.status,'ready',batch.message);assert.ok(sample.environmentInputs.dependencyInputs);
for(const child of sample.children){const owner=f.store.session(child.sessionId),environment=runtime.environmentManager(),plan=await environment.plan(owner,{offline:true});
const job=await(await environment.start(owner,plan.id,plan.fingerprint,{approved:true})).promise;assert.equal(job.status,'prepared',job.summary+'\n'+job.logExcerpt);}
await manager.baseline(f.session,batch.id,batch.fingerprint,sample.id).promise;assert.equal(sample.status,'baseline_complete',batch.message);
const candidate=sample.children.find(child=>child.role==='candidate'),root=f.store.project(candidate.projectId).root;
fs.mkdirSync(path.join(root,'node_modules','external'),{recursive:true});fs.writeFileSync(path.join(root,'node_modules','external','package.json'),'{}');
await manager.start(f.session,batch.id,batch.fingerprint,sample.id).promise;assert.equal(sample.status,'failed');assert.match(sample.error,/依赖环境不同/);assert.equal(requests,0);
await manager.seal(f.session,batch.id,batch.fingerprint,sample.id).promise;const report=manager.readReport(f.session,batch,sample);
assert.equal(report.inputVerified,false);assert.equal(report.environmentComparison.matched,false);assert.equal(report.summary.candidate,'unknown');
});
test('evaluation denominator distinguishes attempted unsealed candidates from unexecuted samples',()=>{
for(const status of ['candidate_running','candidate_complete','awaiting_review','paused','collecting','adopting','sealed']){
const summary=evaluationSummary({evidenceMode:'local-fixture',samples:[{status}]});
assert.equal(summary.samples,1,status);assert.equal(summary.candidate.unknown,1,status);assert.equal(summary.candidate.pending,0,status);assert.equal(summary.inferenceValidated,false);
}
for(const status of ['pending','ready','baseline_running','baseline_complete'])assert.equal(evaluationSummary({samples:[{status}]}).candidate.pending,1,status);
});
test('evaluation summary retains sealed outcomes and stopped or failed original attempts',()=>{
const summary=evaluationSummary({evidenceMode:'local-fixture',samples:[{status:'failed'},{status:'stopped'},{status:'unknown'},{status:'sealed',report:{summary:{original:'failed',candidate:'passed'}}}]});
assert.equal(summary.samples,4);assert.equal(summary.original.failed,1);assert.deepEqual(summary.candidate,{passed:1,failed:1,stopped:1,unknown:1,pending:0});
});
function setup(t,{respond=null,usage=true}={}) {
const f=fixture(t);createDeliveryProject(f);const provider=new Provider(),requests=[];
provider.complete=async(config,messages,project,signal)=>{
requests.push(project.id);if(respond)return respond({project,signal,f});
const owner=[...f.store.sessions.values()].find(session=>session.projectId===project.id),taskId=owner.taskId;
const plan=[...f.store.core.check_plan.values()].find(plan=>plan.taskId===taskId),job=[...f.store.core.check_job.values()].find(job=>job.taskId===taskId);
let calls=[];
if(!owner.changes.length)calls=[call(randomUUID(),'propose_file',{path:'src/add.mjs',content:'export const add = (a,b) => a + b;\n',reason:'本地固定响应中的加法修复'})];
else if(owner.changes.some(change=>change.status==='pending'))calls=[];
else if(!plan)calls=[call(randomUUID(),'plan_checks',{names:['test','build']})];
else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
return {message:{role:'assistant',content:calls.length?'':'固定响应声称完成,仍需实际证据',...(calls.length?{tool_calls:calls}:{})},usage:usage?{total_tokens:168,prompt_tokens:123,completion_tokens:45}:null};
};
const runtime=new Runtime(f.store,provider),manager=runtime.evaluationManager();f.onCleanup(()=>runtime.stopAll());return {...f,runtime,manager,requests};
}
async function prepared(t,options={}) {
const f=setup(t,options),original=ownerBytes(f),batch=await f.manager.plan(f.session,definition()),sample=batch.samples[0];
await f.manager.prepare(f.session,batch.id,batch.fingerprint,sample.id).promise;
assert.equal(sample.status,'ready',batch.message);return {...f,batch,sample,original,group:f.store.core.collaboration.get(sample.groupId)};
}
async function baseline(f){await f.manager.baseline(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.sample.status,'baseline_complete',f.batch.message);}
async function complete(f){await baseline(f);await f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.sample.status,'awaiting_review',f.batch.message);await applyWorkerProposals(f,f.runtime,f.group);await approveWorkerChecks(f,f.runtime,f.group,f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id));assert.equal(f.sample.status,'candidate_complete',f.batch.message);}
async function sealed(f){await complete(f);await f.manager.seal(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.ok(f.sample.report,f.batch.message);return f.manager.readReport(f.session,f.batch,f.sample);}
test('evaluation contract fixes bounded samples, source paths, checks and aggregate budget',t=>{
const f=fixture(t),valid=evaluationInput(definition(3),f.store.config);assert.equal(valid.samples.length,3);
for(const name of ['test/a.mjs','src/a.test.ts','src/config.ts','package.json','scripts/check.mjs','src','../evil.mjs'])assert.throws(()=>evaluationSourcePath(name));
for(const input of [{...definition(),command:'bypass'},{...definition(),samples:[]},{...definition(3),samples:[...definition(3).samples,definition().samples[0]]},{...definition(),budget:{maxSteps:1}},{...definition(),samples:[{...definition().samples[0],checks:['test','test']}]}])assert.throws(()=>evaluationInput(input,f.store.config));
});
test('same-source workspaces seal real commands before execution and preserve original staged and unstaged bytes',async t=>{
const f=await prepared(t);assert.deepEqual(ownerBytes(f),f.original);assert.equal(f.requests.length,0);
assert.equal(f.sample.commands[0].script,'node --test test/*.test.mjs');assert.ok(f.sample.contractSha256);
for(const child of f.group.children){const owner=f.store.session(child.sessionId);assert.throws(()=>proposeFile(f.store,owner,{path:'test/add.test.mjs',content:'pass',reason:'weaken test'}),/原版|范围/);assert.throws(()=>f.runtime.start(owner.id,'replace'),/受限|父协作/);}
const original=f.group.children[0],owner=f.store.session(original.sessionId);
assert.throws(()=>proposeFile(f.store,owner,{path:'src/add.mjs',content:'change',reason:'original'}),/原版/);
assert.throws(()=>f.runtime.workerManager().start(f.session,f.group.id,f.group.fingerprint,original.id),/原版/);
await assert.rejects(f.runtime.checkRunner().plan(f.store.session(f.group.children[1].sessionId),['test']),/完整检查/);
});
test('actual original failures and candidate success are both preserved, then adoption requires parent review, checks and Git',async t=>{
const f=await prepared(t),report=await sealed(f);assert.deepEqual(report.summary,{original:'failed',candidate:'passed'});
assert.equal(report.original.attempts[0].results[0].tests.failed,3);assert.equal(report.candidate.attempts[0].results[0].tests.passed,3);
assert.equal(report.original.usage.requests,0);assert.ok(report.candidate.usage.requests>0);assert.equal(report.inferenceValidated,false);assert.ok(report.candidate.approvals.some(approval=>approval.status==='approved'));
assert.deepEqual(ownerBytes(f),f.original);const source=fs.readFileSync(path.join(f.projectRoot,'src/add.mjs'));
await f.manager.adopt(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.batch.status,'awaiting_delivery',f.batch.message);assert.deepEqual(fs.readFileSync(path.join(f.projectRoot,'src/add.mjs')),source);
await assert.rejects(f.manager.finalize(f.session,f.batch.id,f.batch.fingerprint),/应用|变更集/);
const set=f.store.core.change_set.get(f.group.merge.changeSetId);await (await f.runtime.changeSetManager().start(f.session,set.id,set.fingerprint)).promise;assert.equal(set.status,'completed',set.error?.message);
const plan=await f.runtime.checkRunner().plan(f.session,['test','build']);assert.equal((await (await f.runtime.checkRunner().start(f.session,plan.id)).promise).status,'passed');
const delivery=await f.runtime.gitDelivery().plan(f.session,{message:'Apply evaluated addition candidate'});const result=await (await f.runtime.gitDelivery().start(f.session,delivery.id)).promise;assert.equal(result.status,'completed',result.error?.message);
await f.manager.finalize(f.session,f.batch.id,f.batch.fingerprint);assert.equal(f.batch.adoption.status,'completed');
const after=ownerBytes(f);assert.deepEqual(after.files,f.original.files);assert.notEqual(after.head,f.original.head);assert.match(deliveryGit(f,['diff','--cached','--name-only']),/owner-staged/);
});
test('candidate cannot shrink tests and changing original test bytes rejects execution before any model request',async t=>{
const f=await prepared(t);await baseline(f);const child=f.group.children[1],owner=f.store.session(child.sessionId);
fs.appendFileSync(path.join(f.store.project(child.projectId).root,'test/add.test.mjs'),'\n// changed\n');
await assert.rejects(verifyEvaluationSources(f.store,owner),/原测试/);
await f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.requests.length,0);assert.equal(f.sample.status,'failed');
await f.manager.seal(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;const report=f.manager.readReport(f.session,f.batch,f.sample);assert.equal(report.inputVerified,false);assert.equal(report.summary.candidate,'unknown');
assert.throws(()=>f.manager.adopt(f.session,f.batch.id,f.batch.fingerprint,f.sample.id),/通过/);
});
test('missing upstream usage remains unknown and does not become a free run or invented cost',async t=>{
const f=await prepared(t,{usage:false}),report=await sealed(f);assert.ok(report.candidate.usage.unknownRequests>0);assert.equal(report.candidate.usage.reported,false);assert.equal(report.candidate.usage.cost,null);assert.ok(report.candidate.usage.estimatedUnknownTokens>0);
});
test('missing raw check logs preserve unknown result and original sample denominator',async t=>{
const f=await prepared(t);await complete(f);const job=f.store.core.check_job.get(f.sample.baselineJobId);f.store.core.artifact.delete(job.results[0].logArtifactId);
await f.manager.seal(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;const report=f.manager.readReport(f.session,f.batch,f.sample);assert.equal(report.summary.original,'unknown');assert.ok(report.original.evidenceError);assert.equal(evaluationSummary(f.batch).samples,1);
assert.throws(()=>f.manager.adopt(f.session,f.batch.id,f.batch.fingerprint,f.sample.id),/原版结果完整/);
});
test('source and model changes never silently replace an existing batch input',async t=>{
const f=setup(t),batch=await f.manager.plan(f.session,definition());fs.appendFileSync(path.join(f.projectRoot,'README.md'),'owner later');
await f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[0].id).promise;assert.equal(batch.samples[0].status,'failed');assert.equal(f.store.core.worker.size,0);assert.equal(f.requests.length,0);
});
test('model revision change before preparing a fixed sample cannot silently use the replacement connection',async t=>{
const f=setup(t),batch=await f.manager.plan(f.session,definition());f.store.saveConfig({model:'replacement-connection'});
await f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[0].id).promise;assert.equal(batch.samples[0].status,'failed');assert.equal(f.requests.length,0);assert.equal(f.store.core.worker.size,0);
});
test('HTTP evaluations require host token, original fingerprint and session ownership; worker API cannot bypass evaluation flow',async t=>{
const f=fixture(t),host=await workerHttp(f);assert.equal((await host.raw('/sessions/'+f.session.id+'/evaluations',definition(),'wrong-token')).status,403);
const {batch}=await host.api('/sessions/'+f.session.id+'/evaluations',definition()),sample=batch.samples[0];
assert.equal((await host.raw('/sessions/'+f.session.id+'/evaluations/'+batch.id+'/prepare',{sampleId:sample.id})).status,409);
assert.equal((await host.raw('/sessions/'+f.session.id+'/evaluations/'+batch.id+'/prepare',{fingerprint:batch.fingerprint,sampleId:sample.id,command:'extra'})).status,409);
const other=f.store.createSession(f.project.id);assert.equal((await host.raw('/sessions/'+other.id+'/evaluations/'+batch.id+'/prepare',{fingerprint:batch.fingerprint,sampleId:sample.id})).status,404);
await host.api('/sessions/'+f.session.id+'/evaluations/'+batch.id+'/prepare',{fingerprint:batch.fingerprint,sampleId:sample.id});await until(()=>!host.runtime.evaluationManager().isActive(f.session.id),20000);
const saved=f.store.core.evaluation.get(batch.id);assert.equal(saved.samples[0].status,'ready',saved.message);const group=f.store.core.collaboration.get(saved.samples[0].groupId);
assert.equal((await host.raw('/sessions/'+group.children[1].sessionId+'/evaluations',definition())).status,409);
assert.equal((await host.raw('/sessions/'+f.session.id+'/collaborations/'+group.id+'/start',{fingerprint:group.fingerprint,workerId:group.children[1].id})).status,409);
assert.equal(host.requests.length,0);
});
test('sample queue cannot skip or reset attempted samples, and reports freeze further writes',async t=>{
const f=setup(t),batch=await f.manager.plan(f.session,definition(2));assert.throws(()=>f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[1].id),/顺序/);
const sample=batch.samples[0];await f.manager.prepare(f.session,batch.id,batch.fingerprint,sample.id).promise;assert.equal(sample.status,'ready',batch.message);
assert.throws(()=>f.manager.prepare(f.session,batch.id,batch.fingerprint,sample.id),/尝试/);
await f.manager.seal(f.session,batch.id,batch.fingerprint,sample.id).promise;const child=sample.children[1],owner=f.store.session(child.sessionId);
assert.throws(()=>proposeFile(f.store,owner,{path:'src/add.mjs',content:'change',reason:'sealed'}),/封存/);
await f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[1].id).promise;assert.equal(batch.samples[1].status,'ready',batch.message);
f.manager.retain(f.session,batch.id,batch.fingerprint);assert.equal(evaluationSummary(batch).samples,2);assert.equal(evaluationSummary(batch).finished,1);
});
test('stop cancels only current candidate and preserves partial results instead of replaying a request',async t=>{
const f=await prepared(t,{respond:({signal})=>new Promise((resolve,reject)=>signal.addEventListener('abort',()=>reject(signal.reason),{once:true}))});await baseline(f);
const started=f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id);await until(()=>f.requests.length===1);await f.runtime.stop(f.session.id);await started.promise;
assert.equal(f.requests.length,1);assert.equal(f.runtime.hasActivity(),false);assert.notEqual(f.sample.status,'candidate_complete');assert.deepEqual(ownerBytes(f),f.original);
await f.manager.seal(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;
const report=f.manager.readReport(f.session,f.batch,f.sample);
assert.equal(report.summary.candidate,'stopped');assert.equal(report.inputVerified,false);
assert.equal(report.environmentComparison.matched,false);assert.equal(report.original.attempts[0].status,'failed');assert.equal(f.requests.length,1);
});
test('restart recovers interrupted evaluations as unknown and keeps completed checks and plan identity',async t=>{
const f=await prepared(t);await baseline(f);const fingerprint=f.batch.fingerprint,jobId=f.sample.baselineJobId;
f.batch.status='candidate_running';f.sample.status='candidate_running';f.store.save(f.session);f.store.close();
const reopened=new Store(f.data);f.onCleanup(()=>reopened.close());const saved=reopened.core.evaluation.get(f.batch.id);assert.equal(saved.status,'unknown');assert.equal(saved.samples[0].status,'unknown');assert.equal(saved.fingerprint,fingerprint);assert.equal(saved.samples[0].baselineJobId,jobId);assert.equal(reopened.core.check_job.get(jobId).status,'failed');
});