Files
Agent-Workbench/scripts/verify-working-memory.mjs
T

65 lines
7.6 KiB
JavaScript

// A local HTTP fixture exercises the shipped host. No production config or
// credentials, no implementation supplied, no real-model delivery claim.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {randomUUID,createHash} from 'node:crypto';
import {execFileSync} from 'node:child_process';
import {Store} from '../src/server/store.mjs';
import {verifyPlanEvidence} from '../src/core/context/request-plan.mjs';
import {compatibleFixture,completion,call,until,assertToolPairs,launchWorkbench} from '../tests/helpers.mjs';
const root=path.resolve('.qa','working-memory-host-'+randomUUID()),data=path.join(root,'data'),projectRoot=path.join(root,'project');
const appRoot=path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT??'.');
const input=JSON.parse(fs.readFileSync(new URL('../tests/fixtures/working-memory-t.json',import.meta.url)));
const sha=value=>createHash('sha256').update(value).digest('hex'),disposers=[],t={after:fn=>disposers.push(fn)};
fs.mkdirSync(projectRoot,{recursive:true});
for(const file of input.files){assert.equal(sha(file.content),file.sha256);const target=path.resolve(projectRoot,file.path);assert.ok(target.startsWith(projectRoot+path.sep));fs.mkdirSync(path.dirname(target),{recursive:true});fs.writeFileSync(target,file.content);}
const git=args=>execFileSync('git',['-C',projectRoot,...args],{encoding:'utf8',windowsHide:true});
git(['init','--initial-branch=main']);git(['config','core.autocrlf','false']);git(['add','.']);git(['-c','user.name=QA','-c','user.email=qa@workbench.invalid','commit','--no-gpg-sign','-m','Original read-only T fixture']);
const requests=[],facts=[],checks=[],files=input.files.filter(f=>['test/expense.test.mjs','src/amount.mjs','src/summary.mjs','src/report.mjs'].includes(f.path));
let host,sessionId,report,version,extraInspection=false;
try{
const model=await compatibleFixture(t,async(_route,body)=>{
assertToolPairs(body.messages);requests.push(body);const number=requests.length;assert.ok(number<=8);assert.ok(body.messages[0].content.includes(JSON.stringify(input.goal)));
if(number<=3){const message=structuredClone(input.replies[number-1]);for(const tool of message.tool_calls??[])if(tool.function.name==='run_checks'){
const current=await host.api('/sessions/'+sessionId);tool.function.arguments=JSON.stringify({plan_id:current.execution.plans.at(-1).id});}
return{body:{...completion(),choices:[{message,finish_reason:'tool_calls'}]}};}
const marker='当前工程执行依据(每次请求重新核对,非操作授权):\n';
const fact=JSON.parse(body.messages[0].content.split(marker)[1].split('\n')[0]);facts.push(fact);
assert.equal(fact.has_applied_changes,false);assert.equal(fact.stage,'checks_need_repair');assert.equal(fact.latest_check.current_source,true);
assert.equal(fact.latest_check.results[0].tests.failed,9);assert.equal(fact.latest_check.results[1].status,'passed');
const original=await host.api('/sessions/'+sessionId+'/export'),pages=original.session.providerMessages.filter(m=>m.role==='tool').map(m=>JSON.parse(m.content)).filter(v=>v.path==='test/expense.test.mjs'&&typeof v.content==='string');
const text=files.find(f=>f.path==='test/expense.test.mjs').content,covered=new Set();
for(const p of pages){assert.equal(p.content,text.slice(p.offset,p.next_offset));for(let n=p.offset;n<p.next_offset;n++)covered.add(n);}
let next=0;while(covered.has(next))next++;
if(next<text.length){const latest=pages.at(-1);return{body:completion([call('u-continue-'+number,'read_file',{path:latest.path,start_line:latest.range_start_line,end_line:latest.range_end_line,offset:next,expected_sha256:latest.sha256,limit:16000})])};}
for(const file of files){const positions=new Set();for(const m of body.messages){let view;
if(m.role==='tool'){const v=JSON.parse(m.content);if(v.path===file.path&&typeof v.content==='string')view={offset:v.offset??0,content:v.content};}
if(view){assert.equal(view.content,file.content.slice(view.offset,view.offset+view.content.length));for(let n=view.offset;n<view.offset+view.content.length;n++)positions.add(n);}
if(m.role==='system'){let v;try{v=JSON.parse(m.content.split('\n').at(-1));}catch{}for(const e of v?.data?.entries??[])if(e.observation?.path===file.path){const start=e.observation.offset??0;assert.equal(e.content,file.content.slice(start,start+e.content.length));for(let n=start;n<start+e.content.length;n++)positions.add(n);}}}
assert.equal(positions.size,file.content.length,'Actual request lost previously returned file bytes: '+file.path);}
checks.push('request '+number+': all four original files and current nine failures/build pass actually present');
if(!extraInspection){extraInspection=true;return{body:completion([call('u-inspect-original-state','inspect_session',{})])};}
return{body:completion([],'Offline read-only coverage verified; no engineering delivery claimed.')};
});
const store=new Store(data),project=store.addProject(projectRoot),session=store.createSession(project.id);sessionId=session.id;
store.saveConfig({mode:'compatible',baseUrl:model.baseUrl,model:'working-memory-fixture',contextLimit:16000,maxOutputTokens:2048,maxSteps:8,maxRunTokens:90000,maxTaskTokens:90000,maxRunDurationMs:600000});store.close();
host=await launchWorkbench(t,data,{}, {appRoot});assert.notEqual(new URL(host.url).port,'4318');version=(await host.api('/health')).version;
await host.api('/sessions/'+sessionId+'/messages',{content:input.goal});
let current;
await until(async()=>{current=await host.api('/sessions/'+sessionId);for(const approval of current.approvals.filter(a=>a.status==='pending')){
assert.ok(approval.checkPlanId);assert.deepEqual(approval.checkScripts.map(value=>value.name),['test','build']);
assert.equal(approval.command,'npm run test\nnpm run build');await host.api('/sessions/'+sessionId+'/approvals/'+approval.id,{allow:true});}
return !['running','awaiting_approval','stopping'].includes(current.status);},60000);
assert.equal(current.execution.run.stopReason,'verification_incomplete',JSON.stringify(current.lastFailure));assert.ok(extraInspection);assert.ok(checks.length>=2);assert.equal(current.changes.length,0);
const exported=await host.api('/sessions/'+sessionId+'/export');fs.writeFileSync(path.join(root,'export.json'),JSON.stringify(exported,null,2));
await host.close();host=null;
const reopened=new Store(data);try{const saved=reopened.session(sessionId);for(const plan of reopened.core.context_plan.values()){
verifyPlanEvidence(reopened,saved,plan);assert.equal(plan.budget.contextLimit,16000);assert.equal(plan.budget.outputReservation,2048);assert.equal(plan.budget.safetyTokens,1600);assert.ok(plan.estimatedInputTokens<=12352);}
checks.push('all actual request artifacts independently verified after reopening');}finally{reopened.close();}
for(const file of input.files)assert.equal(sha(fs.readFileSync(path.join(projectRoot,file.path))),file.sha256);assert.equal(git(['status','--porcelain']),'');
report={success:true,version,appRoot,checks,errors:[],requests:requests.length,facts,originalSourceUnchanged:true,originalNineTestsFailed:true,realModelDeliveryPassed:false,purgeBenefitMeasured:false,realModelRequests:0};
}catch(error){report={success:false,appRoot,checks,error:error.stack,requests:requests.length,realModelRequests:0};process.exitCode=1;}
finally{await host?.close();for(const dispose of disposers.reverse())await dispose();fs.writeFileSync(path.join(root,'requests.json'),JSON.stringify(requests,null,2));fs.writeFileSync(path.join(root,'result.json'),JSON.stringify(report,null,2));console.log(JSON.stringify({success:report.success,appRoot,root,checks:checks.length,requests:requests.length,realModelRequests:0,error:report.error}));}