Files
Agent-Workbench/scripts/verify-original-purge-host.mjs
2026-11-02 22:32:31 +08:00

102 lines
9.2 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {randomUUID} from 'node:crypto';
import {fixture,compatibleFixture,call,until} from '../tests/helpers.mjs';
import {createDeliveryProject} from '../tests/fixtures/delivery-project.mjs';
import {ownerBytes} from '../tests/fixtures/workers-project.mjs';
import {Provider} from '../src/server/provider.mjs';
import {Runtime} from '../src/server/runtime.mjs';
import {EngineRunner} from '../src/core/engines/engine-runner.mjs';
import {resolveControlledInstallation} from '../src/core/engines/controlled-installation.mjs';
import {resolveOriginalPurgeSource,ORIGINAL_PURGE_KIND,assertPurgeModelInput,PURGE_DEFAULT_PROMPT} from '../src/core/engines/purge-strategy.mjs';
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
const directory=path.resolve('.qa','original-purge-host-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
const installation=resolveControlledInstallation(process.cwd(),process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT);
assert.equal(installation.controlledStatus,'available',installation.message);
const source=resolveOriginalPurgeSource(process.cwd(),process.env.AGENT_WORKBENCH_PURGE_178_ROOT,installation);assert.equal(source.status,'available',source.message);
const results=[];
const usage={prompt_tokens:123,completion_tokens:45,total_tokens:168};
async function sample(scenario){
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),sample=path.join(directory,scenario);fs.mkdirSync(sample);
let runtime,group,release,started,reported;let first=true;
try{
createDeliveryProject(f);const original=ownerBytes(f),copy=path.join(f.root,'declared-original-source');
fs.mkdirSync(copy);for(const input of source.inputs){const file=path.join(copy,input.relative);fs.mkdirSync(path.dirname(file),{recursive:true});fs.copyFileSync(input.path,file);}
const selected=resolveOriginalPurgeSource(process.cwd(),copy,installation);assert.equal(selected.status,'available',selected.message);
const gate=new Promise(resolve=>{release=resolve;});
const model=await compatibleFixture(t,async(_route,_input,requests)=>{
if(scenario==='stop-in-flight')await gate;
if(scenario==='reported-budget-overrun')return {status:503,body:{error:{message:'Owned upstream 503 with reported usage'},usage:{prompt_tokens:55000,completion_tokens:17,total_tokens:55017}}};
const tools=[];
if(scenario==='source-changed-at-approval'){
const worker=group.children[0],plan=[...f.store.core.check_plan.values()].find(item=>item.taskId===worker.taskId);
tools.push(plan?call('owned-check','run_checks',{plan_id:plan.id}):call('owned-plan','plan_checks',{names:['test','build']}));
}else if(scenario==='denied-scope'&&requests.length===1)tools.push(call('owned-outside','propose_file',{path:'../outside.mjs',content:'denied',reason:'Declared negative scope test'}));
return {choices:[{message:{role:'assistant',content:tools.length?'':'Owned host fixture terminal.',...(tools.length?{tool_calls:tools}:{})},finish_reason:tools.length?'tool_calls':'stop'}],usage};
});
f.store.saveConfig({mode:'compatible',model:'owned-original-host',baseUrl:model.baseUrl,contextLimit:32000,maxOutputTokens:1024});
const provider=new Provider();provider.setKey('owned-original-host-http-fixture');runtime=new Runtime(f.store,provider);
runtime.controlledEngines=new EngineRunner(f.store,provider,{directoryRoot:path.join(f.root,'owned-engines')});
const manager=runtime.workerManager();group=await manager.plan(f.session,{goal:'Verify original Purge '+scenario,
engine:{id:'deepseek-harness',root:installation.root,controlledSha256:installation.controlledSha256,
strategy:{kind:ORIGINAL_PURGE_KIND,root:copy,sourceSha256:selected.sourceSha256,runtimeSha256:selected.runtimeSha256,prompt:PURGE_DEFAULT_PROMPT}},
budget:{maxTokens:scenario==='reported-budget-overrun'?35000:150000,maxSteps:12,maxToolCalls:20,maxDurationMs:120000},
children:[{label:'original-1',goal:'Keep the original project and host controls',paths:['src/add.mjs']}]});
await (await manager.prepare(f.session,group.id,group.fingerprint)).promise;assert.equal(group.status,'ready',group.summary);
const worker=group.children[0],session=f.store.session(worker.sessionId),task=f.store.core.task.get(worker.taskId);
started=manager.start(f.session,group.id,group.fingerprint);
if(scenario==='stop-in-flight'){
await until(()=>model.requests.length===1,45000);await runtime.stop(session.id);release();
}
if(scenario==='source-changed-at-approval'){
await until(()=>session.approvals.some(item=>item.status==='pending'),45000);
fs.appendFileSync(path.join(copy,'lib/host.js'),'\n// Declared source drift during approval\n');
const approval=session.approvals.find(item=>item.status==='pending');runtime.decide(session.id,approval.id,true);
}
await started.promise;release();
const jobs=[...f.store.core.engine_run.values()].filter(job=>job.taskId===worker.taskId),job=jobs.at(-1),run=f.store.core.run.get(job.runId);
assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true);assert.deepEqual(ownerBytes(f),original);
originalEngineRun(f.store,session,job);
for(const request of job.requests){const input=verifyEngineModelInput(f.store,session,job,request);assertPurgeModelInput(input.native,job.installation.strategy);}
assert.ok(model.requests.length>0,'The owned native request must reach the declared HTTP fixture.');
assert.equal(task.usage.requests,model.requests.length);assert.equal((run.consumption??[]).length,model.requests.length);
if(scenario==='reported-budget-overrun'){
assert.equal(model.requests.length,1);assert.equal(task.usage.totalTokens,55017);assert.equal(run.usage.unknownRequests,0);assert.equal(run.stopReason,'token_limit');assert.equal(job.calls.length,0);
}
if(scenario==='denied-scope'){
assert.equal(model.requests.length,2);assert.equal(job.calls.length,1);assert.equal(session.changes.length,0);assert.equal(task.usage.totalTokens,336);
assert.equal(fs.existsSync(path.join(f.store.project(worker.projectId).root,'..','outside.mjs')),false);
}
if(scenario==='stop-in-flight'){
assert.equal(model.requests.length,1);assert.equal(job.calls.length,0);assert.equal(job.status,'cancelled');assert.equal(run.usage.unknownRequests,1);assert.equal(task.usage.unknownRequests,1);
const oldJob=JSON.stringify(job),oldUsage=structuredClone(task.usage),oldRun=JSON.stringify(run);
await (await manager.start(f.session,group.id,group.fingerprint)).promise;
const resumed=[...f.store.core.engine_run.values()].filter(value=>value.taskId===worker.taskId).at(-1),resumedRun=f.store.core.run.get(resumed.runId);
assert.notEqual(resumed.id,job.id);assert.notEqual(resumed.runId,job.runId);assert.equal(resumed.taskId,job.taskId);assert.equal(resumed.status,'completed',resumed.error);
assert.equal(model.requests.length,2);assert.equal(task.usage.requests,2);assert.equal(task.usage.unknownRequests,1);assert.equal(task.usage.totalTokens,oldUsage.totalTokens+168);
assert.equal(task.usage.estimatedUnknownTokens,oldUsage.estimatedUnknownTokens);assert.equal(JSON.stringify(job),oldJob);assert.equal(JSON.stringify(run),oldRun);
assert.equal(resumedRun.consumption.length,1);assert.equal(resumed.native.cleanup.treeConfirmed,true);assert.equal(resumed.calls.length,0);
originalEngineRun(f.store,session,resumed);reported={resumedJob:resumed,resumedRun,originalStopUsage:oldUsage};
}
if(scenario==='source-changed-at-approval'){
assert.equal(model.requests.length,2);assert.equal(f.store.core.check_job.size,0);assert.match(job.error,/已变化/);assert.equal(task.usage.totalTokens,336);
}
reported={...reported,scenario,passed:true,group,task,run,job,modelRequests:model.requests,realModelRequests:0};
console.log('PASS '+scenario+'; HTTP '+model.requests.length+'; task reported '+task.usage.totalTokens+'; unknown '+task.usage.unknownRequests);
}catch(error){reported={scenario,passed:false,error:error.stack,group,realModelRequests:0};throw error;}
finally{
release?.();if(started)await started.promise.catch(()=>{});if(runtime)await runtime.stopAll();
f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');fs.writeFileSync(path.join(sample,'result.json'),JSON.stringify(reported,null,2));
fs.cpSync(f.root,path.join(sample,'original-fixture'),{recursive:true,filter:file=>!file.split(path.sep).includes('node_modules')});
for(const dispose of disposers.reverse())await dispose();
}
return {scenario,passed:true};
}
let report;
try{for(const scenario of ['reported-budget-overrun','denied-scope','stop-in-flight','source-changed-at-approval'])results.push(await sample(scenario));
report={passed:true,directory,results,realModelRequests:0,scope:'Actual pinned Harness and production host routing with owned HTTP fixtures. No real model reasoning or Purge benefit.'};
}catch(error){process.exitCode=1;report={passed:false,directory,results,error:error.stack,realModelRequests:0};}
fs.writeFileSync(path.join(directory,'validation.json'),JSON.stringify(report,null,2));console.log(JSON.stringify(report));