test(live): prepare independent AE React and Python acceptance
This commit is contained in:
@@ -6,6 +6,7 @@ import { DatabaseSync } from 'node:sqlite';
|
||||
import { Agent, fetch } from 'undici';
|
||||
|
||||
export let root = path.resolve('.qa/live-validation-ad'), origin = 'http://127.0.0.1:4318';
|
||||
export function engineeringEvidence() { root = path.resolve('.qa/live-engineering-ae'); }
|
||||
// Used only by the isolated local driver verifier, never by a real batch.
|
||||
export function configureFixture(options) {
|
||||
const target = path.resolve(options.root), url = new URL(options.origin);
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import fs from 'node:fs';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { engineeringEvidence, root, read, save, client, publicMetadata } from '../live-ad/common.mjs';
|
||||
import { publicSnapshot } from '../retained-host-state.mjs';
|
||||
import { inspectorClosed } from '../live-ad/inspector.mjs';
|
||||
import { specs, python } from './project.mjs';
|
||||
const ad=read('summary.json');assert.equal(ad.node.success,true);assert.equal(ad.originalStateAudit.success,true);
|
||||
engineeringEvidence();assert.equal(fs.existsSync(root),false,'Freeze a new AE batch only once.');assert.equal(execFileSync('git',['status','--porcelain'],{encoding:'utf8',windowsHide:true}).trim(),'');
|
||||
const{api,boot,close}=await client();
|
||||
try{
|
||||
await inspectorClosed();const host=await api('/host/status'),center=await api('/model-center');assert.equal(boot.version,'1.25.8');assert.equal(host.activity.active,false);assert.equal(host.desktopWindows.count,0);
|
||||
for(const[key,value]of Object.entries({approvalMode:'full',executionMode:'unrestricted',capacityMode:'passthrough',maxSteps:0,maxRunTokens:0,maxTaskTokens:0}))assert.equal(boot.config[key],value);
|
||||
const connection=center.connections.find(value=>value.id==='1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd');assert.equal(connection.model,'deepseek-flash');assert.equal(connection.hasKey,true);assert.equal(connection.enabled,true);
|
||||
const baseline=await publicSnapshot('http://127.0.0.1:4318',host.identity.dataRoot);
|
||||
save('current.json',{host,config:boot.config,center,connection,originalProjects:boot.projects,originalPublicMetadata:publicMetadata(host.identity.dataRoot),baseline,specs,python,adNodeCommit:ad.node.commit,
|
||||
driverCommit:execFileSync('git',['rev-parse','HEAD'],{encoding:'utf8',windowsHide:true}).trim()});
|
||||
console.log(JSON.stringify({frozen:true,pid:host.identity.pid,version:boot.version,connectionId:connection.id,revision:connection.revision,model:connection.model,projects:baseline.projects,sessions:baseline.database.sessionIds.length,modelRequests:0}));
|
||||
}finally{await close();}
|
||||
@@ -0,0 +1,53 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import { gitEnvironment } from '../../src/core/git-process.mjs';
|
||||
import { sha } from '../live-ad/common.mjs';
|
||||
export const python = path.resolve('.qa/reliability-1.25.2/python-env/Scripts/python.exe');
|
||||
export const specs = {
|
||||
react: { baseline:'benchmarks/react-delivery-v1/baseline',allowed:['src/query.ts','src/summary.ts','src/transition.ts','src/App.tsx'],tests:9 },
|
||||
python: { baseline:'benchmarks/python-delivery-v2/baseline',allowed:['normalize.py','summary.py','render.py'],tests:10 },
|
||||
};
|
||||
export function prepareProject(kind, output, save) {
|
||||
const spec = specs[kind], sourceRoot=path.join(output,'project'); assert.ok(spec); assert.equal(fs.existsSync(sourceRoot),false);
|
||||
fs.mkdirSync(output,{recursive:true}); fs.cpSync(spec.baseline,sourceRoot,{recursive:true,errorOnExist:true});
|
||||
const git = args=>execFileSync('git',['--no-pager','--no-optional-locks','-C',sourceRoot,...args],{encoding:'utf8',windowsHide:true,env:gitEnvironment()});
|
||||
const check = (name, executable, args, prefix)=>{
|
||||
const execution=spawnSync(executable,args,{cwd:sourceRoot,encoding:'utf8',windowsHide:true,env:gitEnvironment(),maxBuffer:32*1024*1024});
|
||||
save(prefix+'/'+name+'.txt',(execution.stdout??'')+(execution.stderr??''));
|
||||
let stats=null;
|
||||
if(name==='test'&&kind==='react') stats=Object.fromEntries(['tests','pass','fail','cancelled','skipped','todo'].map(key=>[key,Number(execution.stdout?.match(new RegExp('^# '+key+' (\\d+)$','m'))?.[1]??NaN)]));
|
||||
if(name==='test'&&kind==='python') {
|
||||
const report=path.join(sourceRoot,args.at(-1).slice('--junitxml='.length));
|
||||
if(fs.existsSync(report)){const xml=fs.readFileSync(report,'utf8');save(prefix+'/pytest.xml',xml);const tag=xml.match(/<testsuite\s[^>]*>/)?.[0]??'';
|
||||
const attr=key=>Number(tag.match(new RegExp('\\b'+key+'="(\\d+)"'))?.[1]??NaN);stats={tests:attr('tests'),fail:attr('failures')+attr('errors'),skipped:attr('skipped')};stats.pass=stats.tests-stats.fail-stats.skipped;}
|
||||
}
|
||||
return {status:execution.status,signal:execution.signal,error:execution.error?.message??null,stats,stdoutSha256:sha(execution.stdout??''),stderrSha256:sha(execution.stderr??'')};
|
||||
};
|
||||
if(!fs.existsSync(path.join(sourceRoot,'.gitignore')))fs.writeFileSync(path.join(sourceRoot,'.gitignore'),'node_modules/\ndist/\n.awb-*\n__pycache__/\n.pytest_cache/\n',{flag:'wx'});
|
||||
for(const name of ['owner-staged.txt','owner-unstaged.txt'])fs.writeFileSync(path.join(sourceRoot,name),'Owner baseline\n',{flag:'wx'});
|
||||
if(kind==='react'){
|
||||
const npm=path.join(path.dirname(process.execPath),'node_modules/npm/bin/npm-cli.js');
|
||||
const install=check('npm-ci',process.execPath,[npm,'ci','--offline','--ignore-scripts','--no-audit','--no-fund'],'prepare');assert.equal(install.status,0,'Offline dependency preparation failed before any model request.');
|
||||
}
|
||||
git(['init','--initial-branch=main']);git(['config','core.autocrlf','false']);git(['config','user.name','Workbench QA']);git(['config','user.email','qa@workbench.invalid']);
|
||||
fs.appendFileSync(path.join(sourceRoot,'.git/info/exclude'),'\n.awb-*\n__pycache__/\n.pytest_cache/\n');
|
||||
git(['add','--','.']);git(['commit','--no-gpg-sign','-m','Seal AE '+kind+' original baseline']);const initialHead=git(['rev-parse','HEAD']).trim();
|
||||
fs.writeFileSync(path.join(sourceRoot,'owner-staged.txt'),'Owner staged change\n');git(['add','--','owner-staged.txt']);
|
||||
fs.writeFileSync(path.join(sourceRoot,'owner-staged.txt'),'Owner unstaged over original staging\n');fs.writeFileSync(path.join(sourceRoot,'owner-unstaged.txt'),'Owner unstaged change\n');fs.writeFileSync(path.join(sourceRoot,'owner-untracked.txt'),'Owner untracked change\n',{flag:'wx'});
|
||||
const all=git(['ls-files','-z']).split('\0').filter(Boolean),protectedNames=all.filter(name=>!spec.allowed.includes(name)).concat('owner-untracked.txt');
|
||||
const protectedBytes=Object.fromEntries(protectedNames.map(name=>[name,fs.readFileSync(path.join(sourceRoot,name))]));
|
||||
const index=git(['ls-files','--stage','--',...protectedNames]);
|
||||
const protect=()=>{for(const[name,bytes]of Object.entries(protectedBytes))assert.deepEqual(fs.readFileSync(path.join(sourceRoot,name)),bytes,'Protected '+name);assert.equal(git(['ls-files','--stage','--',...protectedNames]),index,'Protected original index');};
|
||||
const checks=prefix=>{
|
||||
const result=kind==='react'?{
|
||||
test:check('test',process.execPath,['--test','--test-reporter=tap','test/domain.test.mjs'],prefix),
|
||||
typecheck:check('typecheck',process.execPath,['node_modules/typescript/bin/tsc','--noEmit'],prefix),
|
||||
build:check('build',process.execPath,['node_modules/vite/bin/vite.js','build'],prefix),
|
||||
}:{test:check('test',python,['-m','pytest','-q','--junitxml=.awb-qa-'+prefix.replaceAll('/','-')+'.xml'],prefix),build:check('build',python,['-m','compileall','-q',...spec.allowed],prefix)};
|
||||
save(prefix+'/result.json',result);protect();return result;
|
||||
};
|
||||
save('source-before.json',{initialHead,status:git(['status','--porcelain=v1']),protectedIndex:index,files:Object.fromEntries([...all,'owner-untracked.txt'].map(name=>[name,sha(fs.readFileSync(path.join(sourceRoot,name)))]))});
|
||||
return {spec,sourceRoot,git,initialHead,protect,checks};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { engineeringEvidence,root,save,read,client,sha,captureSession,connectionDefinition,connectionInput,compareConfig } from '../live-ad/common.mjs';
|
||||
import { prepareProject,python,specs } from './project.mjs';
|
||||
import { verifyLiveReactClient } from '../live-react-client.mjs';
|
||||
engineeringEvidence();const kind=process.argv[2];assert.ok(['react','python'].includes(kind));const frozen=read('current.json');
|
||||
if(kind==='python')assert.equal(read('react/result.json').success,true,'React must fully pass before Python.');
|
||||
assert.equal(fs.existsSync(path.join(root,kind)),false,'Never replay an AE sample.');
|
||||
const{api,boot,close}=await client();let session,route,configured,selected=false,original,project,own,failure,result,state;const observations=[],facts=new Set();
|
||||
const output=path.join(root,kind),store=(file,value)=>save(kind+'/'+file,value);
|
||||
const settle=async()=>{if(route){await api(route+'/stop',{},60000);state=await api(route);assert.ok(!['running','stopping','awaiting_approval'].includes(state.status));}};
|
||||
try{
|
||||
const host=await api('/host/status');assert.equal(host.activity.active,false);assert.equal(host.desktopWindows.count,0);assert.equal(boot.version,'1.25.8');compareConfig(frozen.config,boot.config);
|
||||
const center=await api('/model-center');assert.deepEqual(center.selected,frozen.center.selected);assert.deepEqual(center.pools,frozen.center.pools);
|
||||
original=center.connections.find(value=>value.id===frozen.connection.id);const expected=kind==='react'?frozen.connection:read('react/restoration.json').connection;
|
||||
assert.deepEqual(connectionDefinition(original),connectionDefinition(expected));
|
||||
store('contract.json',{kind,connection:original,config:boot.config,source:specs[kind].baseline,allowed:specs[kind].allowed,observationMs:600000,noNewFactMs:180000,implementationSupplied:false,supervisorContinuation:false,localTokenCaps:false});
|
||||
own=prepareProject(kind,output,store);const baseline=own.checks('baseline');assert.equal(baseline.test.stats.tests,own.spec.tests);assert.ok(baseline.test.stats.fail>0);assert.equal(baseline.build.status,0);
|
||||
project=await api('/projects',{root:own.sourceRoot});session=await api('/sessions',{projectId:project.id});route='/sessions/'+session.id;store('owned.json',{project,session});
|
||||
configured=await api('/model-center/connections',{...connectionInput(original),modelIdentityPolicy:'require-match'});assert.equal(configured.credentialRevision,original.credentialRevision);assert.equal(configured.hasKey,original.hasKey);
|
||||
await api('/model-center/select',{type:'connection',id:original.id});selected=true;
|
||||
const goal=kind==='react'?'独立完成这个React/TypeScript看板工程README要求,修复状态筛选、去空白搜索、实际统计、按目标任务切换与空态,保留既有样式和布局。只修改src/query.ts、src/summary.ts、src/transition.ts、src/App.tsx。自行读取源码及原九项测试、运行test/typecheck/build并完成仅这四文件的本地Git提交。依赖已按原锁离线准备;不可改原测试、配置、锁、其它源码/样式、README和owner文件,保留原暂存/未暂存/未跟踪状态。':'独立完成这个Python任务报表工程README的规范化、汇总与渲染契约。只修改normalize.py、summary.py、render.py。自行读取源码及原十项测试,运行pytest与源码编译并完成仅这三文件的本地Git提交。现成解释器为 '+python+',已装pytest,无需安装依赖。不可改原测试、配置、README和owner文件,保留原暂存/未暂存/未跟踪状态。';
|
||||
const prompt=goal+' 当前为完全访问权限,可用工程工具或终端自动执行,无需独立审查,不要只解释方案或口头完成,不推送远端。';store('goal.txt',prompt);await api(route+'/messages',{content:prompt});const started=Date.now();let novel=started,signature='';
|
||||
while(true){state=await api(route);own.protect();for(const event of state.events.filter(value=>value.kind==='tool'&&value.phase==='completed'&&['read_file','read_artifact','list_files','query_code','search_files'].includes(value.tool))){const key=event.tool+JSON.stringify(event.input)+sha(event.output??'');if(!facts.has(key)){facts.add(key);novel=Date.now();}}
|
||||
const current=JSON.stringify({files:own.spec.allowed.map(name=>sha(fs.readFileSync(path.join(own.sourceRoot,name)))),checks:state.execution.checks.map(value=>({status:value.status,source:value.sourceFingerprint}))});if(current!==signature){signature=current;novel=Date.now();}
|
||||
const observation={kind,status:state.status,requests:state.execution.task?.usage.requests,tokens:state.execution.task?.usage.totalTokens,facts:facts.size,changes:state.changes.length,checks:state.execution.checks.map(value=>value.status),lastTool:state.events.filter(value=>value.kind==='tool').at(-1)?.tool};if(JSON.stringify(observation)!==JSON.stringify(observations.at(-1))){observations.push(observation);console.log(JSON.stringify(observation));}
|
||||
if(!['running','stopping','awaiting_approval'].includes(state.status))break;
|
||||
if(Date.now()-started>600000||Date.now()-novel>180000){failure={reason:'experiment_observation_ended',elapsedMs:Date.now()-started,noNewFactMs:Date.now()-novel,productBudgetChanged:false};break;}await delay(750);
|
||||
}
|
||||
await settle();own.protect();const final=own.checks('independent-final');let browser=null;
|
||||
if(kind==='react'&&Object.values(final).every(value=>value.status===0)){try{browser=await verifyLiveReactClient(own.sourceRoot,path.join(output,'browser'));}catch(error){browser={success:false,error:error.stack};}store('browser/result.json',browser);}
|
||||
const commit=own.git(['rev-parse','HEAD']).trim(),paths=own.git(['diff','--name-only',own.initialHead,commit]).trim().split(/\r?\n/).filter(Boolean).sort(),committedCurrent=own.spec.allowed.every(name=>sha(Buffer.from(own.git(['show',commit+':'+name])))===sha(fs.readFileSync(path.join(own.sourceRoot,name))));
|
||||
result={success:!failure&&state.status==='completed'&&Object.values(final).every(value=>value.status===0)&&final.test.stats.tests===own.spec.tests&&final.test.stats.pass===own.spec.tests&&commit!==own.initialHead&&JSON.stringify(paths)===JSON.stringify(own.spec.allowed.slice().sort())&&committedCurrent&&(kind!=='react'||browser?.success===true),
|
||||
kind,executed:true,sessionId:session.id,projectId:project.id,connectionId:original.id,commit,initialHead:own.initialHead,committedPaths:paths,committedCurrent,final,browser,failure:failure??null,productStopReason:state.execution.run?.stopReason??null,ownerBytesAndIndexPreserved:true,supervisorContinuations:0,localTokenCapsChanged:false};
|
||||
}catch(error){result={success:false,kind,executed:!!session,failure:{error:error.stack,noRetry:true},sessionId:session?.id,projectId:project?.id};}
|
||||
finally{
|
||||
const errors=[],restoration={};const guard=async(label,fn)=>{try{await fn();}catch(error){errors.push({label,error:error.stack});}};
|
||||
if(route){await guard('settle',settle);await guard('capture',async()=>{const captured=await captureSession(api,session.id,kind),identitiesPassed=captured.rows.length>0&&captured.rows.every(value=>value.connectionId===original.id&&value.status==='completed'&&value.modelIdentity?.complete&&value.modelIdentity.status==='matched')&&captured.state.execution.task.usage.unknownRequests===0&&captured.state.execution.task.usage.requests===captured.rows.length;result={...result,success:result.success&&identitiesPassed,identitiesPassed,requests:captured.rows.length,usage:captured.state.execution.task.usage,captures:captured.captures};store('git-status.txt',own.git(['status','--porcelain=v1']));store('git-log.txt',own.git(['log','--format=fuller','--stat',own.initialHead+'..HEAD']));});}
|
||||
await guard('restore-connection',async()=>{if(configured){const center=await api('/model-center'),current=center.connections.find(value=>value.id===original.id);if(current.revision===configured.revision){restoration.connection=await api('/model-center/connections',{...connectionInput(original),revision:current.revision,modelIdentityPolicy:original.modelIdentityPolicy??'record'});assert.equal(restoration.connection.credentialRevision,original.credentialRevision);assert.equal(restoration.connection.hasKey,original.hasKey);}else restoration.externalConnectionChangePreserved=true;}});
|
||||
await guard('restore-selection',async()=>{if(selected){const center=await api('/model-center');if(center.selected.type==='connection'&¢er.selected.id===original.id){await api('/model-center/select',frozen.center.selected);restoration.selection=frozen.center.selected;}else restoration.externalSelectionChangePreserved=true;}});
|
||||
await guard('config',async()=>{restoration.config=(await api('/bootstrap')).config;restoration.configComparison=compareConfig(frozen.config,restoration.config);});
|
||||
restoration.keysReadOrResaved=false;restoration.errors=errors;if(errors.length)result={...result,success:false,cleanupErrors:errors};store('restoration.json',restoration);store('observations.json',observations);store('result.json',result);await close();
|
||||
}
|
||||
console.log(JSON.stringify({kind,success:result.success,requests:result.requests,usage:result.usage,commit:result.commit,failure:result.failure,cleanupErrors:result.cleanupErrors}));if(!result.success)process.exitCode=1;
|
||||
@@ -0,0 +1,17 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { prepareProject } from './project.mjs';
|
||||
const output=path.resolve('.qa/ae-driver-'+randomUUID());fs.mkdirSync(output);const results=[];let result;
|
||||
try{
|
||||
for(const kind of ['react','python']){
|
||||
const dir=path.join(output,kind);fs.mkdirSync(dir);
|
||||
const save=(file,value)=>{const target=path.join(dir,file);fs.mkdirSync(path.dirname(target),{recursive:true});fs.writeFileSync(target,typeof value==='string'?value:JSON.stringify(value,null,2),{flag:'wx'});};
|
||||
const prepared=prepareProject(kind,dir,save),baseline=prepared.checks('baseline');
|
||||
assert.equal(baseline.test.stats.tests,prepared.spec.tests);assert.ok(baseline.test.stats.fail>0);assert.equal(baseline.build.status,0);if(kind==='react')assert.equal(baseline.typecheck.status,0);
|
||||
prepared.protect();results.push({kind,originalTests:baseline.test.stats.tests,originalFailed:baseline.test.stats.fail,baselineBuildPassed:true,ownerAndIndexPreserved:true});
|
||||
}
|
||||
result={success:true,results,realModelRequests:0,implementationSupplied:false};
|
||||
}catch(error){result={success:false,error:error.stack,results,realModelRequests:0};process.exitCode=1;}
|
||||
fs.writeFileSync(path.join(output,'result.json'),JSON.stringify(result,null,2));console.log(JSON.stringify({output,...result}));
|
||||
Reference in New Issue
Block a user