68 lines
13 KiB
JavaScript
68 lines
13 KiB
JavaScript
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import http from 'node:http';
|
|
import assert from 'node:assert/strict';
|
|
import {randomUUID,createHash} from 'node:crypto';
|
|
import {chromium,expect} from 'playwright/test';
|
|
import {fixture,launchWorkbench,until,call,completion,assertToolPairs} from '../tests/helpers.mjs';
|
|
import {createDeliveryProject,deliveryGit} from '../tests/fixtures/delivery-project.mjs';
|
|
import {ownerBytes} from '../tests/fixtures/workers-project.mjs';
|
|
|
|
if(process.argv.includes('--help')){console.log('node scripts/verify-evaluations-release.mjs [--app-root <absolute-package>] — actual compiled host/bundled Node, fixed HTTP candidate, original failing checks, review, desktop/390px comparison/export, parent checks/Git, stop and source rejection. Independent data; no real inference.');process.exit(0);}
|
|
const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1]));
|
|
const appRoot=fs.realpathSync.native(args[1]??process.cwd()),nodePath=args.length?path.join(appRoot,'runtime/node.exe'):process.execPath;
|
|
const expectedVersion=JSON.parse(fs.readFileSync(path.join(appRoot,fs.existsSync(path.join(appRoot,'package.json'))?'package.json':'MANIFEST.json'),'utf8')).version;
|
|
const directory=path.resolve('.qa','engineering-evals-release-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
|
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t);createDeliveryProject(f,{broken:true,ownerChanges:true});const original=ownerBytes(f);f.store.close();
|
|
const hash=bytes=>createHash('sha256').update(bytes).digest('hex'),requests=[],held=new Set(),sockets=new Set(),assertions=[],screenshots=[],pageErrors=[];
|
|
let host,childRoute,hold=false,browser,report;
|
|
const model=http.createServer(async(req,res)=>{try{
|
|
const chunks=[];for await(const chunk of req)chunks.push(chunk);const body=JSON.parse(Buffer.concat(chunks));requests.push({route:req.url,body});assert.equal(req.url,'/v1/chat/completions');assertToolPairs(body.messages);
|
|
if(hold){res.writeHead(200,{'Content-Type':'application/json'});res.flushHeaders();held.add(res);res.on('close',()=>held.delete(res));return;}
|
|
const current=await host.api(childRoute),task=current.taskId,changes=current.changes.filter(row=>row.taskId===task),plan=current.execution.plans.at(-1),job=current.execution.checks.at(-1);let calls=[];
|
|
if(!changes.length)calls=[call(randomUUID(),'propose_file',{path:'src/add.mjs',content:'export const add = (a,b) => a + b;\n',reason:'随包固定HTTP候选,保留原测试'})];
|
|
else if(!changes.some(row=>row.status==='pending')){if(!plan)calls=[call(randomUUID(),'plan_checks',{names:['test','build']})];else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];}
|
|
res.writeHead(200,{'Content-Type':'application/json'});res.end(JSON.stringify(completion(calls,calls.length?'':'固定响应完成,仍需核对实际结果')));
|
|
}catch(error){res.writeHead(500,{'Content-Type':'application/json'});res.end(JSON.stringify({error:{message:error.message}}));}});
|
|
model.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));});await new Promise(resolve=>model.listen(0,'127.0.0.1',resolve));
|
|
const mark=value=>{assertions.push(value);console.log('PASS '+value);},route='/sessions/'+f.session.id;
|
|
const definition=title=>({title,evidenceMode:'local-fixture',samples:[{label:'原加法边界',goal:'修复加法,保留原调用与测试。',criteria:'原3测试和构建通过,原测试/脚本保持不变。',paths:['src/add.mjs'],checks:['test','build']}],budget:{maxTokens:200000,maxSteps:32,maxToolCalls:64,maxDurationMs:600000}});
|
|
const get=async id=>(await host.api(route+'/evaluations')).batches.find(row=>row.id===id);
|
|
async function operation(batch,name){await host.api(route+'/evaluations/'+batch.id+'/'+name,{fingerprint:batch.fingerprint,sampleId:batch.samples[0].id});await until(async()=>!['preparing','baseline_running','candidate_running','collecting','adopting'].includes((await get(batch.id)).status),120000);return get(batch.id);}
|
|
async function apply(target,set){await host.api(target+'/change-sets/'+set.id+'/apply',{fingerprint:set.fingerprint});let value;await until(async()=>{value=await host.api(target+'/change-sets/'+set.id);return !['prepared','applying'].includes(value.status);});assert.equal(value.status,'completed',value.summary);}
|
|
try{
|
|
host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath});assert.equal(host.bootstrap.version,expectedVersion,'Compiled service identity must match its selected package.');assert.equal(fs.realpathSync.native(host.bootstrap.runtimeRoot),appRoot);assert.notEqual(new URL(host.url).port,'4318');
|
|
await host.api('/config',{mode:'compatible',model:'owned-evaluation-release',baseUrl:'http://127.0.0.1:'+model.address().port+'/v1',apiKey:'self-owned-evaluation-release',maxSteps:32});
|
|
let batch=(await host.api(route+'/evaluations',definition('随包工程评测'))).batch;batch=await operation(batch,'prepare');assert.equal(batch.samples[0].status,'ready',batch.message);
|
|
const group=(await host.api(route+'/collaborations')).groups.find(row=>row.id===batch.samples[0].groupId);childRoute='/sessions/'+group.children.find(row=>row.evaluation.role==='candidate').sessionId;
|
|
batch=await operation(batch,'baseline');assert.equal(batch.samples[0].status,'baseline_complete',batch.message);assert.equal(requests.length,0);assert.deepEqual(ownerBytes(f),original);
|
|
mark('selected compiled host prepares same-source workspaces and retains original three-test failure with zero original model requests and unchanged owner staging');
|
|
batch=await operation(batch,'start');assert.equal(batch.samples[0].status,'awaiting_review',batch.message);let current=await host.api(childRoute);
|
|
const set=(await host.api(childRoute+'/change-sets',{changeIds:current.changes.filter(row=>row.status==='pending').map(row=>row.id)})).execution.changeSets.at(-1);await apply(childRoute,set);
|
|
await host.api(route+'/evaluations/'+batch.id+'/start',{fingerprint:batch.fingerprint,sampleId:batch.samples[0].id});
|
|
await until(async()=>{current=await host.api(childRoute);for(const approval of current.approvals.filter(row=>row.status==='pending'))await host.api(childRoute+'/approvals/'+approval.id,{allow:true});batch=await get(batch.id);return !['candidate_running'].includes(batch.status);},120000);
|
|
assert.equal(batch.samples[0].status,'candidate_complete',batch.message);assert.equal(batch.summary.candidate.unknown,1);assert.equal(batch.summary.candidate.pending,0);
|
|
batch=await operation(batch,'seal');const comparison=await host.api(route+'/evaluations/'+batch.id+'/report?sampleId='+batch.samples[0].id);assert.deepEqual(comparison.summary,{original:'failed',candidate:'passed'});assert.equal(comparison.original.attempts[0].results[0].tests.failed,3);assert.equal(comparison.candidate.attempts.at(-1).results[0].tests.passed,3);assert.equal(comparison.inferenceValidated,false);assert.deepEqual(ownerBytes(f),original);
|
|
fs.writeFileSync(path.join(directory,'original-comparison.json'),JSON.stringify(comparison,null,2));
|
|
mark('real owned HTTP tools, applied candidate review and approved original checks pass; unsealed attempted outcomes stay unknown and sealed report retains original failure');
|
|
browser=await chromium.launch({channel:'msedge',headless:true});const page=await browser.newPage({viewport:{width:1440,height:1000},acceptDownloads:true});page.on('pageerror',error=>pageErrors.push(error.message));
|
|
await page.addInitScript(({projectId,sessionId})=>{localStorage.setItem('awb-project',projectId);localStorage.setItem('awb-session',sessionId);},{projectId:f.project.id,sessionId:f.session.id});
|
|
await page.goto(host.url);await page.waitForLoadState('networkidle');await openWorkbenchTool(page, '工程评测');const panel=page.getByLabel('工程评测控制台',{exact:true});await expect(panel).toContainText('随包工程评测');
|
|
const refresh=panel.getByRole('button',{name:'刷新',exact:true});await refresh.focus();assert.ok(await refresh.evaluate(node=>node.matches(':focus')));await page.keyboard.press('Tab');
|
|
await panel.getByRole('button',{name:'查看原版 / 候选对照',exact:true}).click();const dialog=page.getByRole('dialog',{name:'原版与候选对照',exact:true});await expect(dialog).toContainText('测试 0/3 · 失败 3');await expect(dialog).toContainText('测试 3/3 · 失败 0');await expect(dialog).toContainText('本地固定响应');await expect(dialog).toContainText('费用未报告');
|
|
for(const viewport of [{width:1440,height:1000},{width:390,height:844}]){await page.setViewportSize(viewport);if(viewport.width===390)await dialog.getByText('测试 3/3 · 失败 0',{exact:true}).scrollIntoViewIfNeeded();assert.ok(await page.locator('body').evaluate(node=>node.scrollWidth<=innerWidth));const file=path.join(directory,'comparison-'+viewport.width+'.png');await page.screenshot({path:file,fullPage:true});screenshots.push({file,sha256:hash(fs.readFileSync(file))});}
|
|
const downloaded=page.waitForEvent('download');await dialog.getByRole('button',{name:'导出原报告',exact:true}).click();const file=path.join(directory,'downloaded-report.json');await(await downloaded).saveAs(file);assert.deepEqual(JSON.parse(fs.readFileSync(file)).summary,comparison.summary);await page.keyboard.press('Escape');await page.reload();await page.waitForLoadState('networkidle');await openWorkbenchTool(page, '工程评测');await expect(panel).toContainText('随包工程评测');
|
|
mark('compiled desktop/390px UI renders original and candidate checks, fixed-response label, unknown fee, keyboard focus and downloadable report; refresh preserves the same batch');
|
|
batch=await operation(batch,'adopt');assert.equal(batch.status,'awaiting_delivery',batch.message);const adopted=(await host.api(route+'/collaborations')).groups.find(row=>row.id===group.id);await apply(route,await host.api(route+'/change-sets/'+adopted.merge.changeSetId));
|
|
const plan=(await host.api(route+'/check-plans',{names:['test','build']})).execution.plans.at(-1),job=(await host.api(route+'/check-jobs',{planId:plan.id})).execution.checks.at(-1);let checked;await until(async()=>{checked=await host.api(route+'/check-jobs/'+job.id);return !['queued','running'].includes(checked.status);});assert.equal(checked.status,'passed',checked.summary);
|
|
const deliveryPlan=(await host.api(route+'/delivery-plans',{message:'Adopt compiled evaluation candidate'})).execution.deliveryPlans.at(-1),deliveryJob=(await host.api(route+'/delivery-jobs',{planId:deliveryPlan.id})).execution.deliveries.at(-1);let delivered;await until(async()=>{delivered=await host.api(route+'/delivery-jobs/'+deliveryJob.id);return ['completed','failed','cancelled','unknown'].includes(delivered.status);},120000);assert.equal(delivered.status,'completed',delivered.summary);batch=await operation(batch,'finalize');assert.equal(batch.adoption.status,'completed');assert.deepEqual(ownerBytes(f).files,original.files);assert.equal(deliveryGit(f,['diff-tree','--no-commit-id','--name-only','-r','HEAD']).trim(),'src/add.mjs');
|
|
mark('fresh parent review, actual original parent checks and exact-scope local Git complete adoption while preserving owner staged/unstaged/untracked bytes');
|
|
hold=true;let stopped=(await host.api(route+'/evaluations',definition('随包停止原请求'))).batch;stopped=await operation(stopped,'prepare');const stoppedGroup=(await host.api(route+'/collaborations')).groups.find(row=>row.id===stopped.samples[0].groupId);childRoute='/sessions/'+stoppedGroup.children.find(row=>row.evaluation.role==='candidate').sessionId;stopped=await operation(stopped,'baseline');const before=requests.length;
|
|
await host.api(route+'/evaluations/'+stopped.id+'/start',{fingerprint:stopped.fingerprint,sampleId:stopped.samples[0].id});await until(()=>requests.length===before+1);await host.api(route+'/evaluations/'+stopped.id+'/stop',{fingerprint:stopped.fingerprint,sampleId:stopped.samples[0].id});await until(()=>held.size===0);stopped=await operation(stopped,'seal');assert.equal(stopped.samples[0].report.summary.candidate,'stopped');assert.equal(requests.length,before+1);
|
|
const stale=(await host.api(route+'/evaluations',definition('随包来源变化'))).batch;fs.appendFileSync(path.join(f.projectRoot,'README.md'),'\nOwner source changed\n');const rejected=await operation(stale,'prepare');assert.equal(rejected.samples[0].status,'failed');assert.match(rejected.message,/来源工程已变化/);
|
|
mark('bundled candidate stop closes only its original owned HTTP request and keeps stopped outcome; changed source rejects before new workspaces without replay');
|
|
assert.deepEqual(pageErrors,[]);report={success:true,version:host.bootstrap.version,appRoot,nodePath,containedRuntime:!!args.length,directory,assertions,screenshots,pageErrors,requests,comparison,parentCheck:checked,delivery:delivered,stopped,rejected,realModelRequests:0};
|
|
}catch(error){report={success:false,error:error.stack,directory,appRoot,assertions,screenshots,pageErrors,requests,realModelRequests:0};throw error;}
|
|
finally{await browser?.close();await host?.close();for(const response of held)response.destroy();for(const socket of sockets)socket.destroy();await new Promise(resolve=>model.close(resolve));fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));for(const dispose of disposers.reverse())await dispose();console.log('EVIDENCE '+directory);}
|