81 lines
15 KiB
JavaScript
81 lines
15 KiB
JavaScript
import { openWorkbenchTool } from './ui-navigation.mjs';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, expect } from 'playwright/test';
|
|
import { fixture, call } from '../tests/helpers.mjs';
|
|
import { workerHttp } from '../tests/fixtures/workers-http.mjs';
|
|
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
|
|
|
if(process.argv.includes('--help')){console.log('node scripts/verify-evaluations-ui.mjs — independent local HTTP host, fixed-response candidate, real Git/test/build, desktop/390px queue, review, comparison, adoption, stop and source errors. No real model inference.');process.exit(0);}
|
|
assert.equal(process.argv.length,2);
|
|
const directory=path.resolve('.qa','engineering-evals-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
|
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),hash=bytes=>createHash('sha256').update(bytes).digest('hex');
|
|
let hold=false;
|
|
const host=await workerHttp(f,{broken:true,respond:async({project,signal})=>{
|
|
if(hold)return new Promise((resolve,reject)=>{if(signal.aborted)reject(signal.reason);else signal.addEventListener('abort',()=>reject(signal.reason),{once:true});});
|
|
const owner=[...f.store.sessions.values()].find(session=>session.projectId===project.id),taskId=owner.taskId;
|
|
const plan=[...f.store.core.check_plan.values()].find(plan=>plan.taskId===taskId),job=[...f.store.core.check_job.values()].find(job=>job.taskId===taskId);
|
|
let calls=[];
|
|
if(!owner.changes.length)calls=[call(randomUUID(),'propose_file',{path:'src/add.mjs',content:'export const add = (a,b) => a + b;\n',reason:'本地固定响应修复加法'})];
|
|
else if(owner.changes.some(change=>change.status==='pending'))calls=[];
|
|
else if(!plan)calls=[call(randomUUID(),'plan_checks',{names:['test','build']})];
|
|
else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
|
|
return {message:{role:'assistant',content:calls.length?'':'固定响应已返回,仍需实际工程证据',...(calls.length?{tool_calls:calls}:{})},usage:{total_tokens:168,prompt_tokens:123,completion_tokens:45}};
|
|
}});
|
|
const browser=await chromium.launch({channel:'msedge',headless:true}),page=await browser.newPage({viewport:{width:1440,height:1000},acceptDownloads:true});
|
|
const assertions=[],screenshots=[],errors=[],exports=[];page.on('pageerror',error=>errors.push(error.message));
|
|
const panel=()=>page.getByLabel('工程评测控制台',{exact:true});
|
|
const mark=value=>{assertions.push(value);console.log('PASS '+value);};
|
|
const shot=async name=>{const file=path.join(directory,name+'.png');await page.screenshot({path:file,fullPage:true});screenshots.push({file,sha256:hash(fs.readFileSync(file))});assert.ok(await page.locator('body').evaluate(node=>node.scrollWidth<=innerWidth),'body overflow '+name);assert.ok(await panel().evaluate(node=>node.scrollWidth<=node.clientWidth+1),'panel overflow '+name);};
|
|
const open=async()=>{await openWorkbenchTool(page, '工程评测');await expect(panel().getByRole('button',{name:'刷新',exact:true})).toBeEnabled();};
|
|
const confirm=async(name,button)=>{await panel().getByRole('button',{name,exact:true}).click();await page.getByRole('dialog',{name:'确认评测操作',exact:true}).getByRole('button',{name:button,exact:true}).click();};
|
|
const refresh=async()=>{await panel().getByRole('button',{name:'刷新',exact:true}).click();};
|
|
const latest=()=>[...f.store.core.evaluation.values()].at(-1);
|
|
const parent=async()=>{await openWorkbenchTool(page, '隔离协作');await page.getByLabel('隔离协作控制台',{exact:true}).getByRole('button',{name:'返回父目标',exact:true}).click();await expect.poll(()=>page.evaluate(()=>localStorage.getItem('awb-session'))).toBe(f.session.id);await open();};
|
|
async function fill(title){await panel().getByLabel('评测名称',{exact:true}).fill(title);await panel().getByLabel('评测来源',{exact:true}).selectOption('local-fixture');await panel().getByLabel('样本名称',{exact:true}).fill('正负数和整数加法边界');await panel().getByLabel('具体需求',{exact:true}).fill('修复加法运算,保留原测试及已有调用方式');await panel().getByLabel('验收条件',{exact:true}).fill('原3项加法测试与构建通过,原测试文件和运行声明保持不变');await panel().getByLabel('允许修改的源码文件',{exact:true}).fill('src/add.mjs');}
|
|
async function apply(prepared=false){if(await page.getByRole('button',{name:'查看改动',exact:true}).isVisible())await page.getByRole('button',{name:'查看改动',exact:true}).click();else await page.locator('.inspector-tabs').getByRole('button',{name:/^改动/}).click();await page.getByRole('button',{name:prepared?'审查变更集':'组合审查',exact:true}).click();if(!prepared)await page.getByRole('dialog',{name:'选择要一起审查的提案',exact:true}).getByRole('button',{name:'封存审查范围',exact:true}).click();const dialog=page.getByRole('dialog',{name:'多文件变更集审查',exact:true});await dialog.getByRole('button',{name:'确认应用全部 1 项',exact:true}).click();await expect(dialog).toContainText('全部应用完成',{timeout:20000});await page.keyboard.press('Escape');}
|
|
let result;
|
|
try{
|
|
await page.addInitScript(({projectId,sessionId})=>{localStorage.setItem('awb-project',projectId);localStorage.setItem('awb-session',sessionId);},{projectId:f.project.id,sessionId:f.session.id});
|
|
await page.goto(host.url);await page.waitForLoadState('networkidle');await open();await shot('desktop-empty');
|
|
await expect(panel().getByRole('button',{name:'封存评测计划',exact:true})).toBeDisabled();await fill('固定工程改进 · 中文完整评测');
|
|
await panel().getByLabel('允许修改的源码文件',{exact:true}).fill('test/add.test.mjs');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect(panel().getByRole('alert')).toContainText('测试');await expect(panel().getByLabel('具体需求',{exact:true})).toHaveValue('修复加法运算,保留原测试及已有调用方式');
|
|
await page.setViewportSize({width:390,height:844});await shot('mobile-form-scope-error');await page.setViewportSize({width:1440,height:1000});
|
|
await panel().getByLabel('允许修改的源码文件',{exact:true}).fill('src/add.mjs');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect(panel()).toContainText('计划已封存');
|
|
const batch=latest(),sample=batch.samples[0],original=ownerBytes(f);assert.equal(batch.evidenceMode,'local-fixture');assert.equal(host.requests.length,0);
|
|
await page.reload();await page.waitForLoadState('networkidle');await open();await expect(panel()).toContainText(batch.title);
|
|
mark('desktop/390px form rejects candidate test edits, keeps entered requirements, persists fixed model/source/budget and reloads the original plan');
|
|
await panel().getByRole('button',{name:'准备同源工作区',exact:true}).click();await page.keyboard.press('Escape');assert.equal(f.store.core.worker.size,0);
|
|
await confirm('准备同源工作区','确认准备');await expect.poll(()=>sample.status,{timeout:30000}).toBe('ready');await expect(panel()).toContainText('node --test test/*.test.mjs');assert.deepEqual(ownerBytes(f),original);await shot('desktop-ready');
|
|
await confirm('核对并运行原版','确认运行原版');await expect.poll(()=>sample.status,{timeout:30000}).toBe('baseline_complete');assert.equal(host.requests.length,0);assert.equal(f.store.core.check_job.get(sample.baselineJobId).results[0].tests.failed,3);
|
|
mark('explicit preparation seals two same-source checkouts and actual commands; original real three-test failure is retained with zero model requests');
|
|
await confirm('开始候选改进','确认开始候选');await expect.poll(()=>sample.status,{timeout:30000}).toBe('awaiting_review');
|
|
await panel().getByRole('button',{name:'审查候选改动',exact:true}).click();await apply();await parent();
|
|
await confirm('继续候选任务','确认开始候选');const group=f.store.core.collaboration.get(sample.groupId),child=group.children.find(child=>child.evaluation.role==='candidate');
|
|
await expect.poll(()=>f.store.session(child.sessionId).status,{timeout:15000}).toBe('awaiting_approval');await expect(panel().getByRole('button',{name:'核对并审批检查',exact:true})).toBeVisible();
|
|
await panel().getByRole('button',{name:'核对并审批检查',exact:true}).click();await page.locator('.approval-card').getByRole('button',{name:'确认执行',exact:true}).click();await parent();
|
|
await expect.poll(()=>sample.status,{timeout:30000}).toBe('candidate_complete');assert.deepEqual(ownerBytes(f),original);await shot('desktop-candidate-complete');
|
|
mark('candidate navigates to its original restricted task for real change review and check approval, runs the original test/build and preserves the parent source');
|
|
await confirm('封存当前结果','确认封存结果');await expect.poll(()=>batch.status,{timeout:15000}).toBe('completed');await panel().getByRole('button',{name:'查看原版 / 候选对照',exact:true}).click();
|
|
const report=page.getByRole('dialog',{name:'原版与候选对照',exact:true});await expect(report).toContainText('测试 0/3 · 失败 3');await expect(report).toContainText('测试 3/3 · 失败 0');await expect(report).toContainText('不证明模型自主完成率');await shot('desktop-comparison');
|
|
const download=page.waitForEvent('download');await report.getByRole('button',{name:'导出原报告',exact:true}).click();const file=path.join(directory,'downloaded-report.json');await (await download).saveAs(file);const downloaded=JSON.parse(fs.readFileSync(file,'utf8'));assert.deepEqual(downloaded.summary,{original:'failed',candidate:'passed'});exports.push({file,sha256:hash(fs.readFileSync(file))});
|
|
await page.setViewportSize({width:390,height:844});await shot('mobile-comparison');await report.getByText('测试 3/3 · 失败 0',{exact:true}).scrollIntoViewIfNeeded();await expect(report.getByText('测试 3/3 · 失败 0',{exact:true})).toBeVisible();await shot('mobile-candidate-evidence');await page.keyboard.press('Escape');await page.reload();await page.waitForLoadState('networkidle');await open();await panel().getByRole('button',{name:'查看原版 / 候选对照',exact:true}).click();await expect(page.getByRole('dialog',{name:'原版与候选对照',exact:true})).toContainText('测试 0/3 · 失败 3');await page.keyboard.press('Escape');await page.setViewportSize({width:1440,height:1000});
|
|
mark('parallel comparison preserves original failure, candidate pass, real approval/application counts and absent cost; export and refresh retain the original report and fixture label');
|
|
await confirm('采纳此候选,准备父工程审查','确认准备采纳');await expect.poll(()=>batch.status,{timeout:30000}).toBe('awaiting_delivery');await panel().getByRole('button',{name:'审查父工程变更集',exact:true}).click();await apply(true);
|
|
const plan=await host.runtime.checkRunner().plan(f.session,['test','build']);assert.equal((await (await host.runtime.checkRunner().start(f.session,plan.id)).promise).status,'passed');
|
|
const delivery=await host.runtime.gitDelivery().plan(f.session,{message:'Adopt fixed engineering evaluation candidate'});assert.equal((await (await host.runtime.gitDelivery().start(f.session,delivery.id)).promise).status,'completed');await open();await panel().getByRole('button',{name:'核对父工程检查与 Git',exact:true}).click();await expect.poll(()=>batch.adoption.status).toBe('completed');assert.deepEqual(ownerBytes(f).files,original.files);await shot('desktop-delivered');
|
|
mark('adoption forms a fresh parent review batch; completion requires applied source, fresh real parent checks and local Git delivery while owner modifications remain exact');
|
|
hold=true;await panel().getByRole('button',{name:'新建下一批评测',exact:true}).click();await fill('停止中的候选,保留原失败分母');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect.poll(()=>f.store.core.evaluation.size).toBe(2);await expect(panel()).toContainText('停止中的候选,保留原失败分母');const stopped=latest(),stoppedSample=stopped.samples[0];
|
|
await confirm('准备同源工作区','确认准备');await expect.poll(()=>stoppedSample.status,{timeout:30000}).toBe('ready');await confirm('核对并运行原版','确认运行原版');await expect.poll(()=>stoppedSample.status,{timeout:30000}).toBe('baseline_complete');const requests=host.requests.length;
|
|
await confirm('开始候选改进','确认开始候选');await expect.poll(()=>host.requests.length).toBe(requests+1);await panel().getByRole('button',{name:'停止本批执行',exact:true}).click();await expect.poll(()=>host.runtime.hasActivity(),{timeout:10000}).toBe(false);await refresh();assert.notEqual(stoppedSample.status,'candidate_complete');
|
|
await confirm('封存当前结果','确认封存结果');await expect.poll(()=>!!stoppedSample.report).toBe(true);assert.equal(stoppedSample.report.summary.candidate,'stopped');assert.equal(host.requests.length,requests+1);await page.setViewportSize({width:390,height:844});await shot('mobile-stopped');
|
|
mark('stop waits for the owned candidate request, preserves stopped outcome and never turns cancellation into a passed sample or a replay');
|
|
await panel().getByRole('button',{name:'新建下一批评测',exact:true}).click();await fill('来源变化时保持失效原因');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect.poll(()=>f.store.core.evaluation.size).toBe(3);await expect(panel()).toContainText('来源变化时保持失效原因');const stale=latest();fs.appendFileSync(path.join(f.projectRoot,'README.md'),'\nOwner changed after plan\n');await confirm('准备同源工作区','确认准备');await expect.poll(()=>stale.samples[0].status,{timeout:10000}).toBe('failed');await expect(panel()).toContainText('来源工程已变化');await shot('mobile-source-error');
|
|
mark('changed owner source is rejected before workspace creation and the error stays attached to the original batch instead of a generic chat reply');
|
|
for(const batch of f.store.core.evaluation.values())for(const sample of batch.samples){if(sample.report){const file=path.join(directory,'report-'+sample.id+'.json');fs.writeFileSync(file,JSON.stringify(host.runtime.evaluationManager().readReport(f.session,batch,sample),null,2));exports.push({file,sha256:hash(fs.readFileSync(file))});}}
|
|
assert.deepEqual(errors,[]);result={passed:true,version:JSON.parse(fs.readFileSync('package.json','utf8')).version,evidenceKind:'owned fixed response, actual isolated Git/checks and desktop/390px UI',realModelRequests:0,fixtureRequests:host.requests.length,assertions,screenshots,exports,errors};
|
|
}catch(error){result={passed:false,error:error.stack,assertions,screenshots,exports,errors};await shot('failure').catch(()=>{});throw error;}
|
|
finally{await browser.close();await host.runtime.stopAll();f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');fs.cpSync(f.root,path.join(directory,'original-fixture'),{recursive:true});fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(result,null,2));if(result.passed)fs.writeFileSync('docs/engineering-evals-ui-validation.json',JSON.stringify(result,null,2));for(const dispose of disposers.reverse())await dispose();console.log('EVIDENCE '+directory);}
|