Files
Agent-Workbench/scripts/verify-vision-ui.mjs
T

92 lines
15 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import https from 'node:https';
import assert from 'node:assert/strict';
import { createHash,randomUUID,X509Certificate } from 'node:crypto';
import { chromium,expect } from 'playwright/test';
import { fixture,launchWorkbench,until } from '../tests/helpers.mjs';
import { requestImages } from '../src/shared/contracts/model-request.mjs';
import { PNG } from 'pngjs';
if(process.argv.includes('--help')){console.log('node scripts/verify-vision-ui.mjs [--app-root <absolute-package>] — owned image HTTP, desktop/390px upload/history/download, exact Task budget resume, TLS companion read/revoke and original SSE stop. No configured model calls or iPhone claim.');process.exit(0);}
const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1]));
const appRoot=fs.realpathSync.native(args[1]??process.cwd()),directory=path.resolve('.qa','vision-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t);f.store.close();
const requests=[],sockets=new Set(),held=new Set();let mode='success',closed=0;
const message=text=>({type:'message',role:'assistant',content:[{type:'output_text',text}]}),result=output=>({status:'completed',output,usage:{input_tokens:200,output_tokens:30,total_tokens:230}});
const model=http.createServer(async(req,res)=>{try{
const chunks=[];for await(const chunk of req)chunks.push(chunk);const body=JSON.parse(Buffer.concat(chunks));requests.push({route:req.url,body});
assert.ok(requestImages(body).length===1,'one screenshot per request');
assert.notEqual(body.model,'text-only','text member must never dispatch');
if(mode==='hold'){
assert.equal(body.stream,true);res.writeHead(200,{'Content-Type':'text/event-stream'});
res.write('data: '+JSON.stringify({type:'response.output_text.delta',sequence_number:0,delta:'截图正在核对;参数尚未完整。'})+'\n\n');
held.add(res);res.on('close',()=>{held.delete(res);closed++;});return;
}
const latestUser=body.input.findLastIndex(value=>value.role==='user'&&typeof value.content==='string');
const output=body.input.slice(latestUser+1).some(value=>value.type==='function_call_output')?[message('原截图与实际工程读取结果已经核对。')]
:[{type:'function_call',call_id:'image-files-'+requests.length,name:'list_files',arguments:'{"path":"","depth":1}'}];
res.writeHead(200,{'Content-Type':'application/json'});res.end(JSON.stringify(result(output)));
}catch(error){res.writeHead(500,{'Content-Type':'application/json'});res.end(JSON.stringify({error:{message:error.message}}));}});
model.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));});await new Promise(resolve=>model.listen(0,'127.0.0.1',resolve));
const host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath:args.length?path.join(appRoot,'runtime/node.exe'):process.execPath});assert.notEqual(new URL(host.url).port,'4318');
const browser=await chromium.launch({channel:'msedge',headless:true}),page=await browser.newPage({viewport:{width:1440,height:1000}}),passed=[],screenshots=[],errors=[];
page.on('pageerror',error=>errors.push(error.message));const get=()=>host.api('/sessions/'+f.session.id),mark=value=>{passed.push(value);console.log('PASS '+value);};
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
async function shot(target,name){const file=path.join(directory,name+'.png');await target.screenshot({path:file});screenshots.push(file);assert.ok(await target.locator('body').evaluate(node=>node.scrollWidth<=innerWidth),'page overflow '+name);}
let gateway,relay,cookie='',mobile;
try{
const raw=new PNG({width:1280,height:720});for(let y=0;y<720;y++)for(let x=0;x<1280;x++)raw.data.set(x>900?[190,60,50,255]:[28,76,74,255],(y*1280+x)*4);
const imageFile=path.join(directory,'engineering-screenshot.png');fs.writeFileSync(imageFile,PNG.sync.write(raw));
const baseUrl='http://127.0.0.1:'+model.address().port+'/v1';
const vision=await host.api('/model-center/connections',{name:'截图验收 Responses',model:'owned-vision',protocol:'responses',baseUrl,requiresKey:false,contextLimit:64000,capabilityDeclarations:{vision:'supported'}});
const center=await host.api('/model-center'),visionId=vision.id??center.connections.find(row=>row.name==='截图验收 Responses').id;
await host.api('/model-center/connections',{name:'纯文本排除样本',model:'text-only',baseUrl,requiresKey:false,contextLimit:64000});
const textId=(await host.api('/model-center')).connections.find(row=>row.model==='text-only').id;
const pool=await host.api('/model-center/pools',{name:'视觉验收池',members:[{connectionId:textId},{connectionId:visionId}]});
const poolId=pool.id??(await host.api('/model-center')).pools.find(row=>row.name==='视觉验收池').id;
await host.api('/model-center/select',{type:'pool',id:poolId});
await page.addInitScript(({projectId,sessionId})=>{localStorage.setItem('awb-project',projectId);localStorage.setItem('awb-session',sessionId);},{projectId:f.project.id,sessionId:f.session.id});
await page.goto(host.url);await page.waitForLoadState('networkidle');
await page.getByLabel('任务内容',{exact:true}).fill('根据这张截图核对当前工程,保留原工具结果。');
await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(imageFile);await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');
await expect(page.locator('.input-route')).toContainText('视觉验收池');await shot(page,'desktop-draft');
await page.getByRole('button',{name:'查看截图 engineering-screenshot.png',exact:true}).click();const dialog=page.getByRole('dialog');await expect(dialog).toContainText('1280 × 720');
const downloadPromise=page.waitForEvent('download');await dialog.getByRole('button',{name:'下载原图',exact:true}).click();const download=await downloadPromise,file=path.join(directory,'downloaded-original.png');await download.saveAs(file);assert.equal(sha(fs.readFileSync(file)),sha(fs.readFileSync(imageFile)));await page.keyboard.press('Escape');
await page.reload();await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');await page.locator('.input-draft input[type=checkbox]').check();
await page.getByLabel('任务内容',{exact:true}).fill('根据截图核对工程。');await page.getByRole('button',{name:'发送任务',exact:true}).click();await until(async()=>(await get()).status==='completed',15000);
let session=await get();assert.equal(requests.length,2);assert.equal(session.execution.progress.completedTools,1);assert.equal(session.inputAttachments.filter(row=>row.status==='attached').length,1);
await expect(page.locator('.message-user .input-receipt')).toContainText('SHA-256 已核对');await shot(page,'desktop-attached');await page.setViewportSize({width:390,height:844});await shot(page,'narrow-attached');await page.setViewportSize({width:1440,height:1000});
mark('desktop upload, explicit route, refreshed draft, original download SHA, one paired tool and narrow task history');
const invalid=path.join(directory,'forged.png');fs.writeFileSync(invalid,'not a real PNG');await page.getByLabel('任务内容',{exact:true}).fill('失败后保留输入');await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(invalid);await expect(page.locator('.toast')).toContainText('类型不符');await expect(page.getByLabel('任务内容',{exact:true})).toHaveValue('失败后保留输入');assert.equal(requests.length,2);mark('forged image rejects without losing task text or sending a model request');
// Reuse the actual completed tool group in the same Task after a deliberate
// insufficient context resume; no completed tool is requested again.
// Completed Tasks are immutable history. Start a new screenshot Task, stop at
// one tool round, then exercise budget continuation within that Task.
await host.api('/config',{...host.bootstrap.config,mode:'compatible',model:'unused',baseUrl,contextLimit:16000,maxSteps:1});
await host.api('/model-center/select',{type:'pool',id:poolId});
await page.reload();await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(imageFile);await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');await page.getByLabel('任务内容',{exact:true}).fill('截图任务预算续接验收');await page.getByRole('button',{name:'发送任务',exact:true}).click();await until(async()=>(await get()).status==='paused',15000);
const original=await get(),taskId=original.execution.task.id,count=requests.length;
await host.api('/config',{...(await host.api('/bootstrap')).config,contextLimit:1000});await host.api('/model-center/select',{type:'pool',id:poolId});await host.api('/sessions/'+f.session.id+'/resume',{});await until(async()=>(await get()).execution.run.budgetPause,15000);
await page.reload();await expect(page.getByLabel('任务预算诊断')).toContainText('图像本地预留');await page.getByRole('button',{name:'核对预算并续接原任务',exact:true}).click();
await dialog.getByLabel('上下文窗口',{exact:true}).fill('64000');await dialog.getByLabel('新Run工具轮次',{exact:true}).fill('3');await dialog.getByRole('button',{name:'准备续接预览',exact:true}).click();await expect(dialog.locator('.budget-preview')).toContainText('保留 1 项完成工具、1 张附件');assert.equal(requests.length,count);await shot(page,'desktop-budget-preview');
await page.setViewportSize({width:390,height:844});await shot(page,'narrow-budget-preview');await dialog.getByRole('button',{name:'确认预算并续接',exact:true}).click();await until(async()=>(await get()).status==='completed',15000);
session=await get();assert.equal(session.execution.task.id,taskId);assert.equal(session.execution.progress.completedTools,1);assert.equal(requests.length,count+1);assert.equal((await host.api('/bootstrap')).config.contextLimit,1000);await page.setViewportSize({width:1440,height:1000});mark('explicit budget preview and confirmation retain Task, attachment and completed tool while global capacity stays unchanged');
const initial=await host.api('/companion'),reservation=http.createServer();assert.ok(initial.addresses.length);await new Promise(resolve=>reservation.listen(0,initial.addresses[0].address,resolve));const port=reservation.address().port;await new Promise(resolve=>reservation.close(resolve));
await host.api('/companion/start',{address:initial.addresses[0].address,port});gateway=await host.api('/companion');const certificate=await fetch(host.url+'/api/companion/certificate',{headers:{'X-Workbench-Token':host.bootstrap.token}}),ca=new X509Certificate(Buffer.from(await certificate.arrayBuffer())).toString(),target=new URL(gateway.origin);
const ticket=await host.api('/companion/pair',{projectIds:[f.project.id],permissions:{observe:true,approve:false,stop:false},expiresInHours:1});
relay=http.createServer((request,response)=>{const forwarded=https.request(new URL(request.url,target),{method:request.method,ca,servername:'',headers:{Host:target.host,Origin:target.origin,Connection:'close',...(cookie?{Cookie:cookie}:{}),...(request.headers['content-type']?{'Content-Type':request.headers['content-type']}:{})}},incoming=>{if(incoming.headers['set-cookie']?.length)cookie=incoming.headers['set-cookie'][0].split(';')[0];const headers={...incoming.headers};delete headers['set-cookie'];response.writeHead(incoming.statusCode,headers);incoming.pipe(response);});forwarded.on('error',()=>response.destroy());request.pipe(forwarded);});await new Promise(resolve=>relay.listen(0,'127.0.0.1',resolve));
mobile=await browser.newPage({viewport:{width:390,height:844},isMobile:true,hasTouch:true});mobile.on('pageerror',error=>errors.push(error.message));await mobile.goto('http://127.0.0.1:'+relay.address().port+'/#pair='+new URLSearchParams(new URL(ticket.url).hash.slice(1)).get('pair'));await mobile.getByRole('button',{name:'连接这台电脑',exact:true}).click();await mobile.locator('.mobile-task-row').first().click();
await expect(mobile.locator('.mobile-input-image')).toContainText('SHA-256 已核对');await mobile.locator('.mobile-input-image summary').click();await shot(mobile,'companion-image');await mobile.reload();await expect(mobile.locator('.mobile-task-row').first()).toBeVisible();await mobile.locator('.mobile-task-row').first().click();await expect(mobile.locator('.mobile-input-image')).toContainText('SHA-256 已核对');
const device=(await host.api('/companion')).devices.find(row=>row.status==='active');await host.api('/companion/devices/'+device.id+'/revoke',{revision:device.revision});await mobile.getByRole('button',{name:'刷新任务',exact:true}).click();await expect(mobile.locator('.mobile-input-image')).toHaveCount(0);await expect(mobile.getByRole('button',{name:'连接这台电脑',exact:true})).toBeVisible();mark('paired TLS companion verifies same screenshot, refresh restores it and revoke clears project/image content');
await host.api('/companion/close',{});gateway=null;
// This case measures cancellation, independently of the deliberate low-budget
// case above. Leave room for the current shipped tool definitions and image.
await host.api('/config',{...(await host.api('/bootstrap')).config,contextLimit:64000,maxSteps:3});
const connection=(await host.api('/model-center')).connections.find(row=>row.id===visionId);await host.api('/model-center/connections',{...connection,streaming:true,capabilityDeclarations:{...connection.capabilityDeclarations,streaming:'supported'}});await host.api('/model-center/select',{type:'connection',id:visionId});mode='hold';
await page.reload();await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(imageFile);await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');await page.getByLabel('任务内容',{exact:true}).fill('截图流式停止验收');await page.getByRole('button',{name:'发送任务',exact:true}).click();await expect(page.getByLabel('模型输出片段',{exact:true}).last()).toContainText('截图正在核对');const stopping=await get();await page.getByRole('button',{name:'停止',exact:true}).click();await until(async()=>(await get()).status==='paused'&&closed===1,15000);session=await get();assert.equal(session.events.filter(row=>row.kind==='tool'&&row.runId===stopping.runId).length,0);await shot(page,'desktop-image-stopped');mark('stopping screenshot SSE closes original HTTP and dispatches zero partial tools');
assert.deepEqual(errors,[]);fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify({success:true,version:host.bootstrap.version,appRoot,containedRuntime:args.length>0,realModelRequests:0,passed,screenshots,requests:requests.map(value=>({route:value.route,sha256:sha(JSON.stringify(value.body)),imageCount:requestImages(value.body).length})),pageErrors:errors},null,2));
}catch(error){await shot(page,'failure-desktop').catch(()=>{});if(mobile)await shot(mobile,'failure-mobile').catch(()=>{});fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify({success:false,error:error.stack,passed,screenshots,pageErrors:errors},null,2));throw error;
}finally{console.log('EVIDENCE '+directory);await browser.close();if(relay)await new Promise(resolve=>{relay.closeAllConnections();relay.close(resolve);});if(gateway)await host.api('/companion/close',{});for(const response of held)response.destroy();for(const socket of sockets)socket.destroy();await new Promise(resolve=>model.close(resolve));for(const dispose of disposers.reverse())await dispose();}