92 lines
15 KiB
JavaScript
92 lines
15 KiB
JavaScript
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import http from 'node:http';
|
||
import https from 'node:https';
|
||
import assert from 'node:assert/strict';
|
||
import { createHash,randomUUID,X509Certificate } from 'node:crypto';
|
||
import { chromium,expect } from 'playwright/test';
|
||
import { fixture,launchWorkbench,until } from '../tests/helpers.mjs';
|
||
import { requestImages } from '../src/shared/contracts/model-request.mjs';
|
||
import { PNG } from 'pngjs';
|
||
|
||
if(process.argv.includes('--help')){console.log('node scripts/verify-vision-ui.mjs [--app-root <absolute-package>] — owned image HTTP, desktop/390px upload/history/download, exact Task budget resume, TLS companion read/revoke and original SSE stop. No configured model calls or iPhone claim.');process.exit(0);}
|
||
const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1]));
|
||
const appRoot=fs.realpathSync.native(args[1]??process.cwd()),directory=path.resolve('.qa','vision-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t);f.store.close();
|
||
const requests=[],sockets=new Set(),held=new Set();let mode='success',closed=0;
|
||
const message=text=>({type:'message',role:'assistant',content:[{type:'output_text',text}]}),result=output=>({status:'completed',output,usage:{input_tokens:200,output_tokens:30,total_tokens:230}});
|
||
const model=http.createServer(async(req,res)=>{try{
|
||
const chunks=[];for await(const chunk of req)chunks.push(chunk);const body=JSON.parse(Buffer.concat(chunks));requests.push({route:req.url,body});
|
||
assert.ok(requestImages(body).length===1,'one screenshot per request');
|
||
assert.notEqual(body.model,'text-only','text member must never dispatch');
|
||
if(mode==='hold'){
|
||
assert.equal(body.stream,true);res.writeHead(200,{'Content-Type':'text/event-stream'});
|
||
res.write('data: '+JSON.stringify({type:'response.output_text.delta',sequence_number:0,delta:'截图正在核对;参数尚未完整。'})+'\n\n');
|
||
held.add(res);res.on('close',()=>{held.delete(res);closed++;});return;
|
||
}
|
||
const latestUser=body.input.findLastIndex(value=>value.role==='user'&&typeof value.content==='string');
|
||
const output=body.input.slice(latestUser+1).some(value=>value.type==='function_call_output')?[message('原截图与实际工程读取结果已经核对。')]
|
||
:[{type:'function_call',call_id:'image-files-'+requests.length,name:'list_files',arguments:'{"path":"","depth":1}'}];
|
||
res.writeHead(200,{'Content-Type':'application/json'});res.end(JSON.stringify(result(output)));
|
||
}catch(error){res.writeHead(500,{'Content-Type':'application/json'});res.end(JSON.stringify({error:{message:error.message}}));}});
|
||
model.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));});await new Promise(resolve=>model.listen(0,'127.0.0.1',resolve));
|
||
const host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath:args.length?path.join(appRoot,'runtime/node.exe'):process.execPath});assert.notEqual(new URL(host.url).port,'4318');
|
||
const browser=await chromium.launch({channel:'msedge',headless:true}),page=await browser.newPage({viewport:{width:1440,height:1000}}),passed=[],screenshots=[],errors=[];
|
||
page.on('pageerror',error=>errors.push(error.message));const get=()=>host.api('/sessions/'+f.session.id),mark=value=>{passed.push(value);console.log('PASS '+value);};
|
||
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||
async function shot(target,name){const file=path.join(directory,name+'.png');await target.screenshot({path:file});screenshots.push(file);assert.ok(await target.locator('body').evaluate(node=>node.scrollWidth<=innerWidth),'page overflow '+name);}
|
||
let gateway,relay,cookie='',mobile;
|
||
try{
|
||
const raw=new PNG({width:1280,height:720});for(let y=0;y<720;y++)for(let x=0;x<1280;x++)raw.data.set(x>900?[190,60,50,255]:[28,76,74,255],(y*1280+x)*4);
|
||
const imageFile=path.join(directory,'engineering-screenshot.png');fs.writeFileSync(imageFile,PNG.sync.write(raw));
|
||
const baseUrl='http://127.0.0.1:'+model.address().port+'/v1';
|
||
const vision=await host.api('/model-center/connections',{name:'截图验收 Responses',model:'owned-vision',protocol:'responses',baseUrl,requiresKey:false,contextLimit:64000,capabilityDeclarations:{vision:'supported'}});
|
||
const center=await host.api('/model-center'),visionId=vision.id??center.connections.find(row=>row.name==='截图验收 Responses').id;
|
||
await host.api('/model-center/connections',{name:'纯文本排除样本',model:'text-only',baseUrl,requiresKey:false,contextLimit:64000});
|
||
const textId=(await host.api('/model-center')).connections.find(row=>row.model==='text-only').id;
|
||
const pool=await host.api('/model-center/pools',{name:'视觉验收池',members:[{connectionId:textId},{connectionId:visionId}]});
|
||
const poolId=pool.id??(await host.api('/model-center')).pools.find(row=>row.name==='视觉验收池').id;
|
||
await host.api('/model-center/select',{type:'pool',id:poolId});
|
||
await page.addInitScript(({projectId,sessionId})=>{localStorage.setItem('awb-project',projectId);localStorage.setItem('awb-session',sessionId);},{projectId:f.project.id,sessionId:f.session.id});
|
||
await page.goto(host.url);await page.waitForLoadState('networkidle');
|
||
await page.getByLabel('任务内容',{exact:true}).fill('根据这张截图核对当前工程,保留原工具结果。');
|
||
await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(imageFile);await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');
|
||
await expect(page.locator('.input-route')).toContainText('视觉验收池');await shot(page,'desktop-draft');
|
||
await page.getByRole('button',{name:'查看截图 engineering-screenshot.png',exact:true}).click();const dialog=page.getByRole('dialog');await expect(dialog).toContainText('1280 × 720');
|
||
const downloadPromise=page.waitForEvent('download');await dialog.getByRole('button',{name:'下载原图',exact:true}).click();const download=await downloadPromise,file=path.join(directory,'downloaded-original.png');await download.saveAs(file);assert.equal(sha(fs.readFileSync(file)),sha(fs.readFileSync(imageFile)));await page.keyboard.press('Escape');
|
||
await page.reload();await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');await page.locator('.input-draft input[type=checkbox]').check();
|
||
await page.getByLabel('任务内容',{exact:true}).fill('根据截图核对工程。');await page.getByRole('button',{name:'发送任务',exact:true}).click();await until(async()=>(await get()).status==='completed',15000);
|
||
let session=await get();assert.equal(requests.length,2);assert.equal(session.execution.progress.completedTools,1);assert.equal(session.inputAttachments.filter(row=>row.status==='attached').length,1);
|
||
await expect(page.locator('.message-user .input-receipt')).toContainText('SHA-256 已核对');await shot(page,'desktop-attached');await page.setViewportSize({width:390,height:844});await shot(page,'narrow-attached');await page.setViewportSize({width:1440,height:1000});
|
||
mark('desktop upload, explicit route, refreshed draft, original download SHA, one paired tool and narrow task history');
|
||
const invalid=path.join(directory,'forged.png');fs.writeFileSync(invalid,'not a real PNG');await page.getByLabel('任务内容',{exact:true}).fill('失败后保留输入');await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(invalid);await expect(page.locator('.toast')).toContainText('类型不符');await expect(page.getByLabel('任务内容',{exact:true})).toHaveValue('失败后保留输入');assert.equal(requests.length,2);mark('forged image rejects without losing task text or sending a model request');
|
||
// Reuse the actual completed tool group in the same Task after a deliberate
|
||
// insufficient context resume; no completed tool is requested again.
|
||
// Completed Tasks are immutable history. Start a new screenshot Task, stop at
|
||
// one tool round, then exercise budget continuation within that Task.
|
||
await host.api('/config',{...host.bootstrap.config,mode:'compatible',model:'unused',baseUrl,contextLimit:16000,maxSteps:1});
|
||
await host.api('/model-center/select',{type:'pool',id:poolId});
|
||
await page.reload();await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(imageFile);await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');await page.getByLabel('任务内容',{exact:true}).fill('截图任务预算续接验收');await page.getByRole('button',{name:'发送任务',exact:true}).click();await until(async()=>(await get()).status==='paused',15000);
|
||
const original=await get(),taskId=original.execution.task.id,count=requests.length;
|
||
await host.api('/config',{...(await host.api('/bootstrap')).config,contextLimit:1000});await host.api('/model-center/select',{type:'pool',id:poolId});await host.api('/sessions/'+f.session.id+'/resume',{});await until(async()=>(await get()).execution.run.budgetPause,15000);
|
||
await page.reload();await expect(page.getByLabel('任务预算诊断')).toContainText('图像本地预留');await page.getByRole('button',{name:'核对预算并续接原任务',exact:true}).click();
|
||
await dialog.getByLabel('上下文窗口',{exact:true}).fill('64000');await dialog.getByLabel('新Run工具轮次',{exact:true}).fill('3');await dialog.getByRole('button',{name:'准备续接预览',exact:true}).click();await expect(dialog.locator('.budget-preview')).toContainText('保留 1 项完成工具、1 张附件');assert.equal(requests.length,count);await shot(page,'desktop-budget-preview');
|
||
await page.setViewportSize({width:390,height:844});await shot(page,'narrow-budget-preview');await dialog.getByRole('button',{name:'确认预算并续接',exact:true}).click();await until(async()=>(await get()).status==='completed',15000);
|
||
session=await get();assert.equal(session.execution.task.id,taskId);assert.equal(session.execution.progress.completedTools,1);assert.equal(requests.length,count+1);assert.equal((await host.api('/bootstrap')).config.contextLimit,1000);await page.setViewportSize({width:1440,height:1000});mark('explicit budget preview and confirmation retain Task, attachment and completed tool while global capacity stays unchanged');
|
||
const initial=await host.api('/companion'),reservation=http.createServer();assert.ok(initial.addresses.length);await new Promise(resolve=>reservation.listen(0,initial.addresses[0].address,resolve));const port=reservation.address().port;await new Promise(resolve=>reservation.close(resolve));
|
||
await host.api('/companion/start',{address:initial.addresses[0].address,port});gateway=await host.api('/companion');const certificate=await fetch(host.url+'/api/companion/certificate',{headers:{'X-Workbench-Token':host.bootstrap.token}}),ca=new X509Certificate(Buffer.from(await certificate.arrayBuffer())).toString(),target=new URL(gateway.origin);
|
||
const ticket=await host.api('/companion/pair',{projectIds:[f.project.id],permissions:{observe:true,approve:false,stop:false},expiresInHours:1});
|
||
relay=http.createServer((request,response)=>{const forwarded=https.request(new URL(request.url,target),{method:request.method,ca,servername:'',headers:{Host:target.host,Origin:target.origin,Connection:'close',...(cookie?{Cookie:cookie}:{}),...(request.headers['content-type']?{'Content-Type':request.headers['content-type']}:{})}},incoming=>{if(incoming.headers['set-cookie']?.length)cookie=incoming.headers['set-cookie'][0].split(';')[0];const headers={...incoming.headers};delete headers['set-cookie'];response.writeHead(incoming.statusCode,headers);incoming.pipe(response);});forwarded.on('error',()=>response.destroy());request.pipe(forwarded);});await new Promise(resolve=>relay.listen(0,'127.0.0.1',resolve));
|
||
mobile=await browser.newPage({viewport:{width:390,height:844},isMobile:true,hasTouch:true});mobile.on('pageerror',error=>errors.push(error.message));await mobile.goto('http://127.0.0.1:'+relay.address().port+'/#pair='+new URLSearchParams(new URL(ticket.url).hash.slice(1)).get('pair'));await mobile.getByRole('button',{name:'连接这台电脑',exact:true}).click();await mobile.locator('.mobile-task-row').first().click();
|
||
await expect(mobile.locator('.mobile-input-image')).toContainText('SHA-256 已核对');await mobile.locator('.mobile-input-image summary').click();await shot(mobile,'companion-image');await mobile.reload();await expect(mobile.locator('.mobile-task-row').first()).toBeVisible();await mobile.locator('.mobile-task-row').first().click();await expect(mobile.locator('.mobile-input-image')).toContainText('SHA-256 已核对');
|
||
const device=(await host.api('/companion')).devices.find(row=>row.status==='active');await host.api('/companion/devices/'+device.id+'/revoke',{revision:device.revision});await mobile.getByRole('button',{name:'刷新任务',exact:true}).click();await expect(mobile.locator('.mobile-input-image')).toHaveCount(0);await expect(mobile.getByRole('button',{name:'连接这台电脑',exact:true})).toBeVisible();mark('paired TLS companion verifies same screenshot, refresh restores it and revoke clears project/image content');
|
||
await host.api('/companion/close',{});gateway=null;
|
||
// This case measures cancellation, independently of the deliberate low-budget
|
||
// case above. Leave room for the current shipped tool definitions and image.
|
||
await host.api('/config',{...(await host.api('/bootstrap')).config,contextLimit:64000,maxSteps:3});
|
||
const connection=(await host.api('/model-center')).connections.find(row=>row.id===visionId);await host.api('/model-center/connections',{...connection,streaming:true,capabilityDeclarations:{...connection.capabilityDeclarations,streaming:'supported'}});await host.api('/model-center/select',{type:'connection',id:visionId});mode='hold';
|
||
await page.reload();await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(imageFile);await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');await page.getByLabel('任务内容',{exact:true}).fill('截图流式停止验收');await page.getByRole('button',{name:'发送任务',exact:true}).click();await expect(page.getByLabel('模型输出片段',{exact:true}).last()).toContainText('截图正在核对');const stopping=await get();await page.getByRole('button',{name:'停止',exact:true}).click();await until(async()=>(await get()).status==='paused'&&closed===1,15000);session=await get();assert.equal(session.events.filter(row=>row.kind==='tool'&&row.runId===stopping.runId).length,0);await shot(page,'desktop-image-stopped');mark('stopping screenshot SSE closes original HTTP and dispatches zero partial tools');
|
||
assert.deepEqual(errors,[]);fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify({success:true,version:host.bootstrap.version,appRoot,containedRuntime:args.length>0,realModelRequests:0,passed,screenshots,requests:requests.map(value=>({route:value.route,sha256:sha(JSON.stringify(value.body)),imageCount:requestImages(value.body).length})),pageErrors:errors},null,2));
|
||
}catch(error){await shot(page,'failure-desktop').catch(()=>{});if(mobile)await shot(mobile,'failure-mobile').catch(()=>{});fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify({success:false,error:error.stack,passed,screenshots,pageErrors:errors},null,2));throw error;
|
||
}finally{console.log('EVIDENCE '+directory);await browser.close();if(relay)await new Promise(resolve=>{relay.closeAllConnections();relay.close(resolve);});if(gateway)await host.api('/companion/close',{});for(const response of held)response.destroy();for(const socket of sockets)socket.destroy();await new Promise(resolve=>model.close(resolve));for(const dispose of disposers.reverse())await dispose();}
|