Files
Agent-Workbench/scripts/verify-model-identity-ui.mjs
T
2026-11-06 14:08:53 +08:00

80 lines
8.3 KiB
JavaScript

import { openWorkbenchTool } from './ui-navigation.mjs';
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import assert from 'node:assert/strict';
import {createHash,randomUUID} from 'node:crypto';
import {chromium,expect} from 'playwright/test';
import {fixture,launchWorkbench,until} from '../tests/helpers.mjs';
import {Provider} from '../src/server/provider.mjs';
if(process.argv.includes('--help')){console.log('node scripts/verify-model-identity-ui.mjs [--app-root ABSOLUTE_ROOT] — compiled or packaged host, owned identity fixture and desktop/390px UI. No configured-model requests.');process.exit(0);}
const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1]),'Use --app-root with an absolute directory.');
const appRoot=args.length?fs.realpathSync.native(args[1]):process.cwd();
const targetVersion=JSON.parse(fs.readFileSync(path.join(appRoot,fs.existsSync(path.join(appRoot,'MANIFEST.json'))?'MANIFEST.json':'package.json'))).version;
const directory=path.resolve('.qa','model-identity-ui-'+randomUUID()),disposers=[],checks=[],screenshots=[],errors=[],requests=[];
fs.mkdirSync(directory,{recursive:true});const t={after:fn=>disposers.push(fn)},f=fixture(t);
const upstream=http.createServer(async(req,res)=>{
const chunks=[];for await(const chunk of req)chunks.push(chunk);requests.push(JSON.parse(Buffer.concat(chunks)));
res.writeHead(200,{'Content-Type':'application/json'});
res.end(JSON.stringify({model:'owned-upstream-snapshot',choices:[{message:{role:'assistant',content:'',tool_calls:[
{id:'owned-ui-call',type:'function',function:{name:'propose_file',arguments:JSON.stringify({path:'unsafe.txt',content:'must not execute',reason:'fixture'})}}]}}],
usage:{prompt_tokens:13,completion_tokens:7,total_tokens:20}}));
});
await new Promise(resolve=>upstream.listen(0,'127.0.0.1',resolve));
const provider=new Provider(),hub=provider.modelHub(f.store),connection=hub.upsert({name:'身份核对 · 自有接口夹具',model:'owned-requested-model',
baseUrl:'http://127.0.0.1:'+upstream.address().port+'/v1',requiresKey:false});
hub.select({type:'connection',id:connection.id});f.store.close();
const nodePath=fs.existsSync(path.join(appRoot,'runtime/node.exe'))?path.join(appRoot,'runtime/node.exe'):process.execPath;
const host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath});assert.notEqual(new URL(host.url).port,'4318');
let browser,page,result;
const capture=async name=>{const file=path.join(directory,name+'.png');await page.screenshot({path:file});screenshots.push({path:file,sha256:createHash('sha256').update(fs.readFileSync(file)).digest('hex')});};
const mark=name=>{checks.push(name);console.log('PASS '+name);};
const openCenter=async()=>{await openWorkbenchTool(page, '打开模型中心');return page.getByRole('dialog',{name:'模型中心',exact:true});};
try{
browser=await chromium.launch({channel:'msedge',headless:true});page=await browser.newPage({viewport:{width:1440,height:1000}});
page.on('pageerror',e=>errors.push(e.message));await page.goto(host.url);await page.waitForLoadState('networkidle');
assert.equal((await host.api('/health')).version,targetVersion);await page.getByLabel('当前项目').selectOption(f.project.id);
let center=await openCenter();const card=()=>center.locator('.mc-connection').filter({has:page.getByRole('heading',{name:connection.name,exact:true})});
await card().getByRole('button',{name:'编辑',exact:true}).click();const form=()=>page.getByRole('form',{name:'编辑模型连接',exact:true});
await form().getByLabel('型号核对',{exact:true}).selectOption('require-match');
await form().getByLabel('允许的返回型号',{exact:false}).fill('snapshot-a, snapshot-b');
await form().getByLabel('型号映射依据',{exact:true}).fill('自有接口映射约定 revision 1');
await capture('settings-desktop');await page.setViewportSize({width:390,height:844});
await form().getByRole('button',{name:'保存模型连接',exact:true}).scrollIntoViewIfNeeded();
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth));await capture('settings-390');
await form().getByRole('button',{name:'保存模型连接',exact:true}).click();await expect(form()).toHaveCount(0);
await page.reload();center=await openCenter();await card().getByRole('button',{name:'编辑',exact:true}).click();
await expect(form().getByLabel('型号核对',{exact:true})).toHaveValue('require-match');
await expect(form().getByLabel('允许的返回型号',{exact:false})).toHaveValue('snapshot-a, snapshot-b');
await expect(form().getByLabel('型号映射依据',{exact:true})).toHaveValue('自有接口映射约定 revision 1');
mark('strict policy and multiple declared aliases save and reload at desktop and 390px without a model call');assert.equal(requests.length,0);
await form().getByLabel('允许的返回型号',{exact:false}).fill('');await form().getByLabel('型号映射依据',{exact:true}).fill('');
await form().getByRole('button',{name:'保存模型连接',exact:true}).click();await expect(form()).toHaveCount(0);await page.keyboard.press('Escape');
await page.setViewportSize({width:1440,height:1000});const prompt='核对当前项目,身份不一致应暂停';
await page.getByLabel('任务内容').fill(prompt);await page.getByRole('button',{name:'发送任务',exact:true}).click();let session;
await until(async()=>{const id=(await host.api('/sessions')).find(v=>v.title===prompt)?.id;session=id&&await host.api('/sessions/'+id);return session?.status==='paused';},15000);
assert.equal(requests.length,1);assert.equal(session.changes.length,0);assert.equal(session.execution.run.stopReason,'model_identity');
await expect(page.locator('.tool-name strong').filter({hasText:'模型返回型号需要核对'})).toHaveCount(1);
await expect(page.locator('.model-identity-details').first()).toContainText('请求与返回型号不一致');await page.reload();
const failure=page.locator('.failure-card').last();await failure.locator('.model-identity-details summary').click();
await expect(failure).toContainText('owned-requested-model');await expect(failure).toContainText('owned-upstream-snapshot');
await expect(failure).toContainText('不一致或无法核对时暂停');await capture('paused-desktop');
await page.setViewportSize({width:390,height:844});await failure.locator('.model-identity-details').scrollIntoViewIfNeeded();await capture('paused-390');
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth));
const downloadPromise=page.waitForEvent('download');await failure.getByRole('button',{name:'下载原响应',exact:true}).click();
const download=await downloadPromise,saved=path.join(directory,'original-response.json');await download.saveAs(saved);
const original=fs.readFileSync(saved),identity=session.execution.run.consumption[0].modelIdentity;
assert.equal(createHash('sha256').update(original).digest('hex'),identity.responseSource.sha256);assert.equal(JSON.parse(original).model,'owned-upstream-snapshot');
mark('strict mismatch pauses before any proposal, retains reported usage, persists on refresh and downloads SHA-verified original');
center=await openCenter();await center.getByRole('button',{name:'请求账本',exact:true}).click();
const attempt=center.locator('.mc-attempts article').first();await attempt.locator('.model-identity-details').first().locator('summary').click();
await expect(attempt).toContainText('owned-upstream-snapshot');await expect(attempt).toContainText(identity.responseSource.sha256);
await capture('ledger-390');assert.deepEqual(errors,[]);mark('physical request ledger retains the same identity and source without page errors');
result={success:true,appRoot,targetVersion,testedVersion:targetVersion,checks,screenshots,errors,modelRequests:requests.length,realModelRequests:0};
}catch(error){if(page)await capture('failure').catch(()=>{});result={success:false,error:error.stack,checks,screenshots,errors,realModelRequests:0};throw error;}
finally{
fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(result,null,2));console.log(JSON.stringify({success:result.success,directory,checks:checks.length}));
await browser?.close();upstream.closeAllConnections();await new Promise(resolve=>upstream.close(resolve));for(const action of disposers.reverse())await action();
}