Files
Agent-Workbench/scripts/verify-live-browser.mjs
T
2026-10-30 09:12:49 +08:00

102 lines
12 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { gitEnvironment } from '../src/core/git-process.mjs';
const previous = JSON.parse(fs.readFileSync('docs/live-react-engineering-validation.json', 'utf8'));
const url = 'http://127.0.0.1:4318', directory = path.resolve('.qa', 'live-browser-' + randomUUID()); fs.mkdirSync(directory);
const sha = value => createHash('sha256').update(Buffer.isBuffer(value) || typeof value === 'string' ? value : JSON.stringify(value)).digest('hex');
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
let token, sessionId, taskId, current, result; const decisions = [], continuations = [], startedAt = Date.now();
async function api(route, input) {
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { Connection: 'close', 'Content-Type': 'application/json', ...(token ? { 'X-Workbench-Token': token } : {}) },
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
const value = await response.json(); if (!response.ok) throw new Error(value.error ?? 'HTTP ' + response.status); return value;
}
const root = previous.workspaceRoot, git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', root, ...args], { env: gitEnvironment(), windowsHide: true, encoding: 'utf8' }).trim();
const sourceFiles = () => {
const walk = (relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
if (entry.name.startsWith('.') || ['node_modules', 'dist', 'build', 'out'].includes(entry.name)) return [];
assert.equal(entry.isSymbolicLink(), false); const file = path.join(relative, entry.name);
return entry.isDirectory() ? walk(file) : [{ path: file, sha256: sha(fs.readFileSync(path.join(root, file))) }];
}); return walk();
};
async function original(job) {
let text = '', offset = 0, page;
do { page = await api('/sessions/' + sessionId + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=16000'); text += page.content; offset += page.content.length; } while (page.truncated);
assert.equal(sha(text), job.resultSha256); const value = JSON.parse(text); assert.equal(value.taskId, taskId);
fs.writeFileSync(path.join(directory, 'original-' + job.id + '.json'), text);
for (const image of job.screenshots) {
const response = await fetch(url + '/api/sessions/' + sessionId + '/browser-jobs/' + job.id + '/images/' + image.artifactId, { headers: { 'X-Workbench-Token': token }, signal: AbortSignal.timeout(15000) });
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), image.sha256);
fs.writeFileSync(path.join(directory, job.id + '-' + image.label + '.png'), bytes);
} return value;
}
try {
assert.equal(previous.success, true);
const before = await api('/bootstrap'); token = before.token; assert.equal(before.version, '1.10.1'); assert.equal(before.config.mode, 'compatible'); assert.equal(before.config.hasKey, true);
const project = before.projects.find(value => value.id === previous.projectId); assert.ok(project); assert.equal(project.root, root);
assert.ok(!before.sessions.some(value => value.projectId === project.id && ['running', 'awaiting_approval', 'stopping'].includes(value.status)));
const baseline = { source: sourceFiles(), head: git(['rev-parse', 'HEAD']), index: sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))),
originalSession: sha(canonical(await api('/sessions/' + previous.sessionId + '/export'))), configuration: sha(canonical(before.config)) };
// exportedAt is a transport timestamp, not a persisted session field.
const old = await api('/sessions/' + previous.sessionId + '/export'); delete old.exportedAt; baseline.originalSession = sha(canonical(old));
const created = await api('/sessions', { projectId: project.id }); sessionId = created.id;
const prompt = '请使用工作台内置plan_browser、run_browser、read_browser_result验证当前已经完成的React工程实际页面。工程代码已经通过测试并提交,本次只检查页面,不修改文件、不执行终端、不准备Git提交。'
+ '先读取实际DOM/元素,再由你根据实际页面选择CSS交互和断言。桌面1280×900:初始3项任务,状态筛选pending为2项;搜索带空格的“ 接口 ”只显示编写接口;将该任务标为完成后待办筛选下变空,统计应变为完成2项、待办工时5小时。'
+ '再以390×844独立浏览器核对手机布局可见且没有横向溢出。每份计划最多20步,需要实际断言与原截图,observed不算通过。根据原错误准备新计划,最后基于原结果向我报告,不把截图或推测当成功。';
await api('/sessions/' + sessionId + '/messages', { content: prompt });
const decided = new Set();
while (Date.now() - startedAt < 210000) {
current = await api('/sessions/' + sessionId); taskId ??= current.execution?.task?.id; assert.equal(current.execution.task.id, taskId);
if (current.changes.some(value => ['pending', 'conflict', 'applied'].includes(value.status))) throw new Error('The page-only task proposed source changes; approval was not given.');
if (current.status === 'awaiting_approval') {
const approval = current.approvals.find(value => value.status === 'pending' && !decided.has(value.id));
if (approval) {
assert.ok(approval.browserPlanId, 'Only the explicitly scoped page execution may be approved.');
const plan = current.execution.browserPlans.find(value => value.id === approval.browserPlanId);
assert.equal(plan.taskId, taskId); assert.equal(plan.projectId, project.id); assert.equal(plan.build.directory, 'dist'); assert.equal(plan.path, '/');
assert.ok([1280, 390].includes(plan.viewport.width)); assert.ok(plan.steps.length <= 20);
decided.add(approval.id); decisions.push({ id: approval.id, taskId, planId: plan.id, fingerprint: plan.fingerprint, viewport: plan.viewport, steps: plan.steps.length });
await api('/sessions/' + sessionId + '/approvals/' + approval.id, { allow: true });
}
} else if (['failed', 'cancelled'].includes(current.status)) throw new Error(current.failure?.message ?? current.status);
else if (['completed', 'paused'].includes(current.status)) {
const passed = current.execution.browserJobs.filter(job => job.taskId === taskId && job.status === 'passed');
if (current.status === 'completed' && passed.some(job => current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 1280)
&& passed.some(job => current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 390)) break;
if (continuations.length >= 2 || current.status !== 'paused' || current.execution.run.stopReason !== 'step_limit') throw new Error('Actual model did not complete both requested page checks: ' + current.status + '/' + current.execution.run.stopReason);
continuations.push({ previousRunId: current.execution.run.id, reason: current.execution.run.stopReason, at: new Date().toISOString() });
await api('/sessions/' + sessionId + '/resume', {});
}
await new Promise(resolve => setTimeout(resolve, 75));
}
assert.equal(current.status, 'completed', 'Actual model task exceeded the acceptance time bound.');
const jobs = current.execution.browserJobs.filter(job => job.taskId === taskId), records = [];
for (const job of jobs) { assert.equal(job.cleanup?.exited, true); records.push({ job, original: await original(job), viewport: current.execution.browserPlans.find(plan => plan.id === job.planId).viewport }); }
const desktop = records.find(value => value.viewport.width === 1280 && value.job.status === 'passed' && ['fill', 'select', 'click'].every(action => value.job.steps.some(step => step.action === action)));
assert.ok(desktop, 'Desktop acceptance needs real search/filter/click, not only screenshots.');
const text = desktop.original.pages.at(-1).text; assert.match(text, /完成任务\s*2/); assert.match(text, /待办工时\s*5\s*小时/);
const mobile = records.find(value => value.viewport.width === 390 && value.job.status === 'passed'); assert.ok(mobile); assert.ok(mobile.original.pages.at(-1).viewport.scrollWidth <= 390);
assert.deepEqual(sourceFiles(), baseline.source); assert.equal(git(['rev-parse', 'HEAD']), baseline.head); assert.equal(sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))), baseline.index);
const afterOld = await api('/sessions/' + previous.sessionId + '/export'); delete afterOld.exportedAt; assert.equal(sha(canonical(afterOld)), baseline.originalSession);
assert.equal(sha(canonical((await api('/bootstrap')).config)), baseline.configuration);
const exported = await api('/sessions/' + sessionId + '/export'); fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n');
result = { success: true, version: '1.10.1', testedAt: new Date().toISOString(), kind: 'configured-model-inner-browser-acceptance', model: before.config.model,
directory, projectId: project.id, sessionId, taskId, root, sourceCommit: baseline.head, sourcePreserved: true, indexPreserved: true, originalEngineeringTaskPreserved: true,
modelStatus: current.status, usage: current.execution.task.usage, elapsedSeconds: (Date.now() - startedAt) / 1000, decisions, continuations,
jobs: records.map(value => ({ id: value.job.id, status: value.job.status, viewport: value.viewport, assertions: value.job.assertions, steps: value.job.steps, resultSha256: value.job.resultSha256, screenshots: value.job.screenshots, cleanup: value.job.cleanup })),
scope: 'Configured model chooses CSS plans and reads original fields through the self harness. The verifier confirms only sealed local page plans and validates original DOM/PNG, source/index and previous Task. This read-only page sample is separate from code-repair success rate and Purge benefit.',
credentialsHandling: 'The original in-memory Provider performs requests; no model key is read, copied, printed or saved.' };
} catch (error) {
if (sessionId) { await api('/sessions/' + sessionId + '/stop', {}).catch(() => {}); current = await api('/sessions/' + sessionId).catch(() => current); }
result = { success: false, version: '1.10.1', testedAt: new Date().toISOString(), directory, sessionId, taskId, modelStatus: current?.status, usage: current?.execution?.task?.usage, decisions, continuations, error: error.stack }; process.exitCode = 1;
if (sessionId) { const exported = await api('/sessions/' + sessionId + '/export').catch(() => null); if (exported) fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n'); }
}
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
fs.writeFileSync('docs/browser-engine-live-model-validation.json', JSON.stringify(result, null, 2) + '\n');
console.log(JSON.stringify({ success: result.success, sessionId, taskId, modelStatus: result.modelStatus, usage: result.usage, elapsedSeconds: result.elapsedSeconds, jobs: result.jobs?.map(value => ({ id: value.id, status: value.status, viewport: value.viewport, assertions: value.assertions })), error: result.error, directory }));