102 lines
12 KiB
JavaScript
102 lines
12 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { gitEnvironment } from '../src/core/git-process.mjs';
|
|
|
|
const previous = JSON.parse(fs.readFileSync('docs/live-react-engineering-validation.json', 'utf8'));
|
|
const url = 'http://127.0.0.1:4318', directory = path.resolve('.qa', 'live-browser-' + randomUUID()); fs.mkdirSync(directory);
|
|
const sha = value => createHash('sha256').update(Buffer.isBuffer(value) || typeof value === 'string' ? value : JSON.stringify(value)).digest('hex');
|
|
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
|
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
|
let token, sessionId, taskId, current, result; const decisions = [], continuations = [], startedAt = Date.now();
|
|
async function api(route, input) {
|
|
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { Connection: 'close', 'Content-Type': 'application/json', ...(token ? { 'X-Workbench-Token': token } : {}) },
|
|
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
|
|
const value = await response.json(); if (!response.ok) throw new Error(value.error ?? 'HTTP ' + response.status); return value;
|
|
}
|
|
const root = previous.workspaceRoot, git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', root, ...args], { env: gitEnvironment(), windowsHide: true, encoding: 'utf8' }).trim();
|
|
const sourceFiles = () => {
|
|
const walk = (relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
|
|
if (entry.name.startsWith('.') || ['node_modules', 'dist', 'build', 'out'].includes(entry.name)) return [];
|
|
assert.equal(entry.isSymbolicLink(), false); const file = path.join(relative, entry.name);
|
|
return entry.isDirectory() ? walk(file) : [{ path: file, sha256: sha(fs.readFileSync(path.join(root, file))) }];
|
|
}); return walk();
|
|
};
|
|
async function original(job) {
|
|
let text = '', offset = 0, page;
|
|
do { page = await api('/sessions/' + sessionId + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=16000'); text += page.content; offset += page.content.length; } while (page.truncated);
|
|
assert.equal(sha(text), job.resultSha256); const value = JSON.parse(text); assert.equal(value.taskId, taskId);
|
|
fs.writeFileSync(path.join(directory, 'original-' + job.id + '.json'), text);
|
|
for (const image of job.screenshots) {
|
|
const response = await fetch(url + '/api/sessions/' + sessionId + '/browser-jobs/' + job.id + '/images/' + image.artifactId, { headers: { 'X-Workbench-Token': token }, signal: AbortSignal.timeout(15000) });
|
|
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), image.sha256);
|
|
fs.writeFileSync(path.join(directory, job.id + '-' + image.label + '.png'), bytes);
|
|
} return value;
|
|
}
|
|
try {
|
|
assert.equal(previous.success, true);
|
|
const before = await api('/bootstrap'); token = before.token; assert.equal(before.version, '1.10.1'); assert.equal(before.config.mode, 'compatible'); assert.equal(before.config.hasKey, true);
|
|
const project = before.projects.find(value => value.id === previous.projectId); assert.ok(project); assert.equal(project.root, root);
|
|
assert.ok(!before.sessions.some(value => value.projectId === project.id && ['running', 'awaiting_approval', 'stopping'].includes(value.status)));
|
|
const baseline = { source: sourceFiles(), head: git(['rev-parse', 'HEAD']), index: sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))),
|
|
originalSession: sha(canonical(await api('/sessions/' + previous.sessionId + '/export'))), configuration: sha(canonical(before.config)) };
|
|
// exportedAt is a transport timestamp, not a persisted session field.
|
|
const old = await api('/sessions/' + previous.sessionId + '/export'); delete old.exportedAt; baseline.originalSession = sha(canonical(old));
|
|
const created = await api('/sessions', { projectId: project.id }); sessionId = created.id;
|
|
const prompt = '请使用工作台内置plan_browser、run_browser、read_browser_result验证当前已经完成的React工程实际页面。工程代码已经通过测试并提交,本次只检查页面,不修改文件、不执行终端、不准备Git提交。'
|
|
+ '先读取实际DOM/元素,再由你根据实际页面选择CSS交互和断言。桌面1280×900:初始3项任务,状态筛选pending为2项;搜索带空格的“ 接口 ”只显示编写接口;将该任务标为完成后待办筛选下变空,统计应变为完成2项、待办工时5小时。'
|
|
+ '再以390×844独立浏览器核对手机布局可见且没有横向溢出。每份计划最多20步,需要实际断言与原截图,observed不算通过。根据原错误准备新计划,最后基于原结果向我报告,不把截图或推测当成功。';
|
|
await api('/sessions/' + sessionId + '/messages', { content: prompt });
|
|
const decided = new Set();
|
|
while (Date.now() - startedAt < 210000) {
|
|
current = await api('/sessions/' + sessionId); taskId ??= current.execution?.task?.id; assert.equal(current.execution.task.id, taskId);
|
|
if (current.changes.some(value => ['pending', 'conflict', 'applied'].includes(value.status))) throw new Error('The page-only task proposed source changes; approval was not given.');
|
|
if (current.status === 'awaiting_approval') {
|
|
const approval = current.approvals.find(value => value.status === 'pending' && !decided.has(value.id));
|
|
if (approval) {
|
|
assert.ok(approval.browserPlanId, 'Only the explicitly scoped page execution may be approved.');
|
|
const plan = current.execution.browserPlans.find(value => value.id === approval.browserPlanId);
|
|
assert.equal(plan.taskId, taskId); assert.equal(plan.projectId, project.id); assert.equal(plan.build.directory, 'dist'); assert.equal(plan.path, '/');
|
|
assert.ok([1280, 390].includes(plan.viewport.width)); assert.ok(plan.steps.length <= 20);
|
|
decided.add(approval.id); decisions.push({ id: approval.id, taskId, planId: plan.id, fingerprint: plan.fingerprint, viewport: plan.viewport, steps: plan.steps.length });
|
|
await api('/sessions/' + sessionId + '/approvals/' + approval.id, { allow: true });
|
|
}
|
|
} else if (['failed', 'cancelled'].includes(current.status)) throw new Error(current.failure?.message ?? current.status);
|
|
else if (['completed', 'paused'].includes(current.status)) {
|
|
const passed = current.execution.browserJobs.filter(job => job.taskId === taskId && job.status === 'passed');
|
|
if (current.status === 'completed' && passed.some(job => current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 1280)
|
|
&& passed.some(job => current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 390)) break;
|
|
if (continuations.length >= 2 || current.status !== 'paused' || current.execution.run.stopReason !== 'step_limit') throw new Error('Actual model did not complete both requested page checks: ' + current.status + '/' + current.execution.run.stopReason);
|
|
continuations.push({ previousRunId: current.execution.run.id, reason: current.execution.run.stopReason, at: new Date().toISOString() });
|
|
await api('/sessions/' + sessionId + '/resume', {});
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 75));
|
|
}
|
|
assert.equal(current.status, 'completed', 'Actual model task exceeded the acceptance time bound.');
|
|
const jobs = current.execution.browserJobs.filter(job => job.taskId === taskId), records = [];
|
|
for (const job of jobs) { assert.equal(job.cleanup?.exited, true); records.push({ job, original: await original(job), viewport: current.execution.browserPlans.find(plan => plan.id === job.planId).viewport }); }
|
|
const desktop = records.find(value => value.viewport.width === 1280 && value.job.status === 'passed' && ['fill', 'select', 'click'].every(action => value.job.steps.some(step => step.action === action)));
|
|
assert.ok(desktop, 'Desktop acceptance needs real search/filter/click, not only screenshots.');
|
|
const text = desktop.original.pages.at(-1).text; assert.match(text, /完成任务\s*2/); assert.match(text, /待办工时\s*5\s*小时/);
|
|
const mobile = records.find(value => value.viewport.width === 390 && value.job.status === 'passed'); assert.ok(mobile); assert.ok(mobile.original.pages.at(-1).viewport.scrollWidth <= 390);
|
|
assert.deepEqual(sourceFiles(), baseline.source); assert.equal(git(['rev-parse', 'HEAD']), baseline.head); assert.equal(sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))), baseline.index);
|
|
const afterOld = await api('/sessions/' + previous.sessionId + '/export'); delete afterOld.exportedAt; assert.equal(sha(canonical(afterOld)), baseline.originalSession);
|
|
assert.equal(sha(canonical((await api('/bootstrap')).config)), baseline.configuration);
|
|
const exported = await api('/sessions/' + sessionId + '/export'); fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n');
|
|
result = { success: true, version: '1.10.1', testedAt: new Date().toISOString(), kind: 'configured-model-inner-browser-acceptance', model: before.config.model,
|
|
directory, projectId: project.id, sessionId, taskId, root, sourceCommit: baseline.head, sourcePreserved: true, indexPreserved: true, originalEngineeringTaskPreserved: true,
|
|
modelStatus: current.status, usage: current.execution.task.usage, elapsedSeconds: (Date.now() - startedAt) / 1000, decisions, continuations,
|
|
jobs: records.map(value => ({ id: value.job.id, status: value.job.status, viewport: value.viewport, assertions: value.job.assertions, steps: value.job.steps, resultSha256: value.job.resultSha256, screenshots: value.job.screenshots, cleanup: value.job.cleanup })),
|
|
scope: 'Configured model chooses CSS plans and reads original fields through the self harness. The verifier confirms only sealed local page plans and validates original DOM/PNG, source/index and previous Task. This read-only page sample is separate from code-repair success rate and Purge benefit.',
|
|
credentialsHandling: 'The original in-memory Provider performs requests; no model key is read, copied, printed or saved.' };
|
|
} catch (error) {
|
|
if (sessionId) { await api('/sessions/' + sessionId + '/stop', {}).catch(() => {}); current = await api('/sessions/' + sessionId).catch(() => current); }
|
|
result = { success: false, version: '1.10.1', testedAt: new Date().toISOString(), directory, sessionId, taskId, modelStatus: current?.status, usage: current?.execution?.task?.usage, decisions, continuations, error: error.stack }; process.exitCode = 1;
|
|
if (sessionId) { const exported = await api('/sessions/' + sessionId + '/export').catch(() => null); if (exported) fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n'); }
|
|
}
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
|
fs.writeFileSync('docs/browser-engine-live-model-validation.json', JSON.stringify(result, null, 2) + '\n');
|
|
console.log(JSON.stringify({ success: result.success, sessionId, taskId, modelStatus: result.modelStatus, usage: result.usage, elapsedSeconds: result.elapsedSeconds, jobs: result.jobs?.map(value => ({ id: value.id, status: value.status, viewport: value.viewport, assertions: value.assertions })), error: result.error, directory }));
|