Files
Agent-Workbench/scripts/verify-vision-projects.mjs

128 lines
14 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { spawnSync } from 'node:child_process';
import { chromium } from 'playwright';
import { fixture, call, completion, until, assertToolPairs } from '../tests/helpers.mjs';
import { specification } from '../tests/fixtures/engineering-specification.mjs';
import { responsesEngineeringHttp } from '../tests/fixtures/responses-engineering-http.mjs';
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { requestImages } from '../src/shared/contracts/model-request.mjs';
import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
if (process.argv.includes('--help')) {
console.log('node scripts/verify-vision-projects.mjs — actual screenshot Task, owned Responses SSE fixture, reviewed Node/React/Python repair, retained failures, Edge and local Git. Requires existing AGENT_WORKBENCH_TEST_PYTHON. Fixed responses verify execution, not image reasoning.'); process.exit(0);
}
assert.equal(process.argv.length, 2);
const version = JSON.parse(fs.readFileSync('package.json')).version;
const directory = path.resolve('.qa', 'vision-projects-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
const python = process.env.AGENT_WORKBENCH_TEST_PYTHON; assert.ok(python && path.isAbsolute(python) && fs.existsSync(python));
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const npm = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
const cases = [], results = [], mark = value => { cases.push(value); console.log('PASS ' + value); };
const browser = await chromium.launch({ channel: 'msedge', headless: true });
const escape = text => text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
async function sample(kind) {
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t), saved = path.join(directory, kind); fs.mkdirSync(saved);
let runtime, result; const reviews = [];
try {
const baseline = kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
write(f, '.gitignore', 'node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n');
write(f, 'owner-staged.txt', 'Owner baseline\n'); write(f, 'owner-unstaged.txt', 'Owner baseline\n');
git(f, ['init', '--initial-branch=main']); git(f, ['config', 'core.autocrlf', 'false']); git(f, ['config', 'user.name', 'Workbench QA']); git(f, ['config', 'user.email', 'qa@workbench.invalid']);
git(f, ['add', '--', '.']); git(f, ['commit', '--no-gpg-sign', '-m', 'Seal screenshot engineering baseline']);
write(f, 'owner-staged.txt', 'Owner staged change\n'); git(f, ['add', '--', 'owner-staged.txt']);
write(f, 'owner-staged.txt', 'Owner working copy after staging\n'); write(f, 'owner-unstaged.txt', 'Owner unstaged change\n'); write(f, 'owner-untracked.txt', 'Owner untracked file\n');
const original = ownerBytes(f), spec = specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort();
const protectedFiles = Object.fromEntries(git(f, ['ls-files', '-z']).split('\0').filter(file => file && !writable.includes(file) && !file.startsWith('owner-')).map(file => [file, fs.readFileSync(path.join(f.projectRoot, file))]));
const verify = () => { for (const [file, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, file)), bytes, 'Protected ' + file); };
if (kind === 'react') {
const child = spawnSync(process.execPath, [npm, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: f.projectRoot, windowsHide: true, timeout: 120000, maxBuffer: 8 * 1024 * 1024 });
fs.writeFileSync(path.join(saved, 'npm-ci-log.txt'), child.stdout); fs.writeFileSync(path.join(saved, 'npm-ci-stderr-log.txt'), child.stderr);
if (child.error) throw child.error; assert.equal(child.status, 0); verify();
}
// This is an actual rendering of the sample's original source. The owned
// response algorithm is deliberately fixed, so it proves no visual reasoning.
const page = await browser.newPage({ viewport: { width: 1280, height: 900 } });
const source = Object.keys(spec.fixes).map(file => file + '\n' + (fs.existsSync(path.join(f.projectRoot, file)) ? fs.readFileSync(path.join(f.projectRoot, file), 'utf8') : '(not yet present)')).join('\n\n');
await page.setContent('<!doctype html><html lang="zh-CN"><meta charset="utf-8"><style>body{font:17px Consolas,monospace;padding:28px;color:#18292b;background:#f8faf9}h1{font:600 28px system-ui}pre{white-space:pre-wrap;line-height:1.6}</style><h1>工程原始源码 · ' + kind + '</h1><pre>' + escape(source) + '</pre></html>');
const screenshot = path.join(saved, 'original-source.png'); await page.screenshot({ path: screenshot }); await page.close();
let phase = 'new';
const model = await responsesEngineeringHttp(t, (route, input) => {
const images = requestImages({ input: input.input });
assert.equal(route, '/v1/chat/completions'); assert.equal(images.length, 1); assertToolPairs(input.messages);
assert.equal(hash(Buffer.from(images[0].split(',')[1], 'base64')), attachment.sentSha256);
const job = [...f.store.core.check_job.values()].filter(value => value.taskId === f.session.taskId).at(-1);
const plan = [...f.store.core.check_plan.values()].filter(value => value.taskId === f.session.taskId).at(-1);
let calls = [];
const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Owned engineering fixture; retain all original tests and declarations.' }));
if (f.session.changes.some(change => change.status === 'pending')) { /* Actual review happens outside the model. */ }
else if (phase === 'new' && spec.initial) { calls = propose(spec.initial); phase = 'initial'; }
else if (phase !== 'fixed' && job) { assert.equal(job.status, 'failed', job.summary); assert.ok(job.results.some(value => value.tests?.failed > 0)); calls = propose(spec.fixes); phase = 'fixed'; }
else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, spec.tests);
else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: spec.checks })];
else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })];
return completion(calls, calls.length ? '' : '原截图任务的检查已完成,仍须独立核对页面及Git。');
});
f.store.saveConfig({ mode: 'compatible', model: 'owned-vision-engineering', baseUrl: model.baseUrl, contextLimit: 128000, maxSteps: 32, maxTaskTokens: 400000 });
const provider = new Provider(), hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned screenshot Responses', model: 'owned-vision-engineering', baseUrl: model.baseUrl, protocol: 'responses', requiresKey: false, contextLimit: 128000, streaming: true, capabilityDeclarations: { vision: 'supported', streaming: 'supported' } });
hub.select({ type: 'connection', id: connection.id }); runtime = new Runtime(f.store, provider); runtime.checkRunner().pythonExecutable = python;
const attachment = runtime.inputAttachments().upload(f.session, { name: kind + '-original-source.png', mediaType: 'image/png', base64: fs.readFileSync(screenshot).toString('base64') });
runtime.start(f.session.id, '核对所附原始源码截图,修复工程功能,保留原始测试并实际检查、构建与审查交付。', false, { attachmentIds: [attachment.id] });
const taskId = f.session.taskId;
for (let turn = 0; turn < 5; turn++) {
await until(() => {
for (const approval of f.session.approvals.filter(value => value.status === 'pending')) {
assert.equal(f.store.core.tool.get(approval.invocationId).tool, 'run_checks'); reviews.push({ kind: 'check-approval', id: approval.id, taskId }); runtime.decide(f.session.id, approval.id, true);
}
return !runtime.isActive(f.session.id);
}, 180000);
const ids = f.session.changes.filter(value => value.status === 'pending').map(value => value.id);
if (!ids.length) break;
const set = await runtime.changeSetManager().prepare(f.session, ids); reviews.push({ kind: 'file-review', id: set.id, fingerprint: set.fingerprint });
await (await runtime.changeSetManager().start(f.session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary); verify();
runtime.continueReviewed(f.session.id);
}
assert.equal(f.session.taskId, taskId); assert.equal(f.session.status, 'completed', JSON.stringify(f.session.messages));
const checks = [...f.store.core.check_job.values()].filter(value => value.taskId === taskId), check = checks.at(-1);
assert.ok(checks.some(value => value.status === 'failed' && value.results.some(row => row.tests?.failed > 0))); assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify();
const plans = [...f.store.core.context_plan.values()].filter(value => value.taskId === taskId); assert.equal(plans.length, model.requests.length);
for (const plan of plans) { assert.equal(plan.imageCount, 1); verifyPlanEvidence(f.store, f.session, plan); }
assert.ok(hub.attempts.every(value => value.protocol === 'responses' && value.stream?.terminal && value.responseArtifactId)); assertToolPairs(f.session.providerMessages);
mark(kind + ': one screenshot in each original Responses SSE request, original failing tests retained and ' + spec.tests + ' functional tests plus build/compile pass after explicit reviews');
const browserJobs = [];
if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
const steps = [{ action: 'assert_count', selector: '.task-list article', count: 3 }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'pending' }, { action: 'assert_count', selector: '.task-list article', count: 2 },
{ action: 'fill', selector: 'input[aria-label="任务搜索"]', value: ' 编写 ' }, { action: 'assert_count', selector: '.task-list article', count: 1 }, { action: 'fill', selector: 'input[aria-label="任务搜索"]', value: '' }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'all' },
{ action: 'click', selector: '.task-list article[data-task="api"] button' }, { action: 'assert_text', selector: '.task-list article[data-task="api"] .tag', text: '已完成' }, { action: 'assert_text', selector: '.task-list article[data-task="tests"] .tag', text: '待完成' },
{ action: 'assert_text', selector: '.summary > div:nth-child(2) strong', text: '2' }, { action: 'assert_text', selector: '.summary > div:nth-child(3) strong', text: viewport.width > 600 ? '5小时' : '5\n小时' }];
const plan = await runtime.browserManager().plan(f.session, { viewport, steps }), job = await runtime.browserManager().start(f.session, plan.id).promise;
assert.equal(job.status, 'passed', job.summary); assert.equal(job.cleanup.exited, true); browserJobs.push(job);
for (const image of job.screenshots) fs.writeFileSync(path.join(saved, 'page-' + viewport.width + '-' + image.artifactId + '.png'), f.store.artifactStore.readBytes(f.store.core.artifact.get(image.artifactId)));
}
if (kind === 'react') mark('react: actual Edge desktop/390px filters, trimmed search, task toggle and totals pass');
const deliveryPlan = await runtime.gitDelivery().plan(f.session, { message: 'Reviewed screenshot ' + kind + ' engineering delivery' }), delivery = await (await runtime.gitDelivery().start(f.session, deliveryPlan.id)).promise;
assert.equal(delivery.status, 'completed', delivery.summary); assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable);
assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n'); verify();
mark(kind + ': local Git contains only reviewed task files; original staged/unstaged/untracked content remains');
result = { success: true, version, kind, taskId, attachment: f.store.core.input_attachment.get(f.store.core.task.get(taskId).attachmentIds[0]), check, checks, reviews, browserJobs, commit: delivery.commit,
requests: model.requests.map(value => ({ sha256: hash(JSON.stringify(value)), imageCount: requestImages(value).length })), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
return result;
} catch (error) { result = { success: false, version, kind, error: error.stack, reviews, messages: f.session.messages, checks: [...f.store.core.check_job.values()] }; throw error; }
finally {
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
fs.cpSync(f.root, path.join(saved, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
fs.writeFileSync(path.join(saved, 'result.json'), JSON.stringify(result, null, 2));
for (const dispose of disposers.reverse()) await dispose();
}
}
let report;
try { for (const kind of ['node', 'react', 'python']) results.push(await sample(kind)); report = { success: true, version, directory, cases, results, realModelRequests: 0, scope: 'Actual screenshot Task/Runtime, owned fixed Responses SSE algorithm, original approvals/failures/checks/build/Edge/Git. No claim of real model visual reasoning or external Harness image support.' }; }
catch (error) { report = { success: false, version, directory, cases, results, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
finally { await browser.close(); fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/vision-engineering-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, error: report.error ?? null })); }