From 01c427e749c33bcbd02df5138da5e8a023b73eaa Mon Sep 17 00:00:00 2001 From: Agent Workbench Dev Date: Sun, 1 Nov 2026 00:52:31 +0800 Subject: [PATCH] test(vision): accept screenshot engineering and wire 1.18 release validation --- docs/QA-1.18.md | 6 + package-lock.json | 4 +- package.json | 4 +- progress.md | 2 + scripts/qa-evidence.mjs | 1 + scripts/upgrade-running.mjs | 2 +- scripts/verify-all-ui.mjs | 25 ++-- scripts/verify-change-sets-ui.mjs | 2 +- scripts/verify-context-ui.mjs | 2 +- scripts/verify-controlled-projects.mjs | 25 +--- scripts/verify-controlled-release.mjs | 2 +- scripts/verify-release.mjs | 14 +- scripts/verify-source-package.mjs | 43 +++++-- scripts/verify-vision-projects.mjs | 127 +++++++++++++++++++ scripts/verify-vision-ui.mjs | 2 +- scripts/verify-workers-ui.mjs | 2 +- src/server/diagnostics.mjs | 2 +- tests/fixtures/engineering-specification.mjs | 28 ++++ 18 files changed, 229 insertions(+), 64 deletions(-) create mode 100644 scripts/verify-vision-projects.mjs create mode 100644 tests/fixtures/engineering-specification.mjs diff --git a/docs/QA-1.18.md b/docs/QA-1.18.md index 70eb3206..1c566773 100644 --- a/docs/QA-1.18.md +++ b/docs/QA-1.18.md @@ -25,3 +25,9 @@ Chat image_url/Responses input_image实际HTTP每请求恰有一次图像,原p 实际Edge桌面1440×1000、390×844及真实自有TLS网关的5组流程通过;首次UI失败源于QA修改全局配置会切回legacy、以及固定响应夹具误用旧Task工具结果,已按实际选路与最后用户消息修正。原失败与截图保留。回看发现预览曾显示被排除的池首成员,改为显示符合当前任务能力的成员,运行时间以分钟显示,窄屏移除/选择触控区域补强;修正后的结果单独记录。 构建通过。回看修正后第三轮5组完整UI再次通过,零pageerror;原失败未覆盖。相关兼容62/64原轮其余均通过,剩余2项为上述预算封存status问题,修正后5/5;完整整轮尚待F。新手机截图流程的Edge/TLS结果不当成iPhone实机,真实模型视觉工程与Purge收益继续按用户选择另行待验。A–E原件与字节身份见vision-inputs-ae-evidence/manifest.json。 + +## F:工程验收与发行接入(进行中) + +版本与独立发行验收入口已接入1.18。新截图Task工程7节点通过:Node8/8、React9/9、Python10/10及实际构建/编译,React实际Edge两视口各12步通过;原失败、审批、完整图像请求SHA与原图/副本保持,Git只提交所选任务文件并保留原暂存/未暂存/未跟踪内容。固定Responses SSE夹具证明执行与协议,不证明视觉推理,也不假定外部Harness支持图像。工程原件见.qa/vision-engineering-location.json及独立结果。 + +F首轮QA将canonical messages中的字符串图像当成原wire图像,断言触发自有服务断开;改为核对原input后通过。首次构建误用工程内不存在的npm路径,改为固定Node发行自带npm后通过。Git证据原CRLF导致diff空白检查拒绝,添加仅原日志binary属性,保留40原件字节核对后提交96a7ed1。所有原失败保留。完整历史回归和独立发行继续,未提前登记为交付。 diff --git a/package-lock.json b/package-lock.json index 075a4d25..638ae66b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-agent-workbench", - "version": "1.17.0", + "version": "1.18.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-agent-workbench", - "version": "1.17.0", + "version": "1.18.0", "dependencies": { "@babel/parser": "7.29.11", "@peculiar/x509": "3.1.0", diff --git a/package.json b/package.json index a0100c00..22b86be5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-agent-workbench", - "version": "1.17.0", + "version": "1.18.0", "private": true, "type": "module", "description": "A local coding workbench with explicit tool execution, reviewable changes and recoverable sessions.", @@ -39,6 +39,8 @@ "verify:desktop-lifecycle": "node scripts/verify-controlled-ui.mjs --desktop", "verify:desktop-install": "node scripts/verify-desktop-install.mjs", "verify:companion": "node scripts/verify-companion-ui.mjs", + "verify:vision": "node scripts/verify-vision-ui.mjs", + "verify:vision-projects": "node scripts/verify-vision-projects.mjs", "verify:model-protocols": "node scripts/verify-model-protocols-ui.mjs" }, "dependencies": { diff --git a/progress.md b/progress.md index 5a405b2c..648e48f6 100644 --- a/progress.md +++ b/progress.md @@ -851,3 +851,5 @@ F冻结前:Responses三工程13节点和父8/8/9/9/10/10、25类191组历史UI 初次构建npm子脚本拾取系统Node16失败,已使用进程局部PATH指向固定Node24.22.1;第二轮构建通过。全局Node和正式PID未更改。后续预算/手机/UI专项与发行仍待验。 1.18 A–E回顾完成:附件12/12、预算5/5及最新联合17/17通过,桌面/390px/自有TLS新UI第三轮5组通过。原core/预算/UI失败及最终原件封存docs/vision-inputs-ae-evidence,原持久归属/预算status/QA选路/旧Task夹具原因分别记录。实际回看补正被排除池首模型的预览文案、分钟单位与窄屏触控区域;全部构建通过。正式只读确认1.17/PID36552/17项目,真实模型0请求。A–E本地Git保存后继续F工程/完整回归及发行,不将待验项称交付。 + +2026-11-01 1.18 F:96a7ed1保存A–E及40份原证据;新截图主Runtime工程7节点通过,Node8/8、React9/9、Python10/10,React两视口12步/真实Git保护通过。共享既有工程规范,版本/QA入口接入1.18;原轮canonical/wire图像断言与npm路径QA失败保留,产品权限未放宽。完整26类UI正在独立端口运行,后续冻结新不可变发行。 diff --git a/scripts/qa-evidence.mjs b/scripts/qa-evidence.mjs index b88acfe0..ec97d72c 100644 --- a/scripts/qa-evidence.mjs +++ b/scripts/qa-evidence.mjs @@ -10,6 +10,7 @@ const prefixes = Object.freeze({ '1.15.0': 'desktop-host', '1.16.0': 'mobile-companion', '1.17.0': 'model-protocols', + '1.18.0': 'vision-inputs', }); export function releaseEvidencePrefix(version) { const prefix = prefixes[version]; diff --git a/scripts/upgrade-running.mjs b/scripts/upgrade-running.mjs index 698007a2..3b154f14 100644 --- a/scripts/upgrade-running.mjs +++ b/scripts/upgrade-running.mjs @@ -64,7 +64,7 @@ export async function upgradeRunning({ url, pid, currentRoot, newRoot, handlerRo if (!fs.existsSync(originalScript) || !fs.existsSync(upgradeScript) || !fs.existsSync(path.join(next, 'dist-client', 'index.html'))) throw new Error('Upgrade bundles are incomplete.'); if (!Number.isInteger(pid) || pid < 1 || pid === process.pid) throw new Error('Invalid target process.'); const before = await readUpgradeJson(origin + '/api/bootstrap', 5000, 'Read original Workbench'); - if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.'); + if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.'); if (before.sessions.some(session => ['running', 'awaiting_approval', 'stopping'].includes(session.status))) throw new Error('Wait for existing tasks and approvals before upgrading.'); let protocol, socket, enabledHere = false, paused = false; const breakpoints = []; diff --git a/scripts/verify-all-ui.mjs b/scripts/verify-all-ui.mjs index 48189409..c583c52f 100644 --- a/scripts/verify-all-ui.mjs +++ b/scripts/verify-all-ui.mjs @@ -9,28 +9,29 @@ import { randomUUID } from 'node:crypto'; if (process.argv.slice(2).some(value => value !== '--help')) throw new Error('Unsupported argument; use --help.'); if (process.argv.includes('--help')) { console.log('node scripts/verify-all-ui.mjs - run serial desktop/390px checks in independent local hosts; use explicit QA Python/Harness/Purge source paths for 1.15.'); process.exit(0); } const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version; -assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version), 'Review evidence paths when introducing the next release.'); +assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version), 'Review evidence paths when introducing the next release.'); const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = []; const qaEnvironment = Object.fromEntries(['AGENT_WORKBENCH_TEST_PYTHON', 'AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_178_ROOT', 'AGENT_WORKBENCH_PURGE_181_ROOT'] .filter(name => process.env[name]).map(name => [name, process.env[name]])); for (const [name, value] of Object.entries(qaEnvironment)) assert.ok(path.isAbsolute(value) && fs.existsSync(value), 'Use an existing absolute QA runtime/source path: ' + name); const cases = [ - ...(version === '1.17.0' ? [['verify-model-protocols-ui.mjs', null]] : []), - ...(['1.16.0', '1.17.0'].includes(version) ? [['verify-companion-ui.mjs', null]] : []), - ...(['1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-controlled-ui.mjs', null], ['verify-purge-ui.mjs', null]] : []), - ...(['1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-workers-ui.mjs', null]] : []), - ...(['1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []), - ...(['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []), + ...(version === '1.18.0' ? [['verify-vision-ui.mjs', null]] : []), + ...(['1.17.0', '1.18.0'].includes(version) ? [['verify-model-protocols-ui.mjs', null]] : []), + ...(['1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-companion-ui.mjs', null]] : []), + ...(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-controlled-ui.mjs', null], ['verify-purge-ui.mjs', null]] : []), + ...(['1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-workers-ui.mjs', null]] : []), + ...(['1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []), + ...(['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []), ['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'], ['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'], ['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`], ['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`], ['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`], - ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []), - ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []), - ...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []), - ...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []), - ...(['1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(version) ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []), + ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []), + ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []), + ...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []), + ...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []), + ...(['1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(version) ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []), ]; fs.mkdirSync('.qa', { recursive: true }); const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']); diff --git a/scripts/verify-change-sets-ui.mjs b/scripts/verify-change-sets-ui.mjs index 2b4d09a1..a23d1374 100644 --- a/scripts/verify-change-sets-ui.mjs +++ b/scripts/verify-change-sets-ui.mjs @@ -34,7 +34,7 @@ child.stdout.on('data', value => { output += value; for (const line of output.sp child.stderr.on('data', value => { errors += value; }); const wait = async predicate => { const deadline = Date.now() + 15000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('UI fixture timed out: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } }; await wait(() => url); const bootstrap = await (await fetch(url + '/api/bootstrap')).json(); -assert.ok(['1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); +assert.ok(['1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); const proofPrefix = releaseEvidencePrefix(bootstrap.version) + '-batch'; const screenshotPrefix = bootstrap.version === '1.4.0' ? '1.4-batch' : proofPrefix; const api = async (route, input) => { const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, diff --git a/scripts/verify-context-ui.mjs b/scripts/verify-context-ui.mjs index 5086d69e..f72c1a03 100644 --- a/scripts/verify-context-ui.mjs +++ b/scripts/verify-context-ui.mjs @@ -77,7 +77,7 @@ child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); }); const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } }; await until(() => url || child.exitCode !== null); assert.ok(url, errors); const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json(); -assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); +assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; }; const proofPrefix = bootstrap.version === '1.6.0' ? 'workspaces-context' : releaseEvidencePrefix(bootstrap.version); const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); diff --git a/scripts/verify-controlled-projects.mjs b/scripts/verify-controlled-projects.mjs index 0eaa61bc..a6581e34 100644 --- a/scripts/verify-controlled-projects.mjs +++ b/scripts/verify-controlled-projects.mjs @@ -13,6 +13,7 @@ import { resolvePurgeSource, assertPurgeModelInput, nativeSystemTexts, PURGE_DEF import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs'; import { ownerBytes } from '../tests/fixtures/workers-project.mjs'; import { desktopProjectQa } from './desktop-project-qa.mjs'; +import { specification } from '../tests/fixtures/engineering-specification.mjs'; import { responsesEngineeringHttp } from '../tests/fixtures/responses-engineering-http.mjs'; const args = process.argv.slice(2); @@ -46,30 +47,6 @@ function inputs() { return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) })); } const frozen = inputs(), mark = message => { cases.push(message); console.log('PASS ' + message); }; -function specification(kind, root) { - if (kind === 'node') return { tests: 8, checks: ['test', 'build'], initial: null, fixes: { - 'src/add.mjs': 'export function add(left, right) {\n if (!Number.isFinite(left) || !Number.isFinite(right)) throw new TypeError("finite numeric inputs required");\n return left + right;\n}\n', - 'src/range.mjs': 'export function inclusiveRange(start, end) {\n if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || end < start || end - start >= 10000) throw new RangeError("bounded ascending integer range required");\n return Array.from({ length: end - start + 1 }, (_, index) => start + index);\n}\n', - } }; - if (kind === 'react') { - const app = fs.readFileSync(path.join(root, 'src/App.tsx'), 'utf8'); assert.ok(app.includes("toggleTask(current, 'missing')")); - return { tests: 9, checks: ['test', 'typecheck', 'build'], initial: null, fixes: { - 'src/query.ts': "import type { Task } from './types.ts';\nexport function selectTasks(tasks: Task[], filter: string, query: string): Task[] {\n const text = query.trim().toLocaleLowerCase();\n return tasks.filter(task => (filter === 'done' ? task.done : filter === 'pending' ? !task.done : true) && task.title.toLocaleLowerCase().includes(text));\n}\n", - 'src/summary.ts': "import type { Task } from './types.ts';\nexport function summarizeTasks(tasks: Task[]) {\n return { total: tasks.length, completed: tasks.filter(task => task.done).length, pendingHours: tasks.filter(task => !task.done).reduce((sum, task) => sum + task.hours, 0) };\n}\n", - 'src/transition.ts': "import type { Task } from './types.ts';\nexport function toggleTask(tasks: Task[], id: string): Task[] {\n return tasks.map(task => task.id === id ? { ...task, done: !task.done } : task);\n}\n", - 'src/App.tsx': app.replace("toggleTask(current, 'missing')", 'toggleTask(current, task.id)'), - } }; - } - const test = file => fs.readFileSync(path.join('benchmarks/workers-delivery-v1/python-tests', file), 'utf8'); - return { tests: 10, checks: ['python:test', 'python:compile'], initial: { - 'money.py': 'def to_cents(value):\n return int(float(value) * 100)\n', - 'schedule.py': 'def normalize_schedule(values):\n return list(values)\n', - 'tests/test_money.py': test('test_money.py'), 'tests/test_schedule.py': test('test_schedule.py'), - }, fixes: { - 'money.py': 'from decimal import Decimal, InvalidOperation, ROUND_HALF_UP\n\ndef to_cents(value):\n try:\n amount = Decimal(str(value))\n if not amount.is_finite():\n raise ValueError("finite amount required")\n return int((amount * 100).quantize(Decimal("1"), rounding=ROUND_HALF_UP))\n except (InvalidOperation, OverflowError, TypeError) as error:\n raise ValueError("valid finite amount required") from error\n', - 'schedule.py': 'import re\nfrom datetime import date\n\ndef normalize_schedule(values):\n dates = []\n for value in values:\n if not isinstance(value, str) or not re.fullmatch(r"\\d{4}-\\d{2}-\\d{2}", value):\n raise ValueError("ISO calendar date required")\n dates.append(date.fromisoformat(value))\n return [value.isoformat() for value in sorted(set(dates))]\n', - } }; -} async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) { const startedAt = Date.now(), label = sampleName; const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName); diff --git a/scripts/verify-controlled-release.mjs b/scripts/verify-controlled-release.mjs index a3f3aeec..b5c918a4 100644 --- a/scripts/verify-controlled-release.mjs +++ b/scripts/verify-controlled-release.mjs @@ -22,7 +22,7 @@ export async function verifyControlledApi({ api, url, token, directory, appRoot, const origin = new URL(url); assert.equal(origin.hostname, '127.0.0.1'); assert.notEqual(origin.port, '4318'); const expectedVersion = JSON.parse(fs.readFileSync(path.join(appRoot, fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json'), 'utf8')).version; - assert.ok(['1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(expectedVersion)); + assert.ok(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(expectedVersion)); const bootstrap = await api('/bootstrap'); assert.equal(bootstrap.version, expectedVersion); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); const root = path.join(directory, 'compiled-controlled-node'), hooks = path.join(directory, 'disabled-hooks'); diff --git a/scripts/verify-release.mjs b/scripts/verify-release.mjs index 145c4cc3..90918330 100644 --- a/scripts/verify-release.mjs +++ b/scripts/verify-release.mjs @@ -34,7 +34,7 @@ for (const name of Object.keys(env)) if (name.toLowerCase() === 'path') delete e const windowsRoot = process.env.SystemRoot || 'C:\\Windows'; env.PATH = [windowsRoot, path.join(windowsRoot, 'System32'), path.join(windowsRoot, 'System32', 'WindowsPowerShell', 'v1.0')].join(path.delimiter); let externalGit = null; -if (['1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(manifest.version)) { +if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(manifest.version)) { // The controlled project requires external Git. Preserve its observed binary // directory while keeping system Node/npm unavailable to the packaged host. const binary = fs.realpathSync.native(execFileSync('where.exe', ['git.exe'], { env: gitEnvironment(), encoding: 'utf8', windowsHide: true, timeout: 10000 }).trim().split(/\r?\n/)[0]); @@ -88,7 +88,7 @@ try { console.log((await launch()).trim()); assert.equal(JSON.parse(fs.readFileSync(path.join(data, 'service-info.json'), 'utf8')).pid, firstInfo.pid); const example = bootstrap.projects.find(project => project.id === bootstrap.exampleProjectId); - if (['1.15.0', '1.16.0', '1.17.0'].includes(manifest.version)) { + if (['1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(manifest.version)) { assert.equal(path.resolve(example.root).toLowerCase(), path.join(data, 'projects', 'starter-project').toLowerCase()); const compareTemplate = relative => { for (const entry of fs.readdirSync(path.join(appRoot, 'examples/starter-project', relative), { withFileTypes: true })) { const next = path.join(relative, entry.name); if (entry.isDirectory()) compareTemplate(next); @@ -114,7 +114,7 @@ try { const checkLog = await api('/artifacts/' + checked.results[0].logArtifactId + '?session=' + session.id); assert.match(checkLog.content, /tests 4/); assert.ok(fs.existsSync(path.join(appRoot, 'runtime/node_modules/npm/bin/npm-cli.js'))); const actualRea = []; - if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(manifest.version)) { + if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(manifest.version)) { const installation = await api('/research/availability'); assert.equal(installation.status, 'available'); assert.equal(installation.version, '6.8.0'); assert.equal(fs.realpathSync.native(installation.packageRoot), fs.realpathSync.native(path.join(appRoot, 'runtime/node_modules/rea-agents'))); assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe'))); @@ -146,7 +146,7 @@ try { console.log('PASS actual contained REA: ASAR, PE imports and PE resources; zero target execution/model requests'); } let actualEngine = null; - if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(manifest.version) && process.argv.includes('--native-engines')) { + if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(manifest.version) && process.argv.includes('--native-engines')) { const installation = await api('/engines/deepseek/availability'); assert.equal(installation.status, 'available', installation.message); assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe'))); const engineRoot = path.join(data, 'engine-project'); fs.mkdirSync(engineRoot); fs.writeFileSync(path.join(engineRoot, 'README.md'), '# Owned portable engine connection\n'); @@ -166,8 +166,8 @@ try { assert.equal((await api('/research')).length, 4); const page = await fetch(url); assert.equal(page.status, 200); assert.match(await page.text(), /Agent Workbench/); - const actualBrowser = ['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; - const actualControlled = ['1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(manifest.version) ? await verifyControlledApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; + const actualBrowser = ['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; + const actualControlled = ['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(manifest.version) ? await verifyControlledApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; console.log((await stop()).trim()); started = false; await until(async () => { try { await fetch(url + '/api/health'); return false; } catch { return true; } }); for (const file of manifest.files) { @@ -176,7 +176,7 @@ try { } const result = { success: true, date: new Date().toISOString(), appRoot, version: manifest.version, nodeVersion: manifest.nodeVersion, verifiedFiles: manifest.files.length, url, reusedPid: firstInfo.pid, checks: { jobId, status: checked.status, tests: checked.results[0].tests }, actualRea, actualEngine, actualBrowser, actualControlled, externalGit, - steps: ['manifest hashes', 'bundled Node launch', 'same-service reuse', 'packaged example and static page', ...(['1.15.0', '1.16.0', '1.17.0'].includes(manifest.version) ? ['starter project copied to owned user data; original template bytes matched'] : []), 'explicit command consent', 'actual bundled Node version', 'actual bundled npm check without system Node/npm', 'check log artifact', ...(actualRea.length ? ['contained REA installation and Node identity', 'actual ASAR/imports/resources', 'original Bundle and target hash binding', 'verified reuse without producer replay'] : []), 'report availability', 'graceful stop', 'every original packaged file remains unchanged'], realModel: false }; + steps: ['manifest hashes', 'bundled Node launch', 'same-service reuse', 'packaged example and static page', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(manifest.version) ? ['starter project copied to owned user data; original template bytes matched'] : []), 'explicit command consent', 'actual bundled Node version', 'actual bundled npm check without system Node/npm', 'check log artifact', ...(actualRea.length ? ['contained REA installation and Node identity', 'actual ASAR/imports/resources', 'original Bundle and target hash binding', 'verified reuse without producer replay'] : []), 'report availability', 'graceful stop', 'every original packaged file remains unchanged'], realModel: false }; fs.writeFileSync('.qa/release-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2)); } finally { if (started) { try { await stop(); } catch {} } } diff --git a/scripts/verify-source-package.mjs b/scripts/verify-source-package.mjs index b9e51c6f..06cc34d4 100644 --- a/scripts/verify-source-package.mjs +++ b/scripts/verify-source-package.mjs @@ -19,7 +19,7 @@ assert.ok(fs.existsSync(npmCli), 'Use Node with its bundled npm.'); const files = (root, relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)) .flatMap(entry => entry.isDirectory() ? files(root, path.join(relative, entry.name)) : [path.join(relative, entry.name)]); const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex'); -const buildFolders = ['dist-client', 'dist-server', ...(['1.15.0', '1.16.0', '1.17.0'].includes(evidence.version) ? ['dist-desktop'] : [])]; +const buildFolders = ['dist-client', 'dist-server', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version) ? ['dist-desktop'] : [])]; const expected = Object.fromEntries(buildFolders.flatMap(folder => files(path.join(workspace, folder)).map(relative => [path.join(folder, relative), digest(path.join(workspace, folder, relative))]))); const allowedEnv = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON|AGENT_WORKBENCH_DEEPSEEK_ROOT|AGENT_WORKBENCH_PURGE_ROOT)$/i; const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => allowedEnv.test(key))); @@ -30,7 +30,7 @@ const directory = path.join(sourceRoot, '.qa', 'package-validation-' + randomUUI fs.mkdirSync(directory, { recursive: true }); const inputFolders = ['src', 'tests', 'scripts', ...['benchmarks', 'public', 'examples', 'research'].filter(folder => fs.existsSync(path.join(sourceRoot, folder)))]; const inputNames = [...inputFolders.flatMap(folder => files(path.join(sourceRoot, folder)).map(relative => path.join(folder, relative))), - 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', ...(['1.16.0', '1.17.0'].includes(evidence.version) ? ['mobile.html'] : []), ...['.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].filter(name => fs.existsSync(path.join(sourceRoot, name)))].sort(); + 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', ...(['1.16.0', '1.17.0', '1.18.0'].includes(evidence.version) ? ['mobile.html'] : []), ...['.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].filter(name => fs.existsSync(path.join(sourceRoot, name)))].sort(); const snapshot = () => inputNames.map(relative => ({ path: relative.replaceAll('\\', '/'), bytes: fs.statSync(path.join(sourceRoot, relative)).size, sha256: digest(path.join(sourceRoot, relative)) })); const inputs = snapshot(); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2)); let stage = 'environment preflight', python = null; @@ -83,7 +83,7 @@ try { assert.deepEqual(actualFiles.slice().sort(), Object.keys(expected).sort()); for (const [relative, sha256] of Object.entries(expected)) assert.equal(digest(path.join(sourceRoot, relative)), sha256, relative); let actualRea = null; - if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) { + if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) { stage = 'contained REA'; const reaOutput = await run(['scripts/verify-rea.mjs', '--app-root', sourceRoot, '--contained'], 'rea.log'); actualRea = JSON.parse(reaOutput.trim().split('\n').at(-1)); @@ -91,7 +91,7 @@ try { assert.ok(actualRea.results.every(item => item.rootExited === true)); } let actualEngine = null; - if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version) && process.argv.includes('--native-engines')) { + if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version) && process.argv.includes('--native-engines')) { stage = 'native DeepSeek SDK'; const output = await run(['scripts/verify-engines.mjs'], 'engines.log'); actualEngine = JSON.parse(output.trim().split('\n').at(-1)); @@ -99,7 +99,7 @@ try { assert.equal(actualEngine.modelRequests, 0); assert.equal(actualEngine.rootExited, true); assert.equal(actualEngine.nativeMethods.length, 6); } let actualBrowser = null; - if (['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) { + if (['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) { stage = 'contained browser runtime'; const output = await run(['scripts/verify-browser.mjs', '--app-root', sourceRoot, '--contained'], 'browser.log'); actualBrowser = JSON.parse(output.trim().split('\n').at(-1)); @@ -109,7 +109,7 @@ try { assert.equal(actualBrowser.modelRequests, 0); } let actualControlled = null; - if (['1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) { + if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) { stage = 'compiled controlled-engine delivery'; const output = await run(['scripts/verify-controlled-release.mjs', '--app-root', sourceRoot], 'controlled-engine.log'); actualControlled = JSON.parse(output.trim().split('\n').at(-1)); @@ -117,7 +117,7 @@ try { assert.equal(actualControlled.realModelRequests, 0); assert.equal(actualControlled.cases, 4); assert.ok(actualControlled.fixtureModelRequests > 0); } let actualCompanion = null; - if (['1.16.0', '1.17.0'].includes(evidence.version)) { + if (['1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) { stage = 'contained companion gateway and UI'; const output = await run(['scripts/verify-companion-ui.mjs', '--app-root', sourceRoot], 'companion-ui.log'); const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); @@ -128,7 +128,7 @@ try { assert.equal(actualCompanion.passed.length, 7); assert.deepEqual(actualCompanion.errors, []); assert.equal(actualCompanion.realModelRequests, 0); } let actualProtocols = null; - if (evidence.version === '1.17.0') { + if (['1.17.0', '1.18.0'].includes(evidence.version)) { stage = 'contained model protocols and mobile streams'; const output = await run(['scripts/verify-model-protocols-ui.mjs', '--app-root', sourceRoot], 'model-protocols-ui.log'); const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); @@ -138,12 +138,23 @@ try { assert.equal(fs.realpathSync.native(actualProtocols.appRoot), sourceRoot); assert.equal(actualProtocols.containedRuntime, true); assert.equal(actualProtocols.passed.length, 5); assert.deepEqual(actualProtocols.pageErrors, []); assert.deepEqual(actualProtocols.requestFailures, []); } + let actualVision = null; + if (evidence.version === '1.18.0') { + stage = 'contained screenshot inputs and exact Task budget resume'; + const output = await run(['scripts/verify-vision-ui.mjs', '--app-root', sourceRoot], 'vision-ui.log'); + const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); + assert.ok(location && path.resolve(location).startsWith(path.join(sourceRoot, '.qa') + path.sep)); + actualVision = JSON.parse(fs.readFileSync(path.join(location, 'result.json'), 'utf8')); + assert.equal(actualVision.success, true); assert.equal(actualVision.version, evidence.version); + assert.equal(fs.realpathSync.native(actualVision.appRoot), sourceRoot); assert.equal(actualVision.containedRuntime, true); + assert.equal(actualVision.passed.length, 5); assert.deepEqual(actualVision.pageErrors, []); assert.equal(actualVision.realModelRequests, 0); + } stage = 'tests'; const output = await run(['--test', '--test-concurrency=2', '--test-reporter=tap', ...files(path.join(sourceRoot, 'tests')).filter(file => !file.includes(path.sep) && file.endsWith('.test.mjs')).map(file => path.join('tests', file))], 'tests.tap'); const stats = Object.fromEntries([...output.matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])])); assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0); let actualDesktop = null; - if (['1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) { + if (['1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) { stage = 'actual contained Electron window'; await run(['scripts/verify-desktop-window.mjs'], 'desktop-window.log'); const window = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/desktop-window-latest.json'), 'utf8')); assert.equal(window.success, true); assert.equal(path.resolve(window.appRoot), sourceRoot); assert.equal(window.results.length, 7); @@ -157,7 +168,7 @@ try { actualDesktop = { window, ui, projects, scope: 'Independently rebuilt source and its shipped Electron binary; original window/engine/engineering assertions. Own credentials/HTTP fixtures; Windows native picker and complete host crash/stop have separate actual evidence.' }; } let actualProtocolEngineering = null; - if (evidence.version === '1.17.0') { + if (['1.17.0', '1.18.0'].includes(evidence.version)) { stage = 'Responses SSE actual Node React Python engineering'; await run(['scripts/verify-controlled-projects.mjs', '--responses'], 'responses-engineering.log'); const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/responses-engineering-location.json'), 'utf8')); @@ -166,9 +177,19 @@ try { assert.equal(actualProtocolEngineering.protocol, 'responses-stream'); assert.equal(actualProtocolEngineering.realModelRequests, 0); assert.deepEqual(actualProtocolEngineering.results.map(value => value.parentCheck.results[0].tests.passed), [8, 9, 10]); } + let actualVisionEngineering = null; + if (evidence.version === '1.18.0') { + stage = 'screenshot Task Node React Python engineering'; + await run(['scripts/verify-vision-projects.mjs'], 'vision-engineering.log'); + const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/vision-engineering-location.json'), 'utf8')); + actualVisionEngineering = JSON.parse(fs.readFileSync(location.result, 'utf8')); + assert.equal(actualVisionEngineering.success, true); assert.equal(actualVisionEngineering.version, evidence.version); + assert.equal(actualVisionEngineering.realModelRequests, 0); + assert.deepEqual(actualVisionEngineering.results.map(value => value.check.results[0].tests.passed), [8, 9, 10]); + } assert.deepEqual(snapshot(), inputs, 'Source package inputs changed during validation.'); const result = { success: true, version: evidence.version, testedAt: new Date().toISOString(), sourceRoot, directory, nodeVersion: process.version, python, offlineInstall: true, - inputs, inputFiles: inputs.length, logs, buildMatchedFiles: Object.keys(expected).length, buildFiles: Object.entries(expected).map(([path, sha256]) => ({ path: path.replaceAll('\\', '/'), sha256 })), tests: stats, actualRea, actualEngine, actualBrowser, actualControlled, actualDesktop, actualCompanion, actualProtocols, actualProtocolEngineering, credentialsHandling: 'Only build/runtime environment paths are inherited; no configured model credentials are used. The controlled-engine stage supplies a declared owned local response fixture.' }; + inputs, inputFiles: inputs.length, logs, buildMatchedFiles: Object.keys(expected).length, buildFiles: Object.entries(expected).map(([path, sha256]) => ({ path: path.replaceAll('\\', '/'), sha256 })), tests: stats, actualRea, actualEngine, actualBrowser, actualControlled, actualDesktop, actualCompanion, actualProtocols, actualProtocolEngineering, actualVision, actualVisionEngineering, credentialsHandling: 'Only build/runtime environment paths are inherited; no configured model credentials are used. The controlled-engine stage supplies a declared owned local response fixture.' }; save(result); console.log(JSON.stringify({ success: true, version: result.version, sourceRoot, directory, tests: stats, nodeVersion: process.version, python, inputFiles: inputs.length, buildMatchedFiles: result.buildMatchedFiles, rea: !!actualRea?.success, engine: !!actualEngine?.success, browser: !!actualBrowser?.success, controlled: !!actualControlled?.success })); } catch (error) { save({ success: false, version: evidence.version, testedAt: new Date().toISOString(), stage, error: error.message, sourceRoot, directory, nodeVersion: process.version, python, inputs, logs }); diff --git a/scripts/verify-vision-projects.mjs b/scripts/verify-vision-projects.mjs new file mode 100644 index 00000000..f21846d8 --- /dev/null +++ b/scripts/verify-vision-projects.mjs @@ -0,0 +1,127 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { chromium } from 'playwright'; +import { fixture, call, completion, until, assertToolPairs } from '../tests/helpers.mjs'; +import { specification } from '../tests/fixtures/engineering-specification.mjs'; +import { responsesEngineeringHttp } from '../tests/fixtures/responses-engineering-http.mjs'; +import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs'; +import { ownerBytes } from '../tests/fixtures/workers-project.mjs'; +import { Runtime } from '../src/server/runtime.mjs'; +import { Provider } from '../src/server/provider.mjs'; +import { requestImages } from '../src/shared/contracts/model-request.mjs'; +import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs'; + +if (process.argv.includes('--help')) { + console.log('node scripts/verify-vision-projects.mjs — actual screenshot Task, owned Responses SSE fixture, reviewed Node/React/Python repair, retained failures, Edge and local Git. Requires existing AGENT_WORKBENCH_TEST_PYTHON. Fixed responses verify execution, not image reasoning.'); process.exit(0); +} +assert.equal(process.argv.length, 2); +const version = JSON.parse(fs.readFileSync('package.json')).version; +const directory = path.resolve('.qa', 'vision-projects-' + randomUUID()); fs.mkdirSync(directory, { recursive: true }); +const python = process.env.AGENT_WORKBENCH_TEST_PYTHON; assert.ok(python && path.isAbsolute(python) && fs.existsSync(python)); +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const npm = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js'); +const cases = [], results = [], mark = value => { cases.push(value); console.log('PASS ' + value); }; +const browser = await chromium.launch({ channel: 'msedge', headless: true }); +const escape = text => text.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>'); +async function sample(kind) { + const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t), saved = path.join(directory, kind); fs.mkdirSync(saved); + let runtime, result; const reviews = []; + try { + const baseline = kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind; + fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') }); + write(f, '.gitignore', 'node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n'); + write(f, 'owner-staged.txt', 'Owner baseline\n'); write(f, 'owner-unstaged.txt', 'Owner baseline\n'); + git(f, ['init', '--initial-branch=main']); git(f, ['config', 'core.autocrlf', 'false']); git(f, ['config', 'user.name', 'Workbench QA']); git(f, ['config', 'user.email', 'qa@workbench.invalid']); + git(f, ['add', '--', '.']); git(f, ['commit', '--no-gpg-sign', '-m', 'Seal screenshot engineering baseline']); + write(f, 'owner-staged.txt', 'Owner staged change\n'); git(f, ['add', '--', 'owner-staged.txt']); + write(f, 'owner-staged.txt', 'Owner working copy after staging\n'); write(f, 'owner-unstaged.txt', 'Owner unstaged change\n'); write(f, 'owner-untracked.txt', 'Owner untracked file\n'); + const original = ownerBytes(f), spec = specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort(); + const protectedFiles = Object.fromEntries(git(f, ['ls-files', '-z']).split('\0').filter(file => file && !writable.includes(file) && !file.startsWith('owner-')).map(file => [file, fs.readFileSync(path.join(f.projectRoot, file))])); + const verify = () => { for (const [file, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, file)), bytes, 'Protected ' + file); }; + if (kind === 'react') { + const child = spawnSync(process.execPath, [npm, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: f.projectRoot, windowsHide: true, timeout: 120000, maxBuffer: 8 * 1024 * 1024 }); + fs.writeFileSync(path.join(saved, 'npm-ci-log.txt'), child.stdout); fs.writeFileSync(path.join(saved, 'npm-ci-stderr-log.txt'), child.stderr); + if (child.error) throw child.error; assert.equal(child.status, 0); verify(); + } + // This is an actual rendering of the sample's original source. The owned + // response algorithm is deliberately fixed, so it proves no visual reasoning. + const page = await browser.newPage({ viewport: { width: 1280, height: 900 } }); + const source = Object.keys(spec.fixes).map(file => file + '\n' + (fs.existsSync(path.join(f.projectRoot, file)) ? fs.readFileSync(path.join(f.projectRoot, file), 'utf8') : '(not yet present)')).join('\n\n'); + await page.setContent('

工程原始源码 · ' + kind + '

' + escape(source) + '
'); + const screenshot = path.join(saved, 'original-source.png'); await page.screenshot({ path: screenshot }); await page.close(); + let phase = 'new'; + const model = await responsesEngineeringHttp(t, (route, input) => { + const images = requestImages({ input: input.input }); + assert.equal(route, '/v1/chat/completions'); assert.equal(images.length, 1); assertToolPairs(input.messages); + assert.equal(hash(Buffer.from(images[0].split(',')[1], 'base64')), attachment.sentSha256); + const job = [...f.store.core.check_job.values()].filter(value => value.taskId === f.session.taskId).at(-1); + const plan = [...f.store.core.check_plan.values()].filter(value => value.taskId === f.session.taskId).at(-1); + let calls = []; + const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Owned engineering fixture; retain all original tests and declarations.' })); + if (f.session.changes.some(change => change.status === 'pending')) { /* Actual review happens outside the model. */ } + else if (phase === 'new' && spec.initial) { calls = propose(spec.initial); phase = 'initial'; } + else if (phase !== 'fixed' && job) { assert.equal(job.status, 'failed', job.summary); assert.ok(job.results.some(value => value.tests?.failed > 0)); calls = propose(spec.fixes); phase = 'fixed'; } + else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, spec.tests); + else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: spec.checks })]; + else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })]; + return completion(calls, calls.length ? '' : '原截图任务的检查已完成,仍须独立核对页面及Git。'); + }); + f.store.saveConfig({ mode: 'compatible', model: 'owned-vision-engineering', baseUrl: model.baseUrl, contextLimit: 128000, maxSteps: 32, maxTaskTokens: 400000 }); + const provider = new Provider(), hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned screenshot Responses', model: 'owned-vision-engineering', baseUrl: model.baseUrl, protocol: 'responses', requiresKey: false, contextLimit: 128000, streaming: true, capabilityDeclarations: { vision: 'supported', streaming: 'supported' } }); + hub.select({ type: 'connection', id: connection.id }); runtime = new Runtime(f.store, provider); runtime.checkRunner().pythonExecutable = python; + const attachment = runtime.inputAttachments().upload(f.session, { name: kind + '-original-source.png', mediaType: 'image/png', base64: fs.readFileSync(screenshot).toString('base64') }); + runtime.start(f.session.id, '核对所附原始源码截图,修复工程功能,保留原始测试并实际检查、构建与审查交付。', false, { attachmentIds: [attachment.id] }); + const taskId = f.session.taskId; + for (let turn = 0; turn < 5; turn++) { + await until(() => { + for (const approval of f.session.approvals.filter(value => value.status === 'pending')) { + assert.equal(f.store.core.tool.get(approval.invocationId).tool, 'run_checks'); reviews.push({ kind: 'check-approval', id: approval.id, taskId }); runtime.decide(f.session.id, approval.id, true); + } + return !runtime.isActive(f.session.id); + }, 180000); + const ids = f.session.changes.filter(value => value.status === 'pending').map(value => value.id); + if (!ids.length) break; + const set = await runtime.changeSetManager().prepare(f.session, ids); reviews.push({ kind: 'file-review', id: set.id, fingerprint: set.fingerprint }); + await (await runtime.changeSetManager().start(f.session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary); verify(); + runtime.continueReviewed(f.session.id); + } + assert.equal(f.session.taskId, taskId); assert.equal(f.session.status, 'completed', JSON.stringify(f.session.messages)); + const checks = [...f.store.core.check_job.values()].filter(value => value.taskId === taskId), check = checks.at(-1); + assert.ok(checks.some(value => value.status === 'failed' && value.results.some(row => row.tests?.failed > 0))); assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify(); + const plans = [...f.store.core.context_plan.values()].filter(value => value.taskId === taskId); assert.equal(plans.length, model.requests.length); + for (const plan of plans) { assert.equal(plan.imageCount, 1); verifyPlanEvidence(f.store, f.session, plan); } + assert.ok(hub.attempts.every(value => value.protocol === 'responses' && value.stream?.terminal && value.responseArtifactId)); assertToolPairs(f.session.providerMessages); + mark(kind + ': one screenshot in each original Responses SSE request, original failing tests retained and ' + spec.tests + ' functional tests plus build/compile pass after explicit reviews'); + const browserJobs = []; + if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) { + const steps = [{ action: 'assert_count', selector: '.task-list article', count: 3 }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'pending' }, { action: 'assert_count', selector: '.task-list article', count: 2 }, + { action: 'fill', selector: 'input[aria-label="任务搜索"]', value: ' 编写 ' }, { action: 'assert_count', selector: '.task-list article', count: 1 }, { action: 'fill', selector: 'input[aria-label="任务搜索"]', value: '' }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'all' }, + { action: 'click', selector: '.task-list article[data-task="api"] button' }, { action: 'assert_text', selector: '.task-list article[data-task="api"] .tag', text: '已完成' }, { action: 'assert_text', selector: '.task-list article[data-task="tests"] .tag', text: '待完成' }, + { action: 'assert_text', selector: '.summary > div:nth-child(2) strong', text: '2' }, { action: 'assert_text', selector: '.summary > div:nth-child(3) strong', text: viewport.width > 600 ? '5小时' : '5\n小时' }]; + const plan = await runtime.browserManager().plan(f.session, { viewport, steps }), job = await runtime.browserManager().start(f.session, plan.id).promise; + assert.equal(job.status, 'passed', job.summary); assert.equal(job.cleanup.exited, true); browserJobs.push(job); + for (const image of job.screenshots) fs.writeFileSync(path.join(saved, 'page-' + viewport.width + '-' + image.artifactId + '.png'), f.store.artifactStore.readBytes(f.store.core.artifact.get(image.artifactId))); + } + if (kind === 'react') mark('react: actual Edge desktop/390px filters, trimmed search, task toggle and totals pass'); + const deliveryPlan = await runtime.gitDelivery().plan(f.session, { message: 'Reviewed screenshot ' + kind + ' engineering delivery' }), delivery = await (await runtime.gitDelivery().start(f.session, deliveryPlan.id)).promise; + assert.equal(delivery.status, 'completed', delivery.summary); assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable); + assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n'); verify(); + mark(kind + ': local Git contains only reviewed task files; original staged/unstaged/untracked content remains'); + result = { success: true, version, kind, taskId, attachment: f.store.core.input_attachment.get(f.store.core.task.get(taskId).attachmentIds[0]), check, checks, reviews, browserJobs, commit: delivery.commit, + requests: model.requests.map(value => ({ sha256: hash(JSON.stringify(value)), imageCount: requestImages(value).length })), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 }; + return result; + } catch (error) { result = { success: false, version, kind, error: error.stack, reviews, messages: f.session.messages, checks: [...f.store.core.check_job.values()] }; throw error; } + finally { + await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + fs.cpSync(f.root, path.join(saved, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') }); + fs.writeFileSync(path.join(saved, 'result.json'), JSON.stringify(result, null, 2)); + for (const dispose of disposers.reverse()) await dispose(); + } +} +let report; +try { for (const kind of ['node', 'react', 'python']) results.push(await sample(kind)); report = { success: true, version, directory, cases, results, realModelRequests: 0, scope: 'Actual screenshot Task/Runtime, owned fixed Responses SSE algorithm, original approvals/failures/checks/build/Edge/Git. No claim of real model visual reasoning or external Harness image support.' }; } +catch (error) { report = { success: false, version, directory, cases, results, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; } +finally { await browser.close(); fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/vision-engineering-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, error: report.error ?? null })); } diff --git a/scripts/verify-vision-ui.mjs b/scripts/verify-vision-ui.mjs index fa63aee8..1e953539 100644 --- a/scripts/verify-vision-ui.mjs +++ b/scripts/verify-vision-ui.mjs @@ -84,6 +84,6 @@ try{ await host.api('/config',{...(await host.api('/bootstrap')).config,contextLimit:16000,maxSteps:3}); const connection=(await host.api('/model-center')).connections.find(row=>row.id===visionId);await host.api('/model-center/connections',{...connection,streaming:true,capabilityDeclarations:{...connection.capabilityDeclarations,streaming:'supported'}});await host.api('/model-center/select',{type:'connection',id:visionId});mode='hold'; await page.reload();await page.getByLabel('选择工程截图',{exact:true}).setInputFiles(imageFile);await expect(page.locator('.input-draft')).toContainText('SHA-256 已核对');await page.getByLabel('任务内容',{exact:true}).fill('截图流式停止验收');await page.getByRole('button',{name:'发送任务',exact:true}).click();await expect(page.getByLabel('模型输出片段',{exact:true}).last()).toContainText('截图正在核对');const stopping=await get();await page.getByRole('button',{name:'停止',exact:true}).click();await until(async()=>(await get()).status==='paused'&&closed===1,15000);session=await get();assert.equal(session.events.filter(row=>row.kind==='tool'&&row.runId===stopping.runId).length,0);await shot(page,'desktop-image-stopped');mark('stopping screenshot SSE closes original HTTP and dispatches zero partial tools'); - assert.deepEqual(errors,[]);fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify({success:true,version:host.bootstrap.version,appRoot,passed,screenshots,requests:requests.map(value=>({route:value.route,sha256:sha(JSON.stringify(value.body)),imageCount:requestImages(value.body).length})),pageErrors:errors},null,2)); + assert.deepEqual(errors,[]);fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify({success:true,version:host.bootstrap.version,appRoot,containedRuntime:args.length>0,realModelRequests:0,passed,screenshots,requests:requests.map(value=>({route:value.route,sha256:sha(JSON.stringify(value.body)),imageCount:requestImages(value.body).length})),pageErrors:errors},null,2)); }catch(error){await shot(page,'failure-desktop').catch(()=>{});if(mobile)await shot(mobile,'failure-mobile').catch(()=>{});fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify({success:false,error:error.stack,passed,screenshots,pageErrors:errors},null,2));throw error; }finally{console.log('EVIDENCE '+directory);await browser.close();if(relay)await new Promise(resolve=>{relay.closeAllConnections();relay.close(resolve);});if(gateway)await host.api('/companion/close',{});for(const response of held)response.destroy();for(const socket of sockets)socket.destroy();await new Promise(resolve=>model.close(resolve));for(const dispose of disposers.reverse())await dispose();} diff --git a/scripts/verify-workers-ui.mjs b/scripts/verify-workers-ui.mjs index 6aca321c..cd8f16c0 100644 --- a/scripts/verify-workers-ui.mjs +++ b/scripts/verify-workers-ui.mjs @@ -8,7 +8,7 @@ import { workerHttp } from '../tests/fixtures/workers-http.mjs'; import { workerDefinition, ownerBytes, applyWorkerProposals, approveWorkerChecks } from '../tests/fixtures/workers-project.mjs'; if (process.argv.includes('--help')) { console.log('node scripts/verify-workers-ui.mjs — owned local fixture, real isolated Git/checks/review, desktop/390px, conflicts, stop, recovery and late source errors'); process.exit(0); } -const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, prefix = version === '1.17.0' ? 'model-protocols-workers' : version === '1.16.0' ? 'mobile-companion-workers' : version === '1.15.0' ? 'desktop-host-workers' : version === '1.14.0' ? 'controlled-engine-workers' : version === '1.13.0' ? 'workers' : 'workers-development'; +const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, prefix = version === '1.18.0' ? 'vision-inputs-workers' : version === '1.17.0' ? 'model-protocols-workers' : version === '1.16.0' ? 'mobile-companion-workers' : version === '1.15.0' ? 'desktop-host-workers' : version === '1.14.0' ? 'controlled-engine-workers' : version === '1.13.0' ? 'workers' : 'workers-development'; const directory = path.resolve('.qa', 'workers-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true }); const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t), host = await workerHttp(f); assert.notEqual(new URL(host.url).port, '4318'); diff --git a/src/server/diagnostics.mjs b/src/server/diagnostics.mjs index 28b23602..d30341bc 100644 --- a/src/server/diagnostics.mjs +++ b/src/server/diagnostics.mjs @@ -1,6 +1,6 @@ import { randomUUID, createHash } from 'node:crypto'; -export const APP_VERSION = '1.17.0'; +export const APP_VERSION = '1.18.0'; export function executionEnvironment() { const windows = process.platform === 'win32'; diff --git a/tests/fixtures/engineering-specification.mjs b/tests/fixtures/engineering-specification.mjs new file mode 100644 index 00000000..ab46281a --- /dev/null +++ b/tests/fixtures/engineering-specification.mjs @@ -0,0 +1,28 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert/strict'; + +export function specification(kind, root) { + if (kind === 'node') return { tests: 8, checks: ['test', 'build'], initial: null, fixes: { + 'src/add.mjs': 'export function add(left, right) {\n if (!Number.isFinite(left) || !Number.isFinite(right)) throw new TypeError("finite numeric inputs required");\n return left + right;\n}\n', + 'src/range.mjs': 'export function inclusiveRange(start, end) {\n if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || end < start || end - start >= 10000) throw new RangeError("bounded ascending integer range required");\n return Array.from({ length: end - start + 1 }, (_, index) => start + index);\n}\n', + } }; + if (kind === 'react') { + const app = fs.readFileSync(path.join(root, 'src/App.tsx'), 'utf8'); assert.ok(app.includes("toggleTask(current, 'missing')")); + return { tests: 9, checks: ['test', 'typecheck', 'build'], initial: null, fixes: { + 'src/query.ts': "import type { Task } from './types.ts';\nexport function selectTasks(tasks: Task[], filter: string, query: string): Task[] {\n const text = query.trim().toLocaleLowerCase();\n return tasks.filter(task => (filter === 'done' ? task.done : filter === 'pending' ? !task.done : true) && task.title.toLocaleLowerCase().includes(text));\n}\n", + 'src/summary.ts': "import type { Task } from './types.ts';\nexport function summarizeTasks(tasks: Task[]) {\n return { total: tasks.length, completed: tasks.filter(task => task.done).length, pendingHours: tasks.filter(task => !task.done).reduce((sum, task) => sum + task.hours, 0) };\n}\n", + 'src/transition.ts': "import type { Task } from './types.ts';\nexport function toggleTask(tasks: Task[], id: string): Task[] {\n return tasks.map(task => task.id === id ? { ...task, done: !task.done } : task);\n}\n", + 'src/App.tsx': app.replace("toggleTask(current, 'missing')", 'toggleTask(current, task.id)'), + } }; + } + const test = file => fs.readFileSync(path.join('benchmarks/workers-delivery-v1/python-tests', file), 'utf8'); + return { tests: 10, checks: ['python:test', 'python:compile'], initial: { + 'money.py': 'def to_cents(value):\n return int(float(value) * 100)\n', + 'schedule.py': 'def normalize_schedule(values):\n return list(values)\n', + 'tests/test_money.py': test('test_money.py'), 'tests/test_schedule.py': test('test_schedule.py'), + }, fixes: { + 'money.py': 'from decimal import Decimal, InvalidOperation, ROUND_HALF_UP\n\ndef to_cents(value):\n try:\n amount = Decimal(str(value))\n if not amount.is_finite():\n raise ValueError("finite amount required")\n return int((amount * 100).quantize(Decimal("1"), rounding=ROUND_HALF_UP))\n except (InvalidOperation, OverflowError, TypeError) as error:\n raise ValueError("valid finite amount required") from error\n', + 'schedule.py': 'import re\nfrom datetime import date\n\ndef normalize_schedule(values):\n dates = []\n for value in values:\n if not isinstance(value, str) or not re.fullmatch(r"\\d{4}-\\d{2}-\\d{2}", value):\n raise ValueError("ISO calendar date required")\n dates.append(date.fromisoformat(value))\n return [value.isoformat() for value in sorted(set(dates))]\n', + } }; +}