diff --git a/CHANGELOG.md b/CHANGELOG.md index 901aff29..8cc98de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Agent Workbench 更新记录 +## 1.20.0 · 固定工程评测与受控候选采纳 + +- 新增工程评测:每批1–3个固定样本,封存已提交来源、需求、验收、明确源码文件、完整原检查和总预算;原版与候选使用同源独立工作区。 +- 原版只执行原检查,候选复用原模型连接/池、提案审查、命令审批与停止恢复;测试、脚本和运行声明保持原输入,执行前后核对来源。 +- 桌面/窄屏展示原版与候选的全部检查尝试、失败/停止/未知、日志SHA、人工审批/应用、用时和上游实际用量;缺失费用显示未报告,固定响应标签持续可见。 +- 全部样本封存后可采纳一项候选,必须经过父工程变更集审查、新检查及精确范围本地Git交付,保留父原暂存/未提交内容。 +- Node8/8、React9/9、Python10/10及来源绑定自身模块9/9的14节点试点通过。自身汇总修复已执行未封存状态误算未执行的问题;不据固定响应宣称模型推理收益或全仓自主升级。完整发行结果见QA-1.20。 + ## 1.19.0 · 来源核对的项目解析缓存 - 在原工程索引内增加有界持久声明/导入材料,绑定项目/工作区、Git身份、解析契约与源码SHA;返回前闭合来源核对。 diff --git a/README.md b/README.md index 5153a8d9..68add6c8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # 本地 Agent 工作台 +1.20.0候选正在进行完整发行验收。「工程评测」可固定需求和原测试、准备同源工作区、对照原版/候选真实结果,再经过父审查/检查/Git采纳。四类工程14节点与桌面/390px7组已通过;正式入口暂时保持1.19。见[使用说明](docs/ENGINEERING-EVALUATIONS.md)、[当前验收](docs/QA-1.20.md)与[回顾](docs/RETROSPECTIVE-1.20.md)。 + 1.19.0已完成本地正式升级和验收。工程「索引 → 解析缓存」可查看当前来源核对、真实复用/失效和容量,管理本机开关或仅清除本项目派生材料。独立590项、203组历史UI、三类实际工程、安装/升级、正式24页及原生回看通过;原连接和全部历史保持。跨实例Node/React有局部收益,冷/热/失效仍有来源核对开销,Python未得稳定收益。见[工作台](http://127.0.0.1:4318/?release=1.19.0)、[缓存](docs/PROJECT-CACHE.md)、[验收](docs/QA-1.19.md)、[身份](docs/project-cache-release-identity.json)和[后续方案](docs/PLAN-1.20.md)。 1.18.0历史交付工程截图、视觉选路与预算续接。可为工程任务添加PNG/JPEG截图,核对实际视觉选路及原图/发送副本;预算暂停会显示具体分解,预览确认后继续同一Task。独立源码576项、196组历史UI及截图Node/React/Python工程通过,原项目、会话和模型连接保持。见[工作台](http://127.0.0.1:4318/?release=1.18.0)、[截图与续接](docs/INPUT-ATTACHMENTS.md)、[验收](docs/QA-1.18.md)、[发行身份](docs/vision-inputs-release-identity.json)和[下一阶段](docs/PLAN-1.19.md)。 diff --git a/docs/ENGINEERING-EVALUATIONS.md b/docs/ENGINEERING-EVALUATIONS.md new file mode 100644 index 00000000..c23fee50 --- /dev/null +++ b/docs/ENGINEERING-EVALUATIONS.md @@ -0,0 +1,15 @@ +# 工程评测与候选采纳 + +在已有项目任务工具栏打开「工程评测」。先提交要作为来源的工程版本,填写需求、验收条件、允许修改的明确源码文件及原检查名称。来源是当前已提交HEAD;原暂存、未提交和未跟踪内容留在父工程。 + +每批1–3个样本,按队列逐个推进。先核对并准备同源工作区,再运行原版检查。原版不请求模型、不接受改动;它保存修复前真实失败。候选使用所选连接或池,仍须在所属子任务审查提案、批准原检查。原测试、配置、脚本和运行声明不在可改范围,执行前后会核对同源字节。 + +候选结束后封存当前结果,查看「原版 / 候选对照」并导出原报告。失败、停止和未知样本进入原分母;报告保留全部检查尝试、原日志SHA、用时、实际审批/应用次数和上游报告的用量。上游不报告的费用保持未知。「本地固定响应」仅验证执行链,不代表模型自主能力;「实际配置模型」也只说明当前固定样本结果。 + +全部样本封存后,选择一个有完整原版结果且候选通过的样本采纳。采纳只生成父工程审查批次;完成应用后重新执行父工程检查,并通过原本地Git交付。最后点击「核对父工程检查与 Git」完成该批。来源、模型、原检查或工件变化会保留具体原因并阻止继续。 + +停止会等待所属执行结束,保留原记录。重启后的未确定副作用显示待核对,不自动重放。可从原隔离协作入口核对现场,或选择保留现场结束;重新评测须新建批次。此为宿主权限和来源核验,不是操作系统沙箱。 + +工作台自身改进也经过相同来源/审查/检查/Git流程。当前工作区仍有1000文件/16MiB边界;超出边界的全仓必须另行设计,不能靠扩大限制宣称支持。1.20只验收来源绑定模块试点,正式发行目录、数据与安装指针不参与候选写入。发布仍经过独立源码、便携、安装、保留状态升级和原生回看。 + +本阶段结果见[1.20验收](QA-1.20.md)与[回顾](RETROSPECTIVE-1.20.md)。真实模型工程、Purge推理收益和新增iPhone流程分别待验。 diff --git a/docs/PLAN-1.21.md b/docs/PLAN-1.21.md new file mode 100644 index 00000000..92aae4e6 --- /dev/null +++ b/docs/PLAN-1.21.md @@ -0,0 +1,24 @@ +# 1.21 后续方案:工程依赖与可复现环境 + +本文件是1.20之后的方案,不表示已经实现。1.20先收齐完整回归、独立发行、保留状态升级和回顾,再开始此阶段。真实模型工程与Purge收益继续按原选择单独验收。 + +## 要解决的具体缺口 + +目前Node/React/Python工程已有真实检查、隔离候选、原版对照和父交付,但React等工程仍需QA或用户显式准备依赖。一个刚克隆的项目若缺少依赖,模型容易把环境失败误当代码失败,也无法证明更改在相同环境中通过。下一阶段优先将环境核对与依赖准备变成可审查、可停止、可复现的所属工程操作。 + +## 实施切片与验收 + +| 切片 | 实现 | 完成门槛 | +| --- | --- | --- | +| A 只读环境诊断 | 识别实际Node/npm或Python/pytest、声明/lockfile、依赖缺失与现有环境来源 | 缺失/不支持明确分类,诊断不安装或改配置;路径与真实二进制SHA绑定 | +| B 固定依赖准备计划 | 在所属隔离工作区创建不可变计划,展示实际命令、来源仓库、锁文件、目标和可能生命周期脚本 | 不执行任意模型拼接命令;锁文件/解释器/目录变化失效;不默认修改全局环境 | +| C 审批与所属执行 | 复用现有审批/命令账本/预算/停止,保存网络失败、退出码和原日志 | 超时与停止只结束所属进程树;未知结果不自动重试;不把安装成功当工程通过 | +| D 检查与评测衔接 | 原版/候选绑定同一依赖输入和实际准备记录,后续测试/构建报告引用环境身份 | 无依赖的工程、已有依赖的工程和依赖变化各有真实结果,失败分母保持 | +| E 中文桌面/窄屏流程 | 环境诊断 → 核对准备计划 → 审批 → 原日志 → 实际检查 | 保存/刷新/停止/具体错误恢复能走通,关键依赖状态和人工操作可见 | +| F 独立发行与回顾 | 全历史门槛、固定Node/React/Python、断网/损坏/变化/重启、独立源码/便携/安装与保留状态升级 | 原用户数据/连接/工件保持,原失败封存,本地Git、明确限制和下一计划 | + +## 保持的边界 + +优先npm固定lockfile和已有受支持Python检查,不同时加入所有包管理器。依赖生命周期脚本按实际风险和来源展示、按既有权限审批,不能把模型选择等同授权。下载与本地缓存分开记账;缓存只作派生材料,不保存密钥。网络或可信源缺失时保留明确失败,不静默切换注册源或降低校验。 + +后续真实多模型复核、其它引擎、语音和更大工程范围,仍按实际问题、契约与可复现实测分别排期;不因一项接入成功宣称全部能力完成。 diff --git a/findings.md b/findings.md index e00bda89..58e4d0cf 100644 --- a/findings.md +++ b/findings.md @@ -766,3 +766,7 @@ Responses已采用typed Items、call_id原配对和同线路加密reasoning续 # 1.20 A–C 回顾及 D 进行中 固定样本与同源原测试封存已接入原worker/检查/父审查/Git,首轮10/10及评测12+原worker6合计18/18通过。采纳改为持有父活动锁,并在应用前核对原报告;原版证据缺失禁止采纳。D已构建,首次实测暴露select可访问名称问题,补aria-label并另开独立收据复验,原失败保留。Windows rg文件通配参数应使用-g,不能把目录通配符当显式路径。QA状态只查实际存在的workbench.sqlite;临时夹具可能随测试完成删除,不按猜测路径诊断正式数据。 + +## 1.20 D/E与F候选的确认 + +四类工程14节点通过;自身原2a44f15函数固定9测试中4失败、候选9通过,主仓已合入并经14/14评测专项验证。D/E698份原件的暂存Git blob已逐字节/SHA核对,本地提交196b58a。原始日志/补丁工件声明binary,未规范化原空行。封存时仍在运行的专项日志保留部分原件并另存最终日志。F新编译服务首3组通过,Git preparing被QA误判终态;修正等待明确终态后独立整轮复验。版本准备脚本断言一个不存在的旧版本枚举后中止,余下门控按实际文件补齐,未重写正式环境。下一候选方案1.21优先工程依赖与可复现环境,解决目前React依赖仍靠QA显式准备的真实缺口。 diff --git a/package-lock.json b/package-lock.json index 6694b0e6..b0345517 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-agent-workbench", - "version": "1.19.0", + "version": "1.20.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-agent-workbench", - "version": "1.19.0", + "version": "1.20.0", "dependencies": { "@babel/parser": "7.29.11", "@peculiar/x509": "3.1.0", diff --git a/package.json b/package.json index 9018e36c..fa33c89a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-agent-workbench", - "version": "1.19.0", + "version": "1.20.0", "private": true, "type": "module", "description": "A local coding workbench with explicit tool execution, reviewable changes and recoverable sessions.", @@ -43,7 +43,10 @@ "verify:vision-projects": "node scripts/verify-vision-projects.mjs", "verify:project-cache": "node scripts/verify-project-cache-ui.mjs", "verify:project-cache-projects": "node scripts/verify-project-cache-projects.mjs", - "verify:model-protocols": "node scripts/verify-model-protocols-ui.mjs" + "verify:model-protocols": "node scripts/verify-model-protocols-ui.mjs", + "verify:evaluations": "node scripts/verify-evaluations-ui.mjs", + "verify:evaluations-projects": "node scripts/verify-evaluations-projects.mjs", + "verify:evaluations-release": "node scripts/verify-evaluations-release.mjs" }, "dependencies": { "@babel/parser": "7.29.11", diff --git a/scripts/qa-evidence.mjs b/scripts/qa-evidence.mjs index 4a8ffae8..c41b46c3 100644 --- a/scripts/qa-evidence.mjs +++ b/scripts/qa-evidence.mjs @@ -12,6 +12,7 @@ const prefixes = Object.freeze({ '1.17.0': 'model-protocols', '1.18.0': 'vision-inputs', '1.19.0': 'project-cache', + '1.20.0': 'engineering-evals', }); export function releaseEvidencePrefix(version) { const prefix = prefixes[version]; diff --git a/scripts/upgrade-running.mjs b/scripts/upgrade-running.mjs index 3b3795a3..40bf6478 100644 --- a/scripts/upgrade-running.mjs +++ b/scripts/upgrade-running.mjs @@ -64,7 +64,7 @@ export async function upgradeRunning({ url, pid, currentRoot, newRoot, handlerRo if (!fs.existsSync(originalScript) || !fs.existsSync(upgradeScript) || !fs.existsSync(path.join(next, 'dist-client', 'index.html'))) throw new Error('Upgrade bundles are incomplete.'); if (!Number.isInteger(pid) || pid < 1 || pid === process.pid) throw new Error('Invalid target process.'); const before = await readUpgradeJson(origin + '/api/bootstrap', 5000, 'Read original Workbench'); - if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.'); + if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.'); if (before.sessions.some(session => ['running', 'awaiting_approval', 'stopping'].includes(session.status))) throw new Error('Wait for existing tasks and approvals before upgrading.'); let protocol, socket, enabledHere = false, paused = false; const breakpoints = []; diff --git a/scripts/verify-all-ui.mjs b/scripts/verify-all-ui.mjs index 83433d04..226da546 100644 --- a/scripts/verify-all-ui.mjs +++ b/scripts/verify-all-ui.mjs @@ -9,30 +9,31 @@ import { randomUUID } from 'node:crypto'; if (process.argv.slice(2).some(value => value !== '--help')) throw new Error('Unsupported argument; use --help.'); if (process.argv.includes('--help')) { console.log('node scripts/verify-all-ui.mjs - run serial desktop/390px checks in independent local hosts; use explicit QA Python/Harness/Purge source paths for 1.15.'); process.exit(0); } const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version; -assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version), 'Review evidence paths when introducing the next release.'); +assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version), 'Review evidence paths when introducing the next release.'); const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = []; const qaEnvironment = Object.fromEntries(['AGENT_WORKBENCH_TEST_PYTHON', 'AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_178_ROOT', 'AGENT_WORKBENCH_PURGE_181_ROOT'] .filter(name => process.env[name]).map(name => [name, process.env[name]])); for (const [name, value] of Object.entries(qaEnvironment)) assert.ok(path.isAbsolute(value) && fs.existsSync(value), 'Use an existing absolute QA runtime/source path: ' + name); const cases = [ - ...(version === '1.19.0' ? [['verify-project-cache-ui.mjs', null]] : []), - ...(['1.18.0', '1.19.0'].includes(version) ? [['verify-vision-ui.mjs', null]] : []), - ...(['1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-model-protocols-ui.mjs', null]] : []), - ...(['1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-companion-ui.mjs', null]] : []), - ...(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-controlled-ui.mjs', null], ['verify-purge-ui.mjs', null]] : []), - ...(['1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-workers-ui.mjs', null]] : []), - ...(['1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []), - ...(['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []), + ...(version === '1.20.0' ? [['verify-evaluations-ui.mjs', null]] : []), + ...(['1.19.0', '1.20.0'].includes(version) ? [['verify-project-cache-ui.mjs', null]] : []), + ...(['1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-vision-ui.mjs', null]] : []), + ...(['1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-model-protocols-ui.mjs', null]] : []), + ...(['1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-companion-ui.mjs', null]] : []), + ...(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-controlled-ui.mjs', null], ['verify-purge-ui.mjs', null]] : []), + ...(['1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-workers-ui.mjs', null]] : []), + ...(['1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []), + ...(['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []), ['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'], ['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'], ['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`], ['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`], ['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`], - ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []), - ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []), - ...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []), - ...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []), - ...(['1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(version) ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []), + ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []), + ...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []), + ...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []), + ...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []), + ...(['1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(version) ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []), ]; fs.mkdirSync('.qa', { recursive: true }); const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']); diff --git a/scripts/verify-context-ui.mjs b/scripts/verify-context-ui.mjs index d1df5c10..66c509f5 100644 --- a/scripts/verify-context-ui.mjs +++ b/scripts/verify-context-ui.mjs @@ -77,7 +77,7 @@ child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); }); const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } }; await until(() => url || child.exitCode !== null); assert.ok(url, errors); const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json(); -assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); +assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; }; const proofPrefix = bootstrap.version === '1.6.0' ? 'workspaces-context' : releaseEvidencePrefix(bootstrap.version); const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); diff --git a/scripts/verify-controlled-release.mjs b/scripts/verify-controlled-release.mjs index 75cd0165..d3544a47 100644 --- a/scripts/verify-controlled-release.mjs +++ b/scripts/verify-controlled-release.mjs @@ -22,7 +22,7 @@ export async function verifyControlledApi({ api, url, token, directory, appRoot, const origin = new URL(url); assert.equal(origin.hostname, '127.0.0.1'); assert.notEqual(origin.port, '4318'); const expectedVersion = JSON.parse(fs.readFileSync(path.join(appRoot, fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json'), 'utf8')).version; - assert.ok(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(expectedVersion)); + assert.ok(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(expectedVersion)); const bootstrap = await api('/bootstrap'); assert.equal(bootstrap.version, expectedVersion); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false); const root = path.join(directory, 'compiled-controlled-node'), hooks = path.join(directory, 'disabled-hooks'); diff --git a/scripts/verify-evaluations-release.mjs b/scripts/verify-evaluations-release.mjs new file mode 100644 index 00000000..2de15188 --- /dev/null +++ b/scripts/verify-evaluations-release.mjs @@ -0,0 +1,65 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import http from 'node:http'; +import assert from 'node:assert/strict'; +import {randomUUID,createHash} from 'node:crypto'; +import {chromium,expect} from 'playwright/test'; +import {fixture,launchWorkbench,until,call,completion,assertToolPairs} from '../tests/helpers.mjs'; +import {createDeliveryProject,deliveryGit} from '../tests/fixtures/delivery-project.mjs'; +import {ownerBytes} from '../tests/fixtures/workers-project.mjs'; + +if(process.argv.includes('--help')){console.log('node scripts/verify-evaluations-release.mjs [--app-root ] — actual compiled host/bundled Node, fixed HTTP candidate, original failing checks, review, desktop/390px comparison/export, parent checks/Git, stop and source rejection. Independent data; no real inference.');process.exit(0);} +const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1])); +const appRoot=fs.realpathSync.native(args[1]??process.cwd()),nodePath=args.length?path.join(appRoot,'runtime/node.exe'):process.execPath; +const directory=path.resolve('.qa','engineering-evals-release-'+randomUUID());fs.mkdirSync(directory,{recursive:true}); +const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t);createDeliveryProject(f,{broken:true,ownerChanges:true});const original=ownerBytes(f);f.store.close(); +const hash=bytes=>createHash('sha256').update(bytes).digest('hex'),requests=[],held=new Set(),sockets=new Set(),assertions=[],screenshots=[],pageErrors=[]; +let host,childRoute,hold=false,browser,report; +const model=http.createServer(async(req,res)=>{try{ + const chunks=[];for await(const chunk of req)chunks.push(chunk);const body=JSON.parse(Buffer.concat(chunks));requests.push({route:req.url,body});assert.equal(req.url,'/v1/chat/completions');assertToolPairs(body.messages); + if(hold){res.writeHead(200,{'Content-Type':'application/json'});res.flushHeaders();held.add(res);res.on('close',()=>held.delete(res));return;} + const current=await host.api(childRoute),task=current.taskId,changes=current.changes.filter(row=>row.taskId===task),plan=current.execution.plans.at(-1),job=current.execution.checks.at(-1);let calls=[]; + if(!changes.length)calls=[call(randomUUID(),'propose_file',{path:'src/add.mjs',content:'export const add = (a,b) => a + b;\n',reason:'随包固定HTTP候选,保留原测试'})]; + else if(!changes.some(row=>row.status==='pending')){if(!plan)calls=[call(randomUUID(),'plan_checks',{names:['test','build']})];else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];} + res.writeHead(200,{'Content-Type':'application/json'});res.end(JSON.stringify(completion(calls,calls.length?'':'固定响应完成,仍需核对实际结果'))); +}catch(error){res.writeHead(500,{'Content-Type':'application/json'});res.end(JSON.stringify({error:{message:error.message}}));}}); +model.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));});await new Promise(resolve=>model.listen(0,'127.0.0.1',resolve)); +const mark=value=>{assertions.push(value);console.log('PASS '+value);},route='/sessions/'+f.session.id; +const definition=title=>({title,evidenceMode:'local-fixture',samples:[{label:'原加法边界',goal:'修复加法,保留原调用与测试。',criteria:'原3测试和构建通过,原测试/脚本保持不变。',paths:['src/add.mjs'],checks:['test','build']}],budget:{maxTokens:200000,maxSteps:32,maxToolCalls:64,maxDurationMs:600000}}); +const get=async id=>(await host.api(route+'/evaluations')).batches.find(row=>row.id===id); +async function operation(batch,name){await host.api(route+'/evaluations/'+batch.id+'/'+name,{fingerprint:batch.fingerprint,sampleId:batch.samples[0].id});await until(async()=>!['preparing','baseline_running','candidate_running','collecting','adopting'].includes((await get(batch.id)).status),120000);return get(batch.id);} +async function apply(target,set){await host.api(target+'/change-sets/'+set.id+'/apply',{fingerprint:set.fingerprint});let value;await until(async()=>{value=await host.api(target+'/change-sets/'+set.id);return !['prepared','applying'].includes(value.status);});assert.equal(value.status,'completed',value.summary);} +try{ + host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath});assert.equal(fs.realpathSync.native(host.bootstrap.runtimeRoot),appRoot);assert.notEqual(new URL(host.url).port,'4318'); + await host.api('/config',{mode:'compatible',model:'owned-evaluation-release',baseUrl:'http://127.0.0.1:'+model.address().port+'/v1',apiKey:'self-owned-evaluation-release',maxSteps:32}); + let batch=(await host.api(route+'/evaluations',definition('随包工程评测'))).batch;batch=await operation(batch,'prepare');assert.equal(batch.samples[0].status,'ready',batch.message); + const group=(await host.api(route+'/collaborations')).groups.find(row=>row.id===batch.samples[0].groupId);childRoute='/sessions/'+group.children.find(row=>row.evaluation.role==='candidate').sessionId; + batch=await operation(batch,'baseline');assert.equal(batch.samples[0].status,'baseline_complete',batch.message);assert.equal(requests.length,0);assert.deepEqual(ownerBytes(f),original); + mark('selected compiled host prepares same-source workspaces and retains original three-test failure with zero original model requests and unchanged owner staging'); + batch=await operation(batch,'start');assert.equal(batch.samples[0].status,'awaiting_review',batch.message);let current=await host.api(childRoute); + const set=(await host.api(childRoute+'/change-sets',{changeIds:current.changes.filter(row=>row.status==='pending').map(row=>row.id)})).execution.changeSets.at(-1);await apply(childRoute,set); + await host.api(route+'/evaluations/'+batch.id+'/start',{fingerprint:batch.fingerprint,sampleId:batch.samples[0].id}); + await until(async()=>{current=await host.api(childRoute);for(const approval of current.approvals.filter(row=>row.status==='pending'))await host.api(childRoute+'/approvals/'+approval.id,{allow:true});batch=await get(batch.id);return !['candidate_running'].includes(batch.status);},120000); + assert.equal(batch.samples[0].status,'candidate_complete',batch.message);assert.equal(batch.summary.candidate.unknown,1);assert.equal(batch.summary.candidate.pending,0); + batch=await operation(batch,'seal');const comparison=await host.api(route+'/evaluations/'+batch.id+'/report?sampleId='+batch.samples[0].id);assert.deepEqual(comparison.summary,{original:'failed',candidate:'passed'});assert.equal(comparison.original.attempts[0].results[0].tests.failed,3);assert.equal(comparison.candidate.attempts.at(-1).results[0].tests.passed,3);assert.equal(comparison.inferenceValidated,false);assert.deepEqual(ownerBytes(f),original); + fs.writeFileSync(path.join(directory,'original-comparison.json'),JSON.stringify(comparison,null,2)); + mark('real owned HTTP tools, applied candidate review and approved original checks pass; unsealed attempted outcomes stay unknown and sealed report retains original failure'); + browser=await chromium.launch({channel:'msedge',headless:true});const page=await browser.newPage({viewport:{width:1440,height:1000},acceptDownloads:true});page.on('pageerror',error=>pageErrors.push(error.message)); + await page.addInitScript(({projectId,sessionId})=>{localStorage.setItem('awb-project',projectId);localStorage.setItem('awb-session',sessionId);},{projectId:f.project.id,sessionId:f.session.id}); + await page.goto(host.url);await page.waitForLoadState('networkidle');await page.getByRole('button',{name:'工程评测',exact:true}).click();const panel=page.getByLabel('工程评测控制台',{exact:true});await expect(panel).toContainText('随包工程评测'); + const refresh=panel.getByRole('button',{name:'刷新',exact:true});await refresh.focus();assert.ok(await refresh.evaluate(node=>node.matches(':focus')));await page.keyboard.press('Tab'); + await panel.getByRole('button',{name:'查看原版 / 候选对照',exact:true}).click();const dialog=page.getByRole('dialog',{name:'原版与候选对照',exact:true});await expect(dialog).toContainText('测试 0/3 · 失败 3');await expect(dialog).toContainText('测试 3/3 · 失败 0');await expect(dialog).toContainText('本地固定响应');await expect(dialog).toContainText('费用未报告'); + for(const viewport of [{width:1440,height:1000},{width:390,height:844}]){await page.setViewportSize(viewport);if(viewport.width===390)await dialog.getByText('测试 3/3 · 失败 0',{exact:true}).scrollIntoViewIfNeeded();assert.ok(await page.locator('body').evaluate(node=>node.scrollWidth<=innerWidth));const file=path.join(directory,'comparison-'+viewport.width+'.png');await page.screenshot({path:file,fullPage:true});screenshots.push({file,sha256:hash(fs.readFileSync(file))});} + const downloaded=page.waitForEvent('download');await dialog.getByRole('button',{name:'导出原报告',exact:true}).click();const file=path.join(directory,'downloaded-report.json');await(await downloaded).saveAs(file);assert.deepEqual(JSON.parse(fs.readFileSync(file)).summary,comparison.summary);await page.keyboard.press('Escape');await page.reload();await page.waitForLoadState('networkidle');await page.getByRole('button',{name:'工程评测',exact:true}).click();await expect(panel).toContainText('随包工程评测'); + mark('compiled desktop/390px UI renders original and candidate checks, fixed-response label, unknown fee, keyboard focus and downloadable report; refresh preserves the same batch'); + batch=await operation(batch,'adopt');assert.equal(batch.status,'awaiting_delivery',batch.message);const adopted=(await host.api(route+'/collaborations')).groups.find(row=>row.id===group.id);await apply(route,await host.api(route+'/change-sets/'+adopted.merge.changeSetId)); + const plan=(await host.api(route+'/check-plans',{names:['test','build']})).execution.plans.at(-1),job=(await host.api(route+'/check-jobs',{planId:plan.id})).execution.checks.at(-1);let checked;await until(async()=>{checked=await host.api(route+'/check-jobs/'+job.id);return !['queued','running'].includes(checked.status);});assert.equal(checked.status,'passed',checked.summary); + const deliveryPlan=(await host.api(route+'/delivery-plans',{message:'Adopt compiled evaluation candidate'})).execution.deliveryPlans.at(-1),deliveryJob=(await host.api(route+'/delivery-jobs',{planId:deliveryPlan.id})).execution.deliveries.at(-1);let delivered;await until(async()=>{delivered=await host.api(route+'/delivery-jobs/'+deliveryJob.id);return ['completed','failed','cancelled','unknown'].includes(delivered.status);},120000);assert.equal(delivered.status,'completed',delivered.summary);batch=await operation(batch,'finalize');assert.equal(batch.adoption.status,'completed');assert.deepEqual(ownerBytes(f).files,original.files);assert.equal(deliveryGit(f,['diff-tree','--no-commit-id','--name-only','-r','HEAD']).trim(),'src/add.mjs'); + mark('fresh parent review, actual original parent checks and exact-scope local Git complete adoption while preserving owner staged/unstaged/untracked bytes'); + hold=true;let stopped=(await host.api(route+'/evaluations',definition('随包停止原请求'))).batch;stopped=await operation(stopped,'prepare');const stoppedGroup=(await host.api(route+'/collaborations')).groups.find(row=>row.id===stopped.samples[0].groupId);childRoute='/sessions/'+stoppedGroup.children.find(row=>row.evaluation.role==='candidate').sessionId;stopped=await operation(stopped,'baseline');const before=requests.length; + await host.api(route+'/evaluations/'+stopped.id+'/start',{fingerprint:stopped.fingerprint,sampleId:stopped.samples[0].id});await until(()=>requests.length===before+1);await host.api(route+'/evaluations/'+stopped.id+'/stop',{fingerprint:stopped.fingerprint,sampleId:stopped.samples[0].id});await until(()=>held.size===0);stopped=await operation(stopped,'seal');assert.equal(stopped.samples[0].report.summary.candidate,'stopped');assert.equal(requests.length,before+1); + const stale=(await host.api(route+'/evaluations',definition('随包来源变化'))).batch;fs.appendFileSync(path.join(f.projectRoot,'README.md'),'\nOwner source changed\n');const rejected=await operation(stale,'prepare');assert.equal(rejected.samples[0].status,'failed');assert.match(rejected.message,/来源工程已变化/); + mark('bundled candidate stop closes only its original owned HTTP request and keeps stopped outcome; changed source rejects before new workspaces without replay'); + assert.deepEqual(pageErrors,[]);report={success:true,version:host.bootstrap.version,appRoot,nodePath,containedRuntime:!!args.length,directory,assertions,screenshots,pageErrors,requests,comparison,parentCheck:checked,delivery:delivered,stopped,rejected,realModelRequests:0}; +}catch(error){report={success:false,error:error.stack,directory,appRoot,assertions,screenshots,pageErrors,requests,realModelRequests:0};throw error;} +finally{await browser?.close();await host?.close();for(const response of held)response.destroy();for(const socket of sockets)socket.destroy();await new Promise(resolve=>model.close(resolve));fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));for(const dispose of disposers.reverse())await dispose();console.log('EVIDENCE '+directory);} diff --git a/scripts/verify-release.mjs b/scripts/verify-release.mjs index 06fcebc7..a46a1a5e 100644 --- a/scripts/verify-release.mjs +++ b/scripts/verify-release.mjs @@ -34,7 +34,7 @@ for (const name of Object.keys(env)) if (name.toLowerCase() === 'path') delete e const windowsRoot = process.env.SystemRoot || 'C:\\Windows'; env.PATH = [windowsRoot, path.join(windowsRoot, 'System32'), path.join(windowsRoot, 'System32', 'WindowsPowerShell', 'v1.0')].join(path.delimiter); let externalGit = null; -if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(manifest.version)) { +if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(manifest.version)) { // The controlled project requires external Git. Preserve its observed binary // directory while keeping system Node/npm unavailable to the packaged host. const binary = fs.realpathSync.native(execFileSync('where.exe', ['git.exe'], { env: gitEnvironment(), encoding: 'utf8', windowsHide: true, timeout: 10000 }).trim().split(/\r?\n/)[0]); @@ -88,7 +88,7 @@ try { console.log((await launch()).trim()); assert.equal(JSON.parse(fs.readFileSync(path.join(data, 'service-info.json'), 'utf8')).pid, firstInfo.pid); const example = bootstrap.projects.find(project => project.id === bootstrap.exampleProjectId); - if (['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(manifest.version)) { + if (['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(manifest.version)) { assert.equal(path.resolve(example.root).toLowerCase(), path.join(data, 'projects', 'starter-project').toLowerCase()); const compareTemplate = relative => { for (const entry of fs.readdirSync(path.join(appRoot, 'examples/starter-project', relative), { withFileTypes: true })) { const next = path.join(relative, entry.name); if (entry.isDirectory()) compareTemplate(next); @@ -114,7 +114,7 @@ try { const checkLog = await api('/artifacts/' + checked.results[0].logArtifactId + '?session=' + session.id); assert.match(checkLog.content, /tests 4/); assert.ok(fs.existsSync(path.join(appRoot, 'runtime/node_modules/npm/bin/npm-cli.js'))); const actualRea = []; - if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(manifest.version)) { + if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(manifest.version)) { const installation = await api('/research/availability'); assert.equal(installation.status, 'available'); assert.equal(installation.version, '6.8.0'); assert.equal(fs.realpathSync.native(installation.packageRoot), fs.realpathSync.native(path.join(appRoot, 'runtime/node_modules/rea-agents'))); assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe'))); @@ -146,7 +146,7 @@ try { console.log('PASS actual contained REA: ASAR, PE imports and PE resources; zero target execution/model requests'); } let actualEngine = null; - if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(manifest.version) && process.argv.includes('--native-engines')) { + if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(manifest.version) && process.argv.includes('--native-engines')) { const installation = await api('/engines/deepseek/availability'); assert.equal(installation.status, 'available', installation.message); assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe'))); const engineRoot = path.join(data, 'engine-project'); fs.mkdirSync(engineRoot); fs.writeFileSync(path.join(engineRoot, 'README.md'), '# Owned portable engine connection\n'); @@ -166,8 +166,8 @@ try { assert.equal((await api('/research')).length, 4); const page = await fetch(url); assert.equal(page.status, 200); assert.match(await page.text(), /Agent Workbench/); - const actualBrowser = ['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; - const actualControlled = ['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(manifest.version) ? await verifyControlledApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; + const actualBrowser = ['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; + const actualControlled = ['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(manifest.version) ? await verifyControlledApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null; console.log((await stop()).trim()); started = false; await until(async () => { try { await fetch(url + '/api/health'); return false; } catch { return true; } }); for (const file of manifest.files) { @@ -176,7 +176,7 @@ try { } const result = { success: true, date: new Date().toISOString(), appRoot, version: manifest.version, nodeVersion: manifest.nodeVersion, verifiedFiles: manifest.files.length, url, reusedPid: firstInfo.pid, checks: { jobId, status: checked.status, tests: checked.results[0].tests }, actualRea, actualEngine, actualBrowser, actualControlled, externalGit, - steps: ['manifest hashes', 'bundled Node launch', 'same-service reuse', 'packaged example and static page', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(manifest.version) ? ['starter project copied to owned user data; original template bytes matched'] : []), 'explicit command consent', 'actual bundled Node version', 'actual bundled npm check without system Node/npm', 'check log artifact', ...(actualRea.length ? ['contained REA installation and Node identity', 'actual ASAR/imports/resources', 'original Bundle and target hash binding', 'verified reuse without producer replay'] : []), 'report availability', 'graceful stop', 'every original packaged file remains unchanged'], realModel: false }; + steps: ['manifest hashes', 'bundled Node launch', 'same-service reuse', 'packaged example and static page', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(manifest.version) ? ['starter project copied to owned user data; original template bytes matched'] : []), 'explicit command consent', 'actual bundled Node version', 'actual bundled npm check without system Node/npm', 'check log artifact', ...(actualRea.length ? ['contained REA installation and Node identity', 'actual ASAR/imports/resources', 'original Bundle and target hash binding', 'verified reuse without producer replay'] : []), 'report availability', 'graceful stop', 'every original packaged file remains unchanged'], realModel: false }; fs.writeFileSync('.qa/release-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2)); } finally { if (started) { try { await stop(); } catch {} } } diff --git a/scripts/verify-source-package.mjs b/scripts/verify-source-package.mjs index a3428474..47cdcff9 100644 --- a/scripts/verify-source-package.mjs +++ b/scripts/verify-source-package.mjs @@ -19,7 +19,7 @@ assert.ok(fs.existsSync(npmCli), 'Use Node with its bundled npm.'); const files = (root, relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)) .flatMap(entry => entry.isDirectory() ? files(root, path.join(relative, entry.name)) : [path.join(relative, entry.name)]); const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex'); -const buildFolders = ['dist-client', 'dist-server', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version) ? ['dist-desktop'] : [])]; +const buildFolders = ['dist-client', 'dist-server', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version) ? ['dist-desktop'] : [])]; const expected = Object.fromEntries(buildFolders.flatMap(folder => files(path.join(workspace, folder)).map(relative => [path.join(folder, relative), digest(path.join(workspace, folder, relative))]))); const allowedEnv = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON|AGENT_WORKBENCH_DEEPSEEK_ROOT|AGENT_WORKBENCH_PURGE_ROOT)$/i; const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => allowedEnv.test(key))); @@ -30,7 +30,7 @@ const directory = path.join(sourceRoot, '.qa', 'package-validation-' + randomUUI fs.mkdirSync(directory, { recursive: true }); const inputFolders = ['src', 'tests', 'scripts', ...['benchmarks', 'public', 'examples', 'research'].filter(folder => fs.existsSync(path.join(sourceRoot, folder)))]; const inputNames = [...inputFolders.flatMap(folder => files(path.join(sourceRoot, folder)).map(relative => path.join(folder, relative))), - 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', ...(['1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version) ? ['mobile.html'] : []), ...['.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].filter(name => fs.existsSync(path.join(sourceRoot, name)))].sort(); + 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', ...(['1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version) ? ['mobile.html'] : []), ...['.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].filter(name => fs.existsSync(path.join(sourceRoot, name)))].sort(); const snapshot = () => inputNames.map(relative => ({ path: relative.replaceAll('\\', '/'), bytes: fs.statSync(path.join(sourceRoot, relative)).size, sha256: digest(path.join(sourceRoot, relative)) })); const inputs = snapshot(); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2)); let stage = 'environment preflight', python = null; @@ -83,7 +83,7 @@ try { assert.deepEqual(actualFiles.slice().sort(), Object.keys(expected).sort()); for (const [relative, sha256] of Object.entries(expected)) assert.equal(digest(path.join(sourceRoot, relative)), sha256, relative); let actualRea = null; - if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'contained REA'; const reaOutput = await run(['scripts/verify-rea.mjs', '--app-root', sourceRoot, '--contained'], 'rea.log'); actualRea = JSON.parse(reaOutput.trim().split('\n').at(-1)); @@ -91,7 +91,7 @@ try { assert.ok(actualRea.results.every(item => item.rootExited === true)); } let actualEngine = null; - if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version) && process.argv.includes('--native-engines')) { + if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version) && process.argv.includes('--native-engines')) { stage = 'native DeepSeek SDK'; const output = await run(['scripts/verify-engines.mjs'], 'engines.log'); actualEngine = JSON.parse(output.trim().split('\n').at(-1)); @@ -99,7 +99,7 @@ try { assert.equal(actualEngine.modelRequests, 0); assert.equal(actualEngine.rootExited, true); assert.equal(actualEngine.nativeMethods.length, 6); } let actualBrowser = null; - if (['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'contained browser runtime'; const output = await run(['scripts/verify-browser.mjs', '--app-root', sourceRoot, '--contained'], 'browser.log'); actualBrowser = JSON.parse(output.trim().split('\n').at(-1)); @@ -109,7 +109,7 @@ try { assert.equal(actualBrowser.modelRequests, 0); } let actualControlled = null; - if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'compiled controlled-engine delivery'; const output = await run(['scripts/verify-controlled-release.mjs', '--app-root', sourceRoot], 'controlled-engine.log'); actualControlled = JSON.parse(output.trim().split('\n').at(-1)); @@ -117,7 +117,7 @@ try { assert.equal(actualControlled.realModelRequests, 0); assert.equal(actualControlled.cases, 4); assert.ok(actualControlled.fixtureModelRequests > 0); } let actualCompanion = null; - if (['1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'contained companion gateway and UI'; const output = await run(['scripts/verify-companion-ui.mjs', '--app-root', sourceRoot], 'companion-ui.log'); const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); @@ -128,7 +128,7 @@ try { assert.equal(actualCompanion.passed.length, 7); assert.deepEqual(actualCompanion.errors, []); assert.equal(actualCompanion.realModelRequests, 0); } let actualProtocols = null; - if (['1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'contained model protocols and mobile streams'; const output = await run(['scripts/verify-model-protocols-ui.mjs', '--app-root', sourceRoot], 'model-protocols-ui.log'); const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); @@ -139,7 +139,7 @@ try { assert.equal(actualProtocols.passed.length, 5); assert.deepEqual(actualProtocols.pageErrors, []); assert.deepEqual(actualProtocols.requestFailures, []); } let actualVision = null; - if (['1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'contained screenshot inputs and exact Task budget resume'; const output = await run(['scripts/verify-vision-ui.mjs', '--app-root', sourceRoot], 'vision-ui.log'); const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); @@ -150,7 +150,7 @@ try { assert.equal(actualVision.passed.length, 5); assert.deepEqual(actualVision.pageErrors, []); assert.equal(actualVision.realModelRequests, 0); } let actualProjectCache = null; - if (evidence.version === '1.19.0') { + if (['1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'contained source-verified parsing cache controls'; const output = await run(['scripts/verify-project-cache-ui.mjs', '--app-root', sourceRoot], 'project-cache-ui.log'); const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); @@ -165,7 +165,7 @@ try { const stats = Object.fromEntries([...output.matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])])); assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0); let actualDesktop = null; - if (['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'actual contained Electron window'; await run(['scripts/verify-desktop-window.mjs'], 'desktop-window.log'); const window = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/desktop-window-latest.json'), 'utf8')); assert.equal(window.success, true); assert.equal(path.resolve(window.appRoot), sourceRoot); assert.equal(window.results.length, 7); @@ -179,7 +179,7 @@ try { actualDesktop = { window, ui, projects, scope: 'Independently rebuilt source and its shipped Electron binary; original window/engine/engineering assertions. Own credentials/HTTP fixtures; Windows native picker and complete host crash/stop have separate actual evidence.' }; } let actualProtocolEngineering = null; - if (['1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.17.0', '1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'Responses SSE actual Node React Python engineering'; await run(['scripts/verify-controlled-projects.mjs', '--responses'], 'responses-engineering.log'); const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/responses-engineering-location.json'), 'utf8')); @@ -189,7 +189,7 @@ try { assert.deepEqual(actualProtocolEngineering.results.map(value => value.parentCheck.results[0].tests.passed), [8, 9, 10]); } let actualVisionEngineering = null; - if (['1.18.0', '1.19.0'].includes(evidence.version)) { + if (['1.18.0', '1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'screenshot Task Node React Python engineering'; await run(['scripts/verify-vision-projects.mjs'], 'vision-engineering.log'); const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/vision-engineering-location.json'), 'utf8')); @@ -199,7 +199,7 @@ try { assert.deepEqual(actualVisionEngineering.results.map(value => value.check.results[0].tests.passed), [8, 9, 10]); } let actualCacheEngineering = null; - if (evidence.version === '1.19.0') { + if (['1.19.0', '1.20.0'].includes(evidence.version)) { stage = 'cached index original requests and Node React Python engineering'; await run(['scripts/verify-project-cache-projects.mjs'], 'project-cache-engineering.log'); const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/project-cache-engineering-location.json'), 'utf8')); @@ -208,9 +208,27 @@ try { assert.equal(actualCacheEngineering.realModelRequests, 0); assert.equal(actualCacheEngineering.cases.length, 9); assert.deepEqual(actualCacheEngineering.results.map(value => value.check.results[0].tests.passed), [8, 9, 10]); } + let actualEvaluations = null, actualEvaluationEngineering = null; + if (evidence.version === '1.20.0') { + stage = 'compiled fixed engineering evaluations'; + const output = await run(['scripts/verify-evaluations-release.mjs', '--app-root', sourceRoot], 'evaluations-release.log'); + const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim(); + assert.ok(location && path.resolve(location).startsWith(path.join(sourceRoot, '.qa') + path.sep)); + actualEvaluations = JSON.parse(fs.readFileSync(path.join(location, 'result.json'), 'utf8')); + assert.equal(actualEvaluations.success, true); assert.equal(actualEvaluations.version, evidence.version); + assert.equal(fs.realpathSync.native(actualEvaluations.appRoot), sourceRoot); assert.equal(actualEvaluations.containedRuntime, true); + assert.equal(actualEvaluations.assertions.length, 5); assert.equal(actualEvaluations.realModelRequests, 0); assert.deepEqual(actualEvaluations.pageErrors, []); + stage = 'fixed Node React Python and source-bound module engineering'; + await run(['scripts/verify-evaluations-projects.mjs'], 'evaluations-engineering.log'); + const engineeringLocation = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/engineering-evals-projects-location.json'), 'utf8')); + actualEvaluationEngineering = JSON.parse(fs.readFileSync(engineeringLocation.result, 'utf8')); + assert.equal(actualEvaluationEngineering.success, true); assert.equal(actualEvaluationEngineering.version, evidence.version); + assert.equal(actualEvaluationEngineering.assertions.length, 14); assert.equal(actualEvaluationEngineering.realModelRequests, 0); + assert.deepEqual(actualEvaluationEngineering.results.map(row => row.parentCheck.results[0].tests.passed), [8, 9, 10, 9]); + } assert.deepEqual(snapshot(), inputs, 'Source package inputs changed during validation.'); const result = { success: true, version: evidence.version, testedAt: new Date().toISOString(), sourceRoot, directory, nodeVersion: process.version, python, offlineInstall: true, - inputs, inputFiles: inputs.length, logs, buildMatchedFiles: Object.keys(expected).length, buildFiles: Object.entries(expected).map(([path, sha256]) => ({ path: path.replaceAll('\\', '/'), sha256 })), tests: stats, actualRea, actualEngine, actualBrowser, actualControlled, actualDesktop, actualCompanion, actualProtocols, actualProtocolEngineering, actualVision, actualVisionEngineering, actualProjectCache, actualCacheEngineering, credentialsHandling: 'Only build/runtime environment paths are inherited; no configured model credentials are used. The controlled-engine stage supplies a declared owned local response fixture.' }; + inputs, inputFiles: inputs.length, logs, buildMatchedFiles: Object.keys(expected).length, buildFiles: Object.entries(expected).map(([path, sha256]) => ({ path: path.replaceAll('\\', '/'), sha256 })), tests: stats, actualRea, actualEngine, actualBrowser, actualControlled, actualDesktop, actualCompanion, actualProtocols, actualProtocolEngineering, actualVision, actualVisionEngineering, actualProjectCache, actualCacheEngineering, actualEvaluations, actualEvaluationEngineering, credentialsHandling: 'Only build/runtime environment paths are inherited; no configured model credentials are used. The controlled-engine stage supplies a declared owned local response fixture.' }; save(result); console.log(JSON.stringify({ success: true, version: result.version, sourceRoot, directory, tests: stats, nodeVersion: process.version, python, inputFiles: inputs.length, buildMatchedFiles: result.buildMatchedFiles, rea: !!actualRea?.success, engine: !!actualEngine?.success, browser: !!actualBrowser?.success, controlled: !!actualControlled?.success })); } catch (error) { save({ success: false, version: evidence.version, testedAt: new Date().toISOString(), stage, error: error.message, sourceRoot, directory, nodeVersion: process.version, python, inputs, logs }); diff --git a/scripts/verify-workers-ui.mjs b/scripts/verify-workers-ui.mjs index c5e05f11..af8806bd 100644 --- a/scripts/verify-workers-ui.mjs +++ b/scripts/verify-workers-ui.mjs @@ -8,7 +8,7 @@ import { workerHttp } from '../tests/fixtures/workers-http.mjs'; import { workerDefinition, ownerBytes, applyWorkerProposals, approveWorkerChecks } from '../tests/fixtures/workers-project.mjs'; if (process.argv.includes('--help')) { console.log('node scripts/verify-workers-ui.mjs — owned local fixture, real isolated Git/checks/review, desktop/390px, conflicts, stop, recovery and late source errors'); process.exit(0); } -const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, prefix = version === '1.19.0' ? 'project-cache-workers' : version === '1.18.0' ? 'vision-inputs-workers' : version === '1.17.0' ? 'model-protocols-workers' : version === '1.16.0' ? 'mobile-companion-workers' : version === '1.15.0' ? 'desktop-host-workers' : version === '1.14.0' ? 'controlled-engine-workers' : version === '1.13.0' ? 'workers' : 'workers-development'; +const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, prefix = version === '1.20.0' ? 'engineering-evals-workers' : version === '1.19.0' ? 'project-cache-workers' : version === '1.18.0' ? 'vision-inputs-workers' : version === '1.17.0' ? 'model-protocols-workers' : version === '1.16.0' ? 'mobile-companion-workers' : version === '1.15.0' ? 'desktop-host-workers' : version === '1.14.0' ? 'controlled-engine-workers' : version === '1.13.0' ? 'workers' : 'workers-development'; const directory = path.resolve('.qa', 'workers-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true }); const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t), host = await workerHttp(f); assert.notEqual(new URL(host.url).port, '4318');