From d5207dc6cc6953c2b9dced2fe820645291c1f2a8 Mon Sep 17 00:00:00 2001 From: Agent Workbench Dev Date: Sat, 7 Nov 2026 23:25:49 +0800 Subject: [PATCH] test(ci): verify complete Python reports and exact model references --- docs/M2-CI-PLAN.md | 7 +++++++ tests/python-api.test.mjs | 10 +++++++++- tests/python-checks.test.mjs | 4 +++- tests/reasoning-settings.test.mjs | 7 ++++++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/M2-CI-PLAN.md b/docs/M2-CI-PLAN.md index 1874e430..e8bd3e97 100644 --- a/docs/M2-CI-PLAN.md +++ b/docs/M2-CI-PLAN.md @@ -37,3 +37,10 @@ - 最小回放准备实读112份已提交来源并核当前字节与Git blob一致;生成65份2452465字节(四engine子树、一R原证明/六回执/请求子树、60锁定公开Purge源码)。INDEX逐SHA/字段来源及公开源码原license/NOTICE保持,原来源前后不变;三个套件保留所有断言,仅改为本地声明输入。新CI前置核全部夹具,漂移/缺失/逃逸反例待专项验证,不将JSON投影冒称完整原会话字节。 - 四engine新/旧原生输入、模型配置、工具回执子树和实际engineContextView输出逐项一致;R原证明/六回执/请求一致,112原来源再次核SHA通过。原三个回归套件加CI负向边界专项35/35(39303ms),未知文件/同长度SHA变化/缺文件/路径逃逸均不能通过;未新增真实模型或实际Harness派发。旧全套仍持续收集,尚不记CI全绿。 - 冒烟清理复核发现原`portClosed:true`仅由HTTP health不可取和PID退出推导,未直接核监听。增加仅针对本次记录127.0.0.1端口的TCP接收/拒绝观察;缺health但仍监听不能通过,异常/超时保持失败,不结束外来进程。同页watch去重避免把一次浏览器事件记两次。首次直接工具专项8/8(没有独立磁盘原日志),规范最终文本后另用记录器核最终输入;实际Edge/Electron仍待最终Windows run。 +- 36c6be4f首完整Windows run自然结束:146文件/1211项,1194通过17失败,0取消/跳过/todo,3534259ms;未运行后置桌面冒烟。原报告/原日志/最终MANIFEST不追加,另在`remaining-before-2/first-complete-ci-review.json`只读核1308源码输入与编译文件前后一致。此前12项由旧断言/包外回放输入修正覆盖,余项仍单独追因,不冒称首全套全绿。 +- 交付报告增长后,Python HTTP测试仍把64KB预览当完整JSON;改用原下载路由并与工件元数据逐字节长度/SHA核对,再保留原Git/工程/测试/文件来源断言,不裁短报告或删账本。虚构`deepseek-owned`前缀不再推出官方厂商;补未知兼容、显式provider和精确官方型号三种来源断言,原原生发送与负向输入核验保留。 +- Python深路径独立复现原ensurepip及开启UTF-8后的失败;内层原错明确为setuptools包路径WinError206,并非编码根因。该用例只运行stdlib unittest,改为真实`venv --without-pip`,149字符项目环境仍静态deferred、实际解释器3.11.15和1项unittest通过。CI自身带哈希pip/pytest安装不省略,产品解释器识别/身份守卫未改。 +- 局部首复验32项27通过5失败;复验器误额外设置`AGENT_WORKBENCH_PYTHON_PATH`,覆盖项目`.venv`选择导致其中3项失败,与首完整CI环境不同。保留原命令/输入/日志,移除复验器该覆盖后另轮验证。其它推理旧断言与研究读取失败再现,提醒原20秒等待在局部原用例通过;不把这些替代首并发失败。 +- 研究原失败JSON逐层复核发现首因是旧16K有限夹具的Evidence字段读取返回“预算无法容纳元数据”,随后本机模型夹具解析失败并触发其重试,才出现第4/5次工具范围断言。完整原状态在`research-first-failure-state-1.json`,这是受控夹具错误,不是外部供应商异常。下一让该研究功能用例采用当前不限/完整历史默认,同时核对开放诊断工具广告与既有执行契约一致;不恢复额度或新增执行拦截。 +- 原提醒五请求/原20秒等待另做逐请求和状态观察,11.6秒独立用例通过;原并发超时的最后状态未封存,不能补造唯一性能根因。后续验收按五个实际请求的进展观察等待,仍核一次发送/当前证据/resolved/跨会话拒读/重启持久,不把总20秒误当产品运行额度。 +- 三处测试修正最终规范文本专项23/23,0失败/取消/跳过/todo,102798ms;正确QA解释器环境下,先前误覆盖的三项识别/身份用例均通过。全部产品/测试/脚本与编译输入前后一致,见`remaining-python-profile-fixed/run-windows-mw6jyjbh-5f925ebf/evidence/`。这仅是复用已建依赖/编译的专项,最后干净安装全套仍待;原首轮与局部失败未改写。 diff --git a/tests/python-api.test.mjs b/tests/python-api.test.mjs index fa747afd..6eac8e88 100644 --- a/tests/python-api.test.mjs +++ b/tests/python-api.test.mjs @@ -2,6 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; +import { createHash } from 'node:crypto'; import { fixture, launchWorkbench, until, compatibleFixture, call, completion, assertToolPairs } from './helpers.mjs'; import { deliveryGit } from './fixtures/delivery-project.mjs'; import { captureWorkspaceSource, assertWorkspaceSource } from './fixtures/workspace-project.mjs'; @@ -103,7 +104,14 @@ test('an isolated three-module Python Task repairs, checks and delivers a source await host.api(route + '/delivery-jobs', { planId: session.execution.deliveryPlans.at(-1).id }); await wait(value => value.execution.deliveries.at(-1).status === 'completed' && !!value.execution.deliveries.at(-1).reportArtifactId); assert.equal(model.requests.length, requests); assert.equal(session.status, 'completed'); - const job = session.execution.deliveries.at(-1), report = JSON.parse((await host.api('/artifacts/' + job.reportArtifactId + '?session=' + session.id + '&limit=64000')).content); + const job = session.execution.deliveries.at(-1), artifactRoute = '/artifacts/' + job.reportArtifactId + '?session=' + session.id; + const metadata = await host.api(artifactRoute + '&limit=1'); + const download = await fetch(host.url + '/api' + artifactRoute + '&download=1', { headers: { 'X-Workbench-Token': host.bootstrap.token, Connection: 'close' } }); + assert.equal(download.status, 200); + const bytes = Buffer.from(await download.arrayBuffer()); + assert.equal(bytes.length, metadata.bytes); + assert.equal(createHash('sha256').update(bytes).digest('hex'), metadata.sha256); + const report = JSON.parse(bytes.toString('utf8')); assert.equal(report.status, 'verified_committed'); assert.equal(report.workspace.workspaceId, workspace.id); assert.equal(report.workspace.sourceRoot, f.projectRoot); assert.equal(report.task.id, taskId); assert.deepEqual(job.paths.slice().sort(), Object.keys(pythonFixes).sort()); diff --git a/tests/python-checks.test.mjs b/tests/python-checks.test.mjs index 6015e403..775f5bce 100644 --- a/tests/python-checks.test.mjs +++ b/tests/python-checks.test.mjs @@ -190,7 +190,9 @@ test('real pytest pass, failure, skip, collection errors and teardown errors use test('a real project virtual environment is detected statically and its actual version is recorded only during the approved check', { timeout: 20000 }, async t => { const f = fixture(t); pythonProject(f, onePass); const host = await resolvePythonRuntime(f.project, { executable: process.env.AGENT_WORKBENCH_TEST_PYTHON }); - execFileSync(host.baseExecutable, ['-I', '-m', 'venv', path.join(f.projectRoot, '.venv')], { windowsHide: true, cwd: f.root, stdio: 'pipe' }); + // This scenario executes stdlib unittest. It needs a real project venv, + // not ensurepip's unrelated setuptools fixtures (overlong Windows paths). + execFileSync(host.baseExecutable, ['-I', '-X', 'utf8', '-m', 'venv', '--without-pip', path.join(f.projectRoot, '.venv')], { windowsHide: true, cwd: f.root, stdio: 'pipe' }); const runner = new CheckRunner(f.store); f.onCleanup(() => runner.stopAll()); const plan = await runner.plan(f.session); assert.equal(plan.checks[0].python.runtime.status, 'deferred'); assert.equal(plan.checks[0].python.runtime.version, null); diff --git a/tests/reasoning-settings.test.mjs b/tests/reasoning-settings.test.mjs index 883fa115..6ef2b937 100644 --- a/tests/reasoning-settings.test.mjs +++ b/tests/reasoning-settings.test.mjs @@ -36,7 +36,12 @@ test('brand and custom-compatible profiles send native reasoning controls per pr assert.equal(body.model, member.model); } assert.equal(actualRequest(plan, cases[0]).thinking, undefined); assert.equal(actualRequest(plan, cases[4]).reasoning_effort, undefined); - assert.equal(modelParameterProfile({ ...config, brand: '自定义' }).id, 'deepseek'); + const unknown = modelParameterProfile({ ...config, brand: '自定义' }); + assert.equal(unknown.id, 'openai'); assert.equal(unknown.sourceKind, 'compatibility_profile'); assert.equal(unknown.officialSource, null); + const explicit = modelParameterProfile({ ...config, providerId: 'deepseek' }); + assert.equal(explicit.id, 'deepseek'); assert.equal(explicit.officialSource, null, 'a provider declaration does not invent an exact model specification'); + const official = modelParameterProfile({ ...config, model: 'deepseek-v4-pro' }); + assert.equal(official.id, 'deepseek'); assert.equal(official.sourceKind, 'official_model_spec'); assert.equal(official.officialSource.provider, 'deepseek'); assert.equal(modelParameterProfile({ ...config, parameterProfile: 'openai' }).id, 'openai'); for (const [protocol, params, profile] of [ ['chat-completions', { thinking: { type: 'adaptive' } }], ['chat-completions', { thinking: { type: 'enabled', budget_tokens: 1024 } }],