test(ci): verify complete Python reports and exact model references
This commit is contained in:
@@ -37,3 +37,10 @@
|
||||
- 最小回放准备实读112份已提交来源并核当前字节与Git blob一致;生成65份2452465字节(四engine子树、一R原证明/六回执/请求子树、60锁定公开Purge源码)。INDEX逐SHA/字段来源及公开源码原license/NOTICE保持,原来源前后不变;三个套件保留所有断言,仅改为本地声明输入。新CI前置核全部夹具,漂移/缺失/逃逸反例待专项验证,不将JSON投影冒称完整原会话字节。
|
||||
- 四engine新/旧原生输入、模型配置、工具回执子树和实际engineContextView输出逐项一致;R原证明/六回执/请求一致,112原来源再次核SHA通过。原三个回归套件加CI负向边界专项35/35(39303ms),未知文件/同长度SHA变化/缺文件/路径逃逸均不能通过;未新增真实模型或实际Harness派发。旧全套仍持续收集,尚不记CI全绿。
|
||||
- 冒烟清理复核发现原`portClosed:true`仅由HTTP health不可取和PID退出推导,未直接核监听。增加仅针对本次记录127.0.0.1端口的TCP接收/拒绝观察;缺health但仍监听不能通过,异常/超时保持失败,不结束外来进程。同页watch去重避免把一次浏览器事件记两次。首次直接工具专项8/8(没有独立磁盘原日志),规范最终文本后另用记录器核最终输入;实际Edge/Electron仍待最终Windows run。
|
||||
- 36c6be4f首完整Windows run自然结束:146文件/1211项,1194通过17失败,0取消/跳过/todo,3534259ms;未运行后置桌面冒烟。原报告/原日志/最终MANIFEST不追加,另在`remaining-before-2/first-complete-ci-review.json`只读核1308源码输入与编译文件前后一致。此前12项由旧断言/包外回放输入修正覆盖,余项仍单独追因,不冒称首全套全绿。
|
||||
- 交付报告增长后,Python HTTP测试仍把64KB预览当完整JSON;改用原下载路由并与工件元数据逐字节长度/SHA核对,再保留原Git/工程/测试/文件来源断言,不裁短报告或删账本。虚构`deepseek-owned`前缀不再推出官方厂商;补未知兼容、显式provider和精确官方型号三种来源断言,原原生发送与负向输入核验保留。
|
||||
- Python深路径独立复现原ensurepip及开启UTF-8后的失败;内层原错明确为setuptools包路径WinError206,并非编码根因。该用例只运行stdlib unittest,改为真实`venv --without-pip`,149字符项目环境仍静态deferred、实际解释器3.11.15和1项unittest通过。CI自身带哈希pip/pytest安装不省略,产品解释器识别/身份守卫未改。
|
||||
- 局部首复验32项27通过5失败;复验器误额外设置`AGENT_WORKBENCH_PYTHON_PATH`,覆盖项目`.venv`选择导致其中3项失败,与首完整CI环境不同。保留原命令/输入/日志,移除复验器该覆盖后另轮验证。其它推理旧断言与研究读取失败再现,提醒原20秒等待在局部原用例通过;不把这些替代首并发失败。
|
||||
- 研究原失败JSON逐层复核发现首因是旧16K有限夹具的Evidence字段读取返回“预算无法容纳元数据”,随后本机模型夹具解析失败并触发其重试,才出现第4/5次工具范围断言。完整原状态在`research-first-failure-state-1.json`,这是受控夹具错误,不是外部供应商异常。下一让该研究功能用例采用当前不限/完整历史默认,同时核对开放诊断工具广告与既有执行契约一致;不恢复额度或新增执行拦截。
|
||||
- 原提醒五请求/原20秒等待另做逐请求和状态观察,11.6秒独立用例通过;原并发超时的最后状态未封存,不能补造唯一性能根因。后续验收按五个实际请求的进展观察等待,仍核一次发送/当前证据/resolved/跨会话拒读/重启持久,不把总20秒误当产品运行额度。
|
||||
- 三处测试修正最终规范文本专项23/23,0失败/取消/跳过/todo,102798ms;正确QA解释器环境下,先前误覆盖的三项识别/身份用例均通过。全部产品/测试/脚本与编译输入前后一致,见`remaining-python-profile-fixed/run-windows-mw6jyjbh-5f925ebf/evidence/`。这仅是复用已建依赖/编译的专项,最后干净安装全套仍待;原首轮与局部失败未改写。
|
||||
|
||||
@@ -2,6 +2,7 @@ import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { fixture, launchWorkbench, until, compatibleFixture, call, completion, assertToolPairs } from './helpers.mjs';
|
||||
import { deliveryGit } from './fixtures/delivery-project.mjs';
|
||||
import { captureWorkspaceSource, assertWorkspaceSource } from './fixtures/workspace-project.mjs';
|
||||
@@ -103,7 +104,14 @@ test('an isolated three-module Python Task repairs, checks and delivers a source
|
||||
await host.api(route + '/delivery-jobs', { planId: session.execution.deliveryPlans.at(-1).id });
|
||||
await wait(value => value.execution.deliveries.at(-1).status === 'completed' && !!value.execution.deliveries.at(-1).reportArtifactId);
|
||||
assert.equal(model.requests.length, requests); assert.equal(session.status, 'completed');
|
||||
const job = session.execution.deliveries.at(-1), report = JSON.parse((await host.api('/artifacts/' + job.reportArtifactId + '?session=' + session.id + '&limit=64000')).content);
|
||||
const job = session.execution.deliveries.at(-1), artifactRoute = '/artifacts/' + job.reportArtifactId + '?session=' + session.id;
|
||||
const metadata = await host.api(artifactRoute + '&limit=1');
|
||||
const download = await fetch(host.url + '/api' + artifactRoute + '&download=1', { headers: { 'X-Workbench-Token': host.bootstrap.token, Connection: 'close' } });
|
||||
assert.equal(download.status, 200);
|
||||
const bytes = Buffer.from(await download.arrayBuffer());
|
||||
assert.equal(bytes.length, metadata.bytes);
|
||||
assert.equal(createHash('sha256').update(bytes).digest('hex'), metadata.sha256);
|
||||
const report = JSON.parse(bytes.toString('utf8'));
|
||||
assert.equal(report.status, 'verified_committed'); assert.equal(report.workspace.workspaceId, workspace.id);
|
||||
assert.equal(report.workspace.sourceRoot, f.projectRoot); assert.equal(report.task.id, taskId);
|
||||
assert.deepEqual(job.paths.slice().sort(), Object.keys(pythonFixes).sort());
|
||||
|
||||
@@ -190,7 +190,9 @@ test('real pytest pass, failure, skip, collection errors and teardown errors use
|
||||
test('a real project virtual environment is detected statically and its actual version is recorded only during the approved check', { timeout: 20000 }, async t => {
|
||||
const f = fixture(t); pythonProject(f, onePass);
|
||||
const host = await resolvePythonRuntime(f.project, { executable: process.env.AGENT_WORKBENCH_TEST_PYTHON });
|
||||
execFileSync(host.baseExecutable, ['-I', '-m', 'venv', path.join(f.projectRoot, '.venv')], { windowsHide: true, cwd: f.root, stdio: 'pipe' });
|
||||
// This scenario executes stdlib unittest. It needs a real project venv,
|
||||
// not ensurepip's unrelated setuptools fixtures (overlong Windows paths).
|
||||
execFileSync(host.baseExecutable, ['-I', '-X', 'utf8', '-m', 'venv', '--without-pip', path.join(f.projectRoot, '.venv')], { windowsHide: true, cwd: f.root, stdio: 'pipe' });
|
||||
const runner = new CheckRunner(f.store); f.onCleanup(() => runner.stopAll());
|
||||
const plan = await runner.plan(f.session);
|
||||
assert.equal(plan.checks[0].python.runtime.status, 'deferred'); assert.equal(plan.checks[0].python.runtime.version, null);
|
||||
|
||||
@@ -36,7 +36,12 @@ test('brand and custom-compatible profiles send native reasoning controls per pr
|
||||
assert.equal(body.model, member.model);
|
||||
}
|
||||
assert.equal(actualRequest(plan, cases[0]).thinking, undefined); assert.equal(actualRequest(plan, cases[4]).reasoning_effort, undefined);
|
||||
assert.equal(modelParameterProfile({ ...config, brand: '自定义' }).id, 'deepseek');
|
||||
const unknown = modelParameterProfile({ ...config, brand: '自定义' });
|
||||
assert.equal(unknown.id, 'openai'); assert.equal(unknown.sourceKind, 'compatibility_profile'); assert.equal(unknown.officialSource, null);
|
||||
const explicit = modelParameterProfile({ ...config, providerId: 'deepseek' });
|
||||
assert.equal(explicit.id, 'deepseek'); assert.equal(explicit.officialSource, null, 'a provider declaration does not invent an exact model specification');
|
||||
const official = modelParameterProfile({ ...config, model: 'deepseek-v4-pro' });
|
||||
assert.equal(official.id, 'deepseek'); assert.equal(official.sourceKind, 'official_model_spec'); assert.equal(official.officialSource.provider, 'deepseek');
|
||||
assert.equal(modelParameterProfile({ ...config, parameterProfile: 'openai' }).id, 'openai');
|
||||
for (const [protocol, params, profile] of [
|
||||
['chat-completions', { thinking: { type: 'adaptive' } }], ['chat-completions', { thinking: { type: 'enabled', budget_tokens: 1024 } }],
|
||||
|
||||
Reference in New Issue
Block a user