diff --git a/docs/PLAN-1.20.md b/docs/PLAN-1.20.md index f1271252..73cc6b14 100644 --- a/docs/PLAN-1.20.md +++ b/docs/PLAN-1.20.md @@ -1,6 +1,6 @@ # 1.20 后续方案:工程质量评测与受控自我迭代 -本文件仅记录下一阶段方案。先完成1.19独立源码、全部历史界面、便携、安装、保留状态升级和回顾,再实施。本阶段优先把已有工程工具、模型池和隔离工作区组织成可核对的质量循环;语音、通用电脑操作和更多外部引擎仍分别排期。 +1.19已完成原生续验、完整F封存及独立ACCEPTANCE,最终Git790350e。本阶段在feat/engineering-evals-1.20实施,优先把已有工程工具、模型池和隔离工作区组织成可核对的质量循环;语音、通用电脑操作和更多外部引擎仍分别排期。各切片实际结果见[验收记录](QA-1.20.md)和[回顾](RETROSPECTIVE-1.20.md)。 ## 用户能完成的任务 @@ -19,6 +19,8 @@ ## 复用与权限 +实施选择:每批最多3个样本,按顺序显式推进;每个样本复用现有两个隔离工作区,其中原版不请求模型、禁止提案写入,只执行用户核对过的固定检查;候选仅开放明确的源码文件,原测试/声明/检查输入保持。创建后在任何执行前封存两个检出的同源文件和实际检查命令。界面采用样本队列与原版/候选并列证据,避免汇总分数掩盖失败。比较的是原工程和候选工程,不将原版检查当作另一个模型的回答。 + 复用现有隔离工作区、父工程审查、任务计划、检查、页面验收、请求预算和交付报告,不建立第二套执行内核。评测报告不包含Key或私有推理内容;原上游usage缺失时不补造费用。每个任务仍经过已有文件、命令、工具及设备权限,评测不是新的自动放权入口。 实际模型任务由用户在工作台选择可用连接并明确启动;开发回归使用自有本地HTTP。模型尚不可用时可验收账本、隔离、失败恢复和真实工具链,但模型自主完成率、多模型复核价值与Purge提升继续标为未验证。 diff --git a/docs/QA-1.20.md b/docs/QA-1.20.md new file mode 100644 index 00000000..885cd239 --- /dev/null +++ b/docs/QA-1.20.md @@ -0,0 +1,24 @@ +# 1.20 工程评测验收记录 + +来源790350e,分支feat/engineering-evals-1.20。当前实施中,正式仍为已验收1.19,不宣称1.20发行完成。 + +| 切片 | 状态 | 证据 | +| --- | --- | --- | +| A 固定契约 | 专项通过 | 1–3固定样本、HEAD/同源字节、明确源码/原测试/检查命令、模型和总预算 | +| B 运行账本 | 专项通过 | 原版0模型请求、候选原工具/审批/预算、停止与重启未知、不重放已尝试准备 | +| C 比较 | 专项通过 | 原失败/全部尝试/日志SHA、原用量缺失及人工审批/应用、不可变报告和父审查/Git门控 | +| D 产品流程 | 待实现 | 桌面/390px准备、审查、比较和采纳 | +| E 实际工程 | 待验 | Node/React/Python及自身模块小范围试点 | +| F 完整发行 | 待验 | 完整源码/历史UI/独立源码/便携/安装/升级/原生回看 | + +开发夹具只证明执行链、边界和真实工具结果。用户实际配置模型的自主完成率、视觉/Purge收益继续单列,不从固定响应推断。 + +## A–C 首轮与回顾 + +首轮10/10通过(`.qa/evaluation-a-first.log`)。回顾补充采纳并发门控、计划准备停止等待、原版证据不完整时禁止采纳、来源/模型修订和HTTP权限检查;12项评测与6项原worker共18/18通过(`.qa/evaluation-abc-reviewed.log`),新增最后门控再按相关路径复验。所有检查在自有独立数据/端口执行,真实模型请求0。实际Node检查与本地Git发生在fixture工程,未触碰正式发行/用户模型连接。 + +已实现桌面/390px界面并构建通过,D尚在真实交互验收。首次UI因select的可访问名称包含选项而无法精确定位,补显式aria-label;原失败截图/fixture保留在`.qa/engineering-evals-ui-3cfe9bbe-d29e-4eee-97d2-0d261b622825`,不计通过。 + +最后原版日志/采纳/HTTP3条路径3/3通过(`.qa/evaluation-abc-final-guards.log`)。首次选择器错误后还发现受控textarea的标签名称会随内容变化,现全部关键textarea均有稳定的aria-label,另开收据复验;原第二次失败`.qa/engineering-evals-ui-b08b6c81-cd2b-47c5-85d4-66ecba9fd922`保持。 + +父工程检查/变更集/Git/工作区各操作的活动门控已纳入评测活动,最终相关2路径2/2通过。A-C原日志按原字节封存于[清单](engineering-evals-abc-evidence/manifest.json)。 diff --git a/docs/RETROSPECTIVE-1.20.md b/docs/RETROSPECTIVE-1.20.md new file mode 100644 index 00000000..1ddcd765 --- /dev/null +++ b/docs/RETROSPECTIVE-1.20.md @@ -0,0 +1,19 @@ +# 1.20 逐切片回顾 + +## 起点 + +1.19原生续验及完整F已完成,原Esc中断与中间证据保持。正式原进程/模型连接和全部历史未改。新阶段先固定工程评测的来源、原测试和结果分母,再做候选生成和可审查采纳。 + +## 结构判断 + +比较过全局评分面板、仅在原协作页追加统计、独立样本队列加原版/候选证据三种结构。选第三种:主对象是具体工程改进,用户先核对同源基线/需求/允许文件和固定检查,再处理候选提案和原失败;比单个综合分数更有助于判断能否采纳。沿用现有中文、React、蓝灰/青绿视觉及模态/差异/任务导航;390px按原版、候选的阅读顺序堆叠,不额外引入装饰或依赖。 + +## 边界 + +原版只检查,不消耗模型请求;候选仍使用原模型连接/池、文件范围和审批。批次最多3样本且逐个显式推进。测试集和运行声明不在候选可改范围,执行前后核对原字节;这不等于OS沙箱。正式运行目录/数据和安装指针不作为候选目录。 + +## A–C 回顾 + +复用了原WorkerManager/WorkspaceManager、CheckRunner、ChangeSet及GitDelivery。原版不伪装成一个有修改的worker成果,评测归集只接受候选;普通协作接口不能绕过评测开始或采纳。全部检查尝试、原日志SHA及上游未知用量进入固定报告,缺失证据显示unknown并阻止采纳,不删除原分母。多样本结束后只能采纳一项;先核对原父来源,再切回所属Task形成独立变更集,不能直接覆盖父文件。 + +回顾发现采纳预检含异步窗口,已增加父工程活动锁和持久采纳意图;重启不恢复副作用。计划停止也等待原准备结束。对源码/模型修订、精确文件、原测试、HTTP归属及停止恢复的12项专项和原worker6项合计18/18通过,原10项首轮及后续修正证据分别保留。工程推理成功率、真实模型费用和Purge效果没有据此判为完成。 diff --git a/docs/engineering-evals-abc-evidence/evaluation-a-first.tap b/docs/engineering-evals-abc-evidence/evaluation-a-first.tap new file mode 100644 index 00000000..599a4db8 --- /dev/null +++ b/docs/engineering-evals-abc-evidence/evaluation-a-first.tap @@ -0,0 +1,18 @@ +✔ evaluation contract fixes bounded samples, source paths, checks and aggregate budget (525.0705ms) +✔ same-source workspaces seal real commands before execution and preserve original staged and unstaged bytes (11083.0145ms) +✔ actual original failures and candidate success are both preserved, then adoption requires parent review, checks and Git (40331.2376ms) +✔ candidate cannot shrink tests and changing original test bytes rejects execution before any model request (15626.3462ms) +✔ missing upstream usage remains unknown and does not become a free run or invented cost (25914.3911ms) +✔ missing raw check logs preserve unknown result and original sample denominator (25759.3229ms) +✔ source and model changes never silently replace an existing batch input (2304.4183ms) +✔ sample queue cannot skip or reset attempted samples, and reports freeze further writes (20862.9625ms) +✔ stop cancels only current candidate and preserves partial results instead of replaying a request (16535.6083ms) +✔ restart recovers interrupted evaluations as unknown and keeps completed checks and plan identity (15402.608ms) +ℹ tests 10 +ℹ suites 0 +ℹ pass 10 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 174485.0587 diff --git a/docs/engineering-evals-abc-evidence/evaluation-abc-final-guards.tap b/docs/engineering-evals-abc-evidence/evaluation-abc-final-guards.tap new file mode 100644 index 00000000..13956184 --- /dev/null +++ b/docs/engineering-evals-abc-evidence/evaluation-abc-final-guards.tap @@ -0,0 +1,11 @@ +✔ actual original failures and candidate success are both preserved, then adoption requires parent review, checks and Git (42230.2742ms) +✔ missing raw check logs preserve unknown result and original sample denominator (31180.7936ms) +✔ HTTP evaluations require host token, original fingerprint and session ownership; worker API cannot bypass evaluation flow (15068.7458ms) +ℹ tests 3 +ℹ suites 0 +ℹ pass 3 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 88729.2571 diff --git a/docs/engineering-evals-abc-evidence/evaluation-abc-operation-locks.tap b/docs/engineering-evals-abc-evidence/evaluation-abc-operation-locks.tap new file mode 100644 index 00000000..6f269156 --- /dev/null +++ b/docs/engineering-evals-abc-evidence/evaluation-abc-operation-locks.tap @@ -0,0 +1,10 @@ +✔ actual original failures and candidate success are both preserved, then adoption requires parent review, checks and Git (41949.8076ms) +✔ HTTP evaluations require host token, original fingerprint and session ownership; worker API cannot bypass evaluation flow (12432.7767ms) +ℹ tests 2 +ℹ suites 0 +ℹ pass 2 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 54628.6602 diff --git a/docs/engineering-evals-abc-evidence/evaluation-abc-reviewed.tap b/docs/engineering-evals-abc-evidence/evaluation-abc-reviewed.tap new file mode 100644 index 00000000..74a80a28 --- /dev/null +++ b/docs/engineering-evals-abc-evidence/evaluation-abc-reviewed.tap @@ -0,0 +1,26 @@ +✔ evaluation contract fixes bounded samples, source paths, checks and aggregate budget (879.7691ms) +✔ same-source workspaces seal real commands before execution and preserve original staged and unstaged bytes (14874.6757ms) +✔ actual original failures and candidate success are both preserved, then adoption requires parent review, checks and Git (49362.4978ms) +✔ candidate cannot shrink tests and changing original test bytes rejects execution before any model request (19490.2731ms) +✔ missing upstream usage remains unknown and does not become a free run or invented cost (28035.4922ms) +✔ missing raw check logs preserve unknown result and original sample denominator (27592.2318ms) +✔ source and model changes never silently replace an existing batch input (2375.6565ms) +✔ model revision change before preparing a fixed sample cannot silently use the replacement connection (2210.0987ms) +✔ HTTP evaluations require host token, original fingerprint and session ownership; worker API cannot bypass evaluation flow (12196.9645ms) +✔ sample queue cannot skip or reset attempted samples, and reports freeze further writes (21512.0204ms) +✔ stop cancels only current candidate and preserves partial results instead of replaying a request (17772.6846ms) +✔ restart recovers interrupted evaluations as unknown and keeps completed checks and plan identity (16171.5361ms) +✔ worker scopes and allocated budgets reject aliases, extra permissions and aggregate overflow (770.0996ms) +✔ two sequential Git workspaces freeze a new parent goal, separate owned tasks and preserve all source bytes (13643.8712ms) +✔ real isolated changes, approvals and npm checks produce original owned outputs, not model completion claims (28402.4151ms) +✔ forged worker tools cannot execute terminal or Git even when absent from the advertised tool list (12969.4039ms) +✔ a completion-only model pauses each worker without fabricated changes or checks (14462.927ms) +✔ restart retains frozen scope and marks interrupted parent and workers unknown without sending requests (12239.9837ms) +ℹ tests 18 +ℹ suites 0 +ℹ pass 18 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 212727.8273 diff --git a/docs/engineering-evals-abc-evidence/manifest.json b/docs/engineering-evals-abc-evidence/manifest.json new file mode 100644 index 00000000..b35903fe --- /dev/null +++ b/docs/engineering-evals-abc-evidence/manifest.json @@ -0,0 +1,48 @@ +{ + "stage": "1.20 A-C", + "first": { + "passed": 10, + "failed": 0 + }, + "reviewed": { + "evaluations": 12, + "workers": 6, + "passed": 18, + "failed": 0 + }, + "finalGuardPaths": { + "passed": 3, + "failed": 0 + }, + "finalOperationLockPaths": { + "passed": 2, + "failed": 0 + }, + "realModelRequests": 0, + "files": [ + { + "source": ".qa\\evaluation-a-first.log", + "file": "evaluation-a-first.tap", + "bytes": 1259, + "sha256": "8d5ba79071a3339a5ac2affdcb13a6af88f64fc9c4932c209bca53a0da6b3caf" + }, + { + "source": ".qa\\evaluation-abc-reviewed.log", + "file": "evaluation-abc-reviewed.tap", + "bytes": 2228, + "sha256": "15b798551aa6dee8e5b12d73fc070272d1c67b6ffdbc6445d41fcb63d95ec8fa" + }, + { + "source": ".qa\\evaluation-abc-final-guards.log", + "file": "evaluation-abc-final-guards.tap", + "bytes": 504, + "sha256": "2db5ee72d4439acb97176bedb14985ffe91cc7ecbc361e32a969b08b39a340f9" + }, + { + "source": ".qa\\evaluation-abc-operation-locks.log", + "file": "evaluation-abc-operation-locks.tap", + "bytes": 405, + "sha256": "f689978a90d947c0561dc8f5a891b51b432b547dc206fe302279566957a6c75d" + } + ] +} diff --git a/findings.md b/findings.md index 26a9423b..e00bda89 100644 --- a/findings.md +++ b/findings.md @@ -1,5 +1,7 @@ # 汇总发现 +- 1.20起点:原40基准和三种工程已提供真实工具链证据,但未形成产品级固定评测账本。复用两个隔离worker:原版只做固定检查,候选仅改明确源码文件;实际命令/测试与同源检出在模型执行前封存。源码自身完整仓库含大量证据,超过现有1000文件/16MiB工作区上限,自身试点须使用来源提交绑定的独立模块工程,不能扩大上限或宣称全仓自迭代。首个计划补丁因错误findings锚点整批拒绝、无写入,核对真实标题后修正。 + - 1.19 A:18次样本测量(72份源码/工程×36声明)显示热复用已避免额外72次解析读取,跨实例Node/React仍重解析;Python空间小。原输入/结果存docs/project-cache-a-evidence。新增缓存仅持久派生声明/导入,保留每轮来源读取;旧内存缓存的配置/分支/不完整扫描隔离与来源闭合需要补齐。 - 1.19起点606b5ec工作区干净,正式4318仍为1.18.0。ProjectIndex已有按项目id/root的进程内文件SHA解析复用和同项目pending合并;inspectProject每轮重新读取源码并核对SHA、Git来源。不能把已有复用当新功能,也不能为速度省略本次来源验证。项目记忆和请求证据复用继续单独审查;新增缓存以实际测量为依据。 @@ -761,3 +763,6 @@ Responses已采用typed Items、call_id原配对和同线路加密reasoning续 1.19.0完整本地交付:有界持久解析缓存、逐次两轮来源SHA核对、严格失效、开关/清除及索引/诊断入口。独立源码590/590、27类203组历史UI、新缓存7组及9节点工程、原生窗口7项/受控流程9组/桌面工程16节点、Responses13节点、截图7节点及Node8/8、React9/9、Python10/10通过。随包缓存/伴随端/协议/视觉、同载荷安装7组、正式24页和升级后原生回看通过。原PID36552/Provider、17项目/30全部会话/2614记录/4701事件、全部表每行/导出/模型中心/工件原字节保持,1466工件记录/1376去重文件核对,9229关闭。本轮真实模型请求0;真实视觉工程/Purge收益及新增iPhone实机继续单列待验。 缓存局部收益与来源核对开销并列;原生启动等待是独立体验问题,不改变已通过的全数据/工件验收结论。质量评测应封存不受候选修改的测试输入,未知和人工介入按原分母记录。 +# 1.20 A–C 回顾及 D 进行中 + +固定样本与同源原测试封存已接入原worker/检查/父审查/Git,首轮10/10及评测12+原worker6合计18/18通过。采纳改为持有父活动锁,并在应用前核对原报告;原版证据缺失禁止采纳。D已构建,首次实测暴露select可访问名称问题,补aria-label并另开独立收据复验,原失败保留。Windows rg文件通配参数应使用-g,不能把目录通配符当显式路径。QA状态只查实际存在的workbench.sqlite;临时夹具可能随测试完成删除,不按猜测路径诊断正式数据。 diff --git a/progress.md b/progress.md index 340e5709..ca51ccbc 100644 --- a/progress.md +++ b/progress.md @@ -910,3 +910,6 @@ F冻结前:Responses三工程13节点和父8/8/9/9/10/10、25类191组历史UI 1.19.0完整本地交付:有界持久解析缓存、逐次两轮来源SHA核对、严格失效、开关/清除及索引/诊断入口。独立源码590/590、27类203组历史UI、新缓存7组及9节点工程、原生窗口7项/受控流程9组/桌面工程16节点、Responses13节点、截图7节点及Node8/8、React9/9、Python10/10通过。随包缓存/伴随端/协议/视觉、同载荷安装7组、正式24页和升级后原生回看通过。原PID36552/Provider、17项目/30全部会话/2614记录/4701事件、全部表每行/导出/模型中心/工件原字节保持,1466工件记录/1376去重文件核对,9229关闭。本轮真实模型请求0;真实视觉工程/Purge收益及新增iPhone实机继续单列待验。 复用既有stage/热升级/activate原收据,无重复副作用;原Esc中断和中间提交bc276b8保持。重新打开原安装入口、实际观察1.19后完成完整F封存,下一1.20按方案推进。 +# 1.20 A–C 实现与首轮回顾 + +接续1.19最终Git790350e。新固定评测/同源保护、原版检查、候选执行、原证据对照与父审查/Git接口已实现;首轮10/10、回顾12评测+6原worker共18/18通过。正式1.19服务与Provider未变。D界面构建通过,正在独立Edge1440/390px实际交互;首轮aria名称错误与原截图已留存,修正后复验中。 diff --git a/src/core/check-runner.mjs b/src/core/check-runner.mjs index c7f4ea68..caab28e2 100644 --- a/src/core/check-runner.mjs +++ b/src/core/check-runner.mjs @@ -13,6 +13,7 @@ import { assertMcpResolved } from './tools/mcp-contract.mjs'; import { assertWorkersResolved, workerForSession } from '../shared/contracts/workers.mjs'; import { bindPythonChecks, preparePythonInvocation, pythonEnvironment, readPythonEvidence, pythonResultBinding } from './python-runtime.mjs'; import { projectEnvironment } from './process-environment.mjs'; +import { verifyEvaluationSources } from './evaluation/provenance.mjs'; const at = () => new Date().toISOString(); const conflict = message => Object.assign(new Error(message), { status: 409 }); @@ -65,6 +66,7 @@ export class CheckRunner { projectActive(projectId) { return this.projects.has(projectId); } async plan(session, names) { + await verifyEvaluationSources(this.store, session, names); assertWorkersResolved(this.store, session.projectId); if (workerForSession(this.store, session)?.status === 'completed') throw conflict('子成果已经封存,请读取原检查记录;重新执行需要新的受限任务。'); assertChangeSetsResolved(this.store, session.projectId); @@ -112,6 +114,7 @@ export class CheckRunner { assertChangeSetsResolved(this.store, session.projectId, { includePrepared: true }); assertWorkspacesResolved(this.store, session.projectId, { includePrepared: true }); const plan = this.planFor(session, planId); + await verifyEvaluationSources(this.store, session, plan.checks.map(check => check.name)); if (this.projectActive(session.projectId) || this.projectBusy(session.projectId, source === 'agent' ? session.id : null)) throw conflict('同一项目正在执行任务或检查,请等待结束后再运行。'); if (session.changes.some(change => ['pending', 'conflict'].includes(change.status))) throw conflict('先应用或撤回本任务的待审查改动,再检查实际工程。'); const controller = new AbortController(), run = this.store.owned('run', session.runId, session.id); @@ -135,6 +138,7 @@ export class CheckRunner { const task = this.store.core.task.get(job.taskId); let unrecordedProcess = null; try { + await verifyEvaluationSources(this.store, session, plan.checks.map(check => check.name)); await verifyWorkspaceProject(this.store, this.store.project(session.projectId)); const before = await inspectProject(this.store.project(session.projectId)); signal.throwIfAborted(); @@ -172,6 +176,7 @@ export class CheckRunner { if (after.sourceFingerprint !== job.sourceFingerprint) { job.stale = true; break; } } const final = await inspectProject(this.store.project(session.projectId)); + await verifyEvaluationSources(this.store, session, plan.checks.map(check => check.name)); job.stale ||= final.sourceFingerprint !== job.sourceFingerprint; if (job.results.some(result => result.status === 'unknown')) { job.status = 'unknown'; job.requiresVerification = true; } else if (signal.aborted || job.results.some(result => result.status === 'cancelled')) job.status = 'cancelled'; diff --git a/src/core/evaluation/evaluation-manager.mjs b/src/core/evaluation/evaluation-manager.mjs new file mode 100644 index 00000000..0c3ff3ad --- /dev/null +++ b/src/core/evaluation/evaluation-manager.mjs @@ -0,0 +1,238 @@ +import { randomUUID } from 'node:crypto'; +import { modelSnapshot } from '../../shared/contracts/execution.mjs'; +import { workerUsage, COLLABORATION_CLOSED } from '../../shared/contracts/workers.mjs'; +import { evaluationInput, evaluationSeal, evaluationSampleSeal, evaluationHash, evaluationSummary, evaluationConflict } from '../../shared/contracts/evaluations.mjs'; +import { evaluationRecord, preparedSample, verifyEvaluationSources } from './provenance.mjs'; +import { createCheckList } from '../project-inspector.mjs'; + +const at = () => new Date().toISOString(); +const identity = source => Object.fromEntries(['root','head','reference','branch','gitDir','commonDir','indexSha256','statusSha256','sourceFingerprint','treeSha256','configuration','filterDrivers'].map(key => [key, source[key]])); +const terminal = new Set(['completed','retained']); +const commandView = checks => checks.map(({ name, kind, script, command, framework, lifecycle, python }) => ({ name, kind, script, command, framework, lifecycle, ...(python ? { python } : {}) })); + +export class EvaluationManager { + constructor(store, runtime) { this.store = store; this.runtime = runtime; this.active = new Map(); } + batches(session) { return [...this.store.core.evaluation.values()].filter(batch => batch.sessionId === session.id); } + isActive(sessionId) { return [...this.active.values()].some(value => value.sessionId === sessionId); } + projectActive(projectId, excludeId = null) { return [...this.active.entries()].some(([id,value]) => id !== excludeId && value.projectId === projectId); } + save(session,batch,phase) { batch.updatedAt = at(); return this.store.putRecord('evaluation',session,batch,'evaluation.'+phase); } + own(session,id,fingerprint = null) { return evaluationRecord(this.store,session,id,fingerprint); } + view(session) { + return { batches: this.batches(session).map(batch => ({ ...batch, summary: evaluationSummary(batch), samples: batch.samples.map(sample => ({ ...sample, + workers: sample.groupId ? this.runtime.workerManager().view(session).groups.find(group => group.id === sample.groupId)?.workers ?? [] : [] })) })) }; + } + artifact(session,batch,content,kind) { + const artifact = this.store.artifactStore.write(session,this.store.owned('run',batch.runId,session.id),content,{kind}); + this.store.core.artifact.set(artifact.id,artifact); return artifact; + } + async plan(session,input) { + this.runtime.workerManager().assertIdle(session); + if (this.batches(session).some(batch => !terminal.has(batch.status)) || [...this.store.core.evaluation.values()].some(batch => batch.projectId === session.projectId && !terminal.has(batch.status))) throw evaluationConflict('工程已有未收尾评测,请先处理原样本或保留现场结束。'); + if ([...this.store.core.collaboration.values()].some(group => group.projectId === session.projectId && !COLLABORATION_CLOSED.has(group.status))) throw evaluationConflict('工程已有协作,请先完成原协作。'); + const initial = structuredClone(this.store.config), config = this.runtime.provider.prepareConfig?.(initial,this.store) ?? initial; + const definition = evaluationInput(input,config), id = randomUUID(); + const active={sessionId:session.id,projectId:session.projectId,controller:new AbortController()}; + active.promise=new Promise(resolve=>{active.settle=resolve;});this.active.set(id,active); + try { + const source = await this.runtime.workerManager().workspaces.snapshot(this.store.project(session.projectId)); + active.controller.signal.throwIfAborted(); + const run = this.store.beginRun(session,definition.title,false,config,{forceNewTask:true}); + session.status = 'paused'; this.store.finishRun(session,run,'evaluation_plan'); + const batch = {schemaVersion:1,id,projectId:session.projectId,sessionId:session.id,taskId:session.taskId,runId:run.id, + ...definition,source:identity(source),model:modelSnapshot(config),samples:definition.samples.map(sample => ({...sample,id:randomUUID(),status:'pending',report:null})), + status:'planned',createdAt:at(),updatedAt:at(),startedAt:null,deadline:null,finishedAt:null,adoption:null, + message:'固定需求、来源、验收条件、检查选择与总预算已封存。尚未创建工作区或请求模型。'}; + batch.fingerprint = evaluationHash(evaluationSeal(batch)); + const artifact = this.artifact(session,batch,evaluationSeal(batch),'evaluation-plan'); + batch.contractArtifactId = artifact.id; batch.contractSha256 = artifact.sha256; + this.store.core.task.get(batch.taskId).origin = 'engineering-evaluation'; + this.store.message(session,'user',definition.title); this.save(session,batch,'planned'); return batch; + } finally { this.active.delete(id); active.settle(); } + } + sample(batch,id) { const sample = batch.samples.find(sample => sample.id === id); if (!sample) throw evaluationConflict('样本不属于当前评测。'); return sample; } + group(session,batch,sample) { return this.runtime.workerManager().own(session,sample.groupId,null,{current:false}); } + launch(session,batch,sample,status,execute,{allowCompleted=false}={}) { + if (this.active.has(batch.id) || this.runtime.projectHasActivity(session.projectId)) throw evaluationConflict('工程已有执行,请等待完成。'); + if ((terminal.has(batch.status) && !(allowCompleted&&batch.status==='completed')) || batch.adoption) throw evaluationConflict('本批已结束或进入采纳审查,不能再次执行样本。'); + const controller = new AbortController(), active = {sessionId:session.id,projectId:session.projectId,controller,promise:null}; + this.active.set(batch.id,active); sample.previousStatus=sample.status; batch.status = status; sample.status = status; this.save(session,batch,status); + active.promise = (async () => { + try { return await execute(controller.signal); } + catch (error) { + const group = sample.groupId && this.store.core.collaboration.get(sample.groupId); + sample.status = group?.requiresVerification ? 'unknown' : controller.signal.aborted ? 'stopped' : 'failed'; + batch.status = sample.status; sample.error = this.runtime.provider.redact(error.message ?? String(error)); batch.message = sample.error; + return batch; + } finally { this.save(session,batch,batch.status); this.active.delete(batch.id); } + })(); + active.promise.catch(()=>{}); return {batch,promise:active.promise}; + } + selectParent(session,group) { + if (session.changes.some(change => ['pending','conflict'].includes(change.status) && change.taskId !== group.taskId)) throw evaluationConflict('父工程有其他任务的待审查改动,请先处理。'); + session.taskId = group.taskId; session.runId = group.runId; session.status = 'paused'; + this.store.save(session,{type:'evaluation.select_task',detail:{groupId:group.id}}); + } + prepare(session,id,fingerprint,sampleId) { + const batch = this.own(session,id,fingerprint), sample = this.sample(batch,sampleId), index = batch.samples.indexOf(sample); + if (sample.status !== 'pending' || batch.samples.slice(0,index).some(value => !value.report)) throw evaluationConflict('请按固定队列顺序准备;已尝试的样本不会自动重建。'); + return this.launch(session,batch,sample,'preparing',async signal => { + const manager = this.runtime.workerManager(), source = await manager.workspaces.snapshot(this.store.project(session.projectId)); + if (JSON.stringify(identity(source)) !== JSON.stringify(batch.source)) throw evaluationConflict('来源工程已变化,请保留原批结果后重新规划。'); + const initial = structuredClone(this.store.config), config = this.runtime.provider.prepareConfig?.(initial,this.store) ?? initial; + if (JSON.stringify(modelSnapshot(config)) !== JSON.stringify(batch.model)) throw evaluationConflict('模型连接或池修订已变化,请重新封存评测。'); + const part = key => Math.floor(batch.budget[key]/batch.samples.length); + const budget = {maxTokens:Math.min(part('maxTokens'),config.maxRunTokens),maxSteps:part('maxSteps'),maxToolCalls:part('maxToolCalls'),maxDurationMs:batch.budget.maxDurationMs}; + const original = {maxTokens:500,maxSteps:1,maxToolCalls:1}, candidate = Object.fromEntries(Object.keys(original).map(key=>[key,budget[key]-original[key]])); + signal.throwIfAborted(); + const group = await manager.plan(session,{goal:sample.goal+'\n验收条件:'+sample.criteria,budget,children:[ + {label:'原版 · '+sample.label,goal:'只执行原固定检查,不修改工程,不请求模型。',paths:sample.paths,budget:original}, + {label:'候选 · '+sample.label,goal:sample.goal+'\n验收条件:'+sample.criteria+'\n必须执行固定检查:'+sample.checks.join('、')+'。不得修改测试或运行声明。',paths:sample.paths,budget:candidate}, + ]},{evaluation:{batchId:batch.id,sampleId:sample.id}}); + sample.groupId = group.id; this.save(session,batch,'workspace_intent'); signal.throwIfAborted(); + await manager.prepare(session,group.id,group.fingerprint).promise; + if (group.status !== 'ready') throw evaluationConflict(group.summary); + const inventories = []; + for (const child of group.children) inventories.push(await manager.inventory(this.store.project(child.projectId))); + if (JSON.stringify(inventories[0].files) !== JSON.stringify(inventories[1].files)) throw evaluationConflict('原版与候选检出不是同一字节来源。'); + if (sample.paths.some(path=>!inventories[0].files.some(file=>file.path===path))) throw evaluationConflict('允许修改范围必须是来源提交中已有的明确源码文件。'); + const commands = inventories.map(value=>commandView(createCheckList(value.inspection,sample.checks))); + if (!commands[0].some(check=>check.kind==='test') || JSON.stringify(commands[0])!==JSON.stringify(commands[1])) throw evaluationConflict('样本必须包含同源的真实测试检查。'); + sample.files = inventories[0].files; sample.commands = commands[0]; sample.children = group.children.map(child=>({id:child.id,role:child.evaluation.role,projectId:child.projectId,sessionId:child.sessionId,taskId:child.taskId})); + const artifact = this.artifact(session,batch,evaluationSampleSeal(batch,sample),'evaluation-sample'); + sample.contractArtifactId = artifact.id; sample.contractSha256 = artifact.sha256; + preparedSample(this.store,batch,sample); signal.throwIfAborted(); + sample.status = 'ready'; batch.status = 'ready'; batch.message = '同源检出、原文件和实际检查命令已封存。请核对后运行原版。'; return batch; + }); + } + remaining(batch) { + batch.startedAt ??= at(); batch.deadline ??= Date.parse(batch.startedAt)+batch.budget.maxDurationMs; + const remaining = batch.deadline-Date.now(); if (remaining<=0) throw evaluationConflict('本批总时限已用完,原记录保留;可封存已有结果或重新规划。'); return remaining; + } + baseline(session,id,fingerprint,sampleId) { + const batch=this.own(session,id,fingerprint),sample=this.sample(batch,sampleId); + if (sample.status!=='ready' || sample.baselineAttemptedAt) throw evaluationConflict('原版尚未就绪或已经尝试,不重放检查。'); + preparedSample(this.store,batch,sample); + return this.launch(session,batch,sample,'baseline_running',async signal=>{ + const remaining=this.remaining(batch), group=this.group(session,batch,sample), child=group.children.find(child=>child.evaluation.role==='original'); + sample.baselineAttemptedAt=at();this.save(session,batch,'baseline_intent'); + const owner=this.store.session(child.sessionId), runner=this.runtime.checkRunner(), plan=await runner.plan(owner,sample.checks); + const combined=AbortSignal.any([signal,AbortSignal.timeout(remaining)]), started=await runner.start(owner,plan.id,{signal:combined,source:'user'}); + sample.baselineJobId=started.job.id;this.save(session,batch,'baseline_started');await started.promise; + const worker=this.store.core.worker.get(child.id);worker.status=started.job.requiresVerification?'unknown':'baseline_complete';worker.finishedAt=at();worker.summary=started.job.summary;worker.requiresVerification=!!started.job.requiresVerification; + this.store.putRecord('worker',owner,worker,'evaluation.original_finished'); + sample.status=started.job.requiresVerification?'unknown':'baseline_complete';batch.status=sample.status;batch.message='原版结果已保存:'+started.job.summary;return batch; + }); + } + start(session,id,fingerprint,sampleId) { + const batch=this.own(session,id,fingerprint),sample=this.sample(batch,sampleId); + if (!['baseline_complete','awaiting_review','paused','failed','stopped'].includes(sample.status)||!sample.baselineJobId||sample.report) throw evaluationConflict('先完成原版固定检查;已封存样本不重放。'); + preparedSample(this.store,batch,sample); + return this.launch(session,batch,sample,'candidate_running',async signal=>{ + const remaining=this.remaining(batch),group=this.group(session,batch,sample),child=group.children.find(child=>child.evaluation.role==='candidate'); + this.selectParent(session,group);await verifyEvaluationSources(this.store,this.store.session(child.sessionId));signal.throwIfAborted(); + group.startedAt ??= at();group.deadline ??= Date.now()+remaining; group.deadline=Math.min(group.deadline,batch.deadline); + const started=this.runtime.workerManager().start(session,group.id,group.fingerprint,child.id); + const stop=()=>{void this.runtime.workerManager().stopParent(session.id);};signal.addEventListener('abort',stop,{once:true}); + try{await started.promise;}finally{signal.removeEventListener('abort',stop);} + const worker=this.store.core.worker.get(child.id);sample.status=worker.requiresVerification?'unknown':signal.aborted?'stopped':worker.status==='completed'?'candidate_complete':worker.status; + batch.status=sample.status;batch.message=worker.summary;return batch; + }); + } + roleEvidence(batch,sample,role) { + const child=sample.children?.find(child=>child.role===role); + if (!child) return {outcome:sample.status==='stopped'?'stopped':'unknown',attempts:[],usage:null,reason:sample.error??'尚未完成同源准备。'}; + const owner=this.store.session(child.sessionId),worker=this.store.core.worker.get(child.id),task=this.store.core.task.get(child.taskId); + const attempts=[...this.store.core.check_job.values()].filter(job=>job.taskId===child.taskId).map(job=>{ + const plan=this.runtime.checkRunner().planFor(owner,job.planId); + if(JSON.stringify(plan.checks.map(check=>check.name))!==JSON.stringify(sample.checks))throw evaluationConflict('历史检查没有执行原完整选择。'); + const results=job.results.map(result=>{ + const artifact=this.store.owned('artifact',result.logArtifactId,owner.id);this.store.artifactStore.read(artifact); + if(artifact.kind!=='check-log'||artifact.taskId!==child.taskId||artifact.runId!==job.runId)throw evaluationConflict('原检查日志归属不一致。'); + const extra={};for(const key of ['metricsArtifactId','inputArtifactId'])if(result[key]){const file=this.store.owned('artifact',result[key],owner.id);this.store.artifactStore.read(file);extra[key]={id:file.id,sha256:file.sha256};} + return {name:result.name,command:result.command,status:result.status,exitCode:result.exitCode,durationMs:result.durationMs,tests:result.tests??null,logArtifactId:artifact.id,logSha256:artifact.sha256,...extra}; + }); + return {id:job.id,planId:plan.id,status:job.status,stale:!!job.stale,requiresVerification:!!job.requiresVerification,startedAt:job.startedAt,finishedAt:job.finishedAt,sourceFingerprint:job.sourceFingerprint,summary:job.summary,results}; + }); + const latest=attempts.at(-1), raw=role==='original'?latest?.status:worker?.status,lastRun=this.store.core.run.get(worker?.currentRunId); + const outcome=worker?.requiresVerification||latest?.requiresVerification||latest?.stale?'unknown':role==='candidate'&&raw==='completed'?'passed':role==='original'&&raw==='passed'?'passed':raw==='failed'?'failed':['stopped','cancelled'].includes(raw)||role==='candidate'&&lastRun?.stopReason==='cancelled'?'stopped':'unknown'; + const usage=workerUsage(this.store,worker); + return {workerId:child.id,projectId:child.projectId,sessionId:child.sessionId,taskId:child.taskId,outcome,workerStatus:worker?.status,reason:worker?.summary??sample.error??null, + startedAt:role==='original'?sample.baselineAttemptedAt:worker?.startedAt,finishedAt:worker?.finishedAt??latest?.finishedAt??null,attempts, + usage:role==='original'?{requests:0,totalTokens:0,reported:true,reason:'原版没有请求模型。'}:{...usage,reported:usage.requests>0&&usage.unknownRequests===0,cost:null,costReason:'上游未报告可核对费用。'}, + approvals:owner.approvals.filter(approval=>approval.taskId===task.id||owner.events.some(event=>event.taskId===task.id&&event.approvalId===approval.id)).map(approval=>({id:approval.id,status:approval.status,tool:approval.tool,decidedAt:approval.decidedAt??null})), + changes:owner.changes.filter(change=>change.taskId===child.taskId).map(change=>({id:change.id,path:change.path,status:change.status,sha256:change.sha256,baseSha256:change.baseSha256,changeSetId:change.changeSetId??null})), + requests:[...this.store.core.context_plan.values()].filter(plan=>plan.taskId===child.taskId).map(plan=>({id:plan.id,runId:plan.runId,artifactId:plan.artifactId??null})), + result:worker?.result??null}; + } + seal(session,id,fingerprint,sampleId) { + const batch=this.own(session,id,fingerprint),sample=this.sample(batch,sampleId); + if(sample.report||sample.status==='pending')throw evaluationConflict('样本尚未尝试或已封存,不能改写原报告。'); + return this.launch(session,batch,sample,'collecting',async()=>{ + let inputVerified=false,inputError=null; + try{preparedSample(this.store,batch,sample);for(const child of sample.children)await verifyEvaluationSources(this.store,this.store.session(child.sessionId));inputVerified=true;}catch(error){inputError=this.runtime.provider.redact(error.message);} + const evidence=role=>{try{return this.roleEvidence(batch,sample,role);}catch(error){const child=sample.children?.find(child=>child.role===role);return {outcome:'unknown',evidenceError:this.runtime.provider.redact(error.message),attempts:[...this.store.core.check_job.values()].filter(job=>job.taskId===child?.taskId).map(job=>({id:job.id,status:job.status,results:job.results.map(result=>({name:result.name,logArtifactId:result.logArtifactId}))})),usage:null};}}; + const original=evidence('original'),candidate=evidence('candidate'); + if(!inputVerified){original.outcome='unknown';candidate.outcome='unknown';} + const report={schemaVersion:1,batchId:batch.id,batchFingerprint:batch.fingerprint,sampleId:sample.id,sampleSha256:sample.contractSha256??null, + label:sample.label,goal:sample.goal,criteria:sample.criteria,source:batch.source,model:batch.model,evidenceMode:batch.evidenceMode,budget:batch.budget, + inputVerified,inputError,commands:sample.commands??[],original,candidate,summary:{original:original.outcome,candidate:candidate.outcome}, + inferenceValidated:false,note:batch.evidenceMode==='local-fixture'?'本地固定响应仅证明执行链与真实工程检查,不证明模型自主完成率。':'当前样本结果仅适用于封存需求和工程;不外推整体模型能力。',sealedAt:at()}; + const artifact=this.artifact(session,batch,report,'evaluation-report');sample.report={artifactId:artifact.id,sha256:artifact.sha256,summary:report.summary};sample.status='sealed'; + const group=sample.groupId&&this.store.core.collaboration.get(sample.groupId); + if(group&&!group.requiresVerification){group.status='resolved_external';group.finishedAt=at();this.runtime.workerManager().save(session,group,'evaluation_sealed');} + batch.status=group?.requiresVerification?'unknown':batch.samples.every(sample=>sample.report)?'completed':'between_samples'; + batch.message='原样本分母及全部尝试已封存。'+(inputError??'可查看对照报告,或继续下一个固定样本。');if(batch.status==='completed')batch.finishedAt=at();return batch; + }); + } + readReport(session,batch,sample) { + this.own(session,batch.id);if(!sample.report)throw evaluationConflict('样本尚未封存报告。'); + const artifact=this.store.owned('artifact',sample.report.artifactId,session.id); + if(artifact.kind!=='evaluation-report'||artifact.taskId!==batch.taskId||artifact.runId!==batch.runId||artifact.sha256!==sample.report.sha256)throw evaluationConflict('报告原工件身份不一致。'); + const report=JSON.parse(this.store.artifactStore.read(artifact)); + if(report.batchId!==batch.id||report.sampleId!==sample.id||report.batchFingerprint!==batch.fingerprint||JSON.stringify(report.summary)!==JSON.stringify(sample.report.summary))throw evaluationConflict('对照报告与原样本不一致。');return report; + } + adoptionFor(session,group) { + const batch=this.own(session,group.evaluation.batchId),sample=this.sample(batch,group.evaluation.sampleId),report=this.readReport(session,batch,sample); + if(!batch.adoption||batch.adoption.sampleId!==sample.id||batch.adoption.reportSha256!==sample.report.sha256||!report.inputVerified||report.summary.candidate!=='passed'||!['passed','failed'].includes(report.summary.original))throw evaluationConflict('候选采纳与原通过报告不一致。'); + return {batch,sample,report}; + } + adopt(session,id,fingerprint,sampleId) { + const batch=this.own(session,id,fingerprint),sample=this.sample(batch,sampleId),report=this.readReport(session,batch,sample); + if(this.runtime.projectHasActivity(session.projectId)||batch.adoption||batch.samples.some(sample=>!sample.report)||!report.inputVerified||report.summary.candidate!=='passed'||!['passed','failed'].includes(report.summary.original))throw evaluationConflict('请先封存全部样本;只有原版结果完整、原输入可核对且通过的候选才能采纳。'); + const manager=this.runtime.workerManager(),group=this.group(session,batch,sample); + if(group.requiresVerification)throw evaluationConflict('原执行仍待核对。'); + return this.launch(session,batch,sample,'adopting',async signal=>{ + await manager.sourceFor(group);await manager.verifiedResult(group,group.children.find(child=>child.evaluation.role==='candidate'));signal.throwIfAborted(); + this.selectParent(session,group);batch.adoption={sampleId:sample.id,groupId:group.id,reportSha256:sample.report.sha256,status:'preparing'};batch.status='adopting';this.save(session,batch,'adopting'); + try{ + group.status='ready_to_collect';manager.save(session,group,'evaluation_adoption'); + await manager.collect(session,group.id,group.fingerprint,{evaluation:true}).promise; + if(group.status!=='collected')throw evaluationConflict(group.summary); + await manager.prepareMerge(session,group.id,group.fingerprint).promise; + if(group.status!=='merge_prepared')throw evaluationConflict(group.summary); + batch.adoption.status='awaiting_review';batch.status='awaiting_delivery';batch.message='候选已形成父工程待审查变更集。请审查应用、检查父工程并完成本地 Git,再核对交付。'; + }catch(error){batch.status='unknown';batch.message=this.runtime.provider.redact(error.message);} + sample.status='sealed';this.save(session,batch,batch.status);return batch; + },{allowCompleted:true}); + } + async finalize(session,id,fingerprint) { + const batch=this.own(session,id,fingerprint);if(batch.status!=='awaiting_delivery')throw evaluationConflict('尚未进入父工程交付。'); + const group=this.store.core.collaboration.get(batch.adoption.groupId);this.adoptionFor(session,group); + await this.runtime.workerManager().finalize(session,group.id,group.fingerprint); + batch.adoption.status='completed';batch.adoption.delivery=group.delivery;batch.status='completed';batch.finishedAt=at();batch.message='原候选已审查应用、父工程重新检查通过并完成本地 Git。';this.save(session,batch,'delivered');return batch; + } + retain(session,id,fingerprint) { + const batch=this.own(session,id,fingerprint); + if(batch.adoption?.status==='completed')throw evaluationConflict('交付已完成,不能改写原交付状态。'); + if(this.runtime.projectHasActivity(session.projectId))throw evaluationConflict('请先停止所属执行。'); + for(const sample of batch.samples){const group=sample.groupId&&this.store.core.collaboration.get(sample.groupId);if(group&&!COLLABORATION_CLOSED.has(group.status)){this.selectParent(session,group);this.runtime.workerManager().discard(session,group.id,group.fingerprint);}} + batch.status='retained';batch.finishedAt=at();batch.message='已保留全部工作区、原结果和未执行样本,结束本批;没有声明交付成功。';this.save(session,batch,'retained');return batch; + } + async stopParent(sessionId) { + const active=[...this.active.values()].find(value=>value.sessionId===sessionId);if(!active)return; + active.controller.abort(new Error('用户停止本批执行。'));await this.runtime.workers?.stopParent(sessionId); + await Promise.allSettled([...this.store.core.evaluation.values()].filter(batch=>batch.sessionId===sessionId).flatMap(batch=>batch.samples.flatMap(sample=>sample.children??[])).map(child=>this.runtime.checks?.stop(child.sessionId))); + await active.promise; + } + async stopAll(){await Promise.all([...new Set([...this.active.values()].map(value=>value.sessionId))].map(id=>this.stopParent(id)));} +} diff --git a/src/core/evaluation/provenance.mjs b/src/core/evaluation/provenance.mjs new file mode 100644 index 00000000..9122c418 --- /dev/null +++ b/src/core/evaluation/provenance.mjs @@ -0,0 +1,44 @@ +import fs from 'node:fs'; +import { workerForSession, workerHash } from '../../shared/contracts/workers.mjs'; +import { evaluationSeal, evaluationSampleSeal, evaluationHash, evaluationConflict } from '../../shared/contracts/evaluations.mjs'; +import { inspectProject, verifyIndexSnapshot } from '../project-inspector.mjs'; +import { safePath } from '../../server/files.mjs'; + +export function evaluationRecord(store, session, id, fingerprint = null) { + const batch = store.owned('evaluation', id, session.id); + if (batch.projectId !== session.projectId || evaluationHash(evaluationSeal(batch)) !== batch.fingerprint || fingerprint !== null && fingerprint !== batch.fingerprint) throw evaluationConflict('评测来源、样本或预算与原封存契约不一致。'); + const artifact = store.owned('artifact', batch.contractArtifactId, session.id); + if (artifact.kind !== 'evaluation-plan' || artifact.taskId !== batch.taskId || artifact.runId !== batch.runId || artifact.sha256 !== batch.contractSha256 + || store.artifactStore.read(artifact) !== JSON.stringify(evaluationSeal(batch))) throw evaluationConflict('评测计划原工件无法核对。'); + return batch; +} +export function preparedSample(store, batch, sample) { + if (!sample.contractArtifactId) throw evaluationConflict('尚未封存同源检出与实际检查命令,不能执行。'); + const artifact = store.owned('artifact', sample.contractArtifactId, batch.sessionId); + const expected = evaluationSampleSeal(batch, sample); + if (artifact.kind !== 'evaluation-sample' || artifact.taskId !== batch.taskId || artifact.runId !== batch.runId || artifact.sha256 !== sample.contractSha256 || store.artifactStore.read(artifact) !== JSON.stringify(expected)) throw evaluationConflict('样本来源、固定检查或原检出字节已变化。'); + return expected; +} +export async function verifyEvaluationSources(store, session, names = null) { + const worker = workerForSession(store, session); + if (!worker?.evaluation) return null; + const owner = store.session(worker.parentSessionId), batch = evaluationRecord(store, owner, worker.evaluation.batchId); + const sample = batch.samples.find(value => value.id === worker.evaluation.sampleId); + if (!sample || sample.groupId !== worker.collaborationId || !sample.children?.some(value => value.id === worker.id && value.role === worker.evaluation.role)) throw evaluationConflict('子工程不属于原评测样本。'); + preparedSample(store, batch, sample); + if (names !== null && (sample.report || ['completed','retained'].includes(batch.status))) throw evaluationConflict('样本已经封存,不能重新执行检查。'); + if (names !== null && JSON.stringify(names) !== JSON.stringify(sample.checks)) throw evaluationConflict('评测必须执行原封存的完整检查选择,不能缩小测试集。'); + const project = store.project(session.projectId), inspection = await inspectProject(project, { includeFiles: true, strictSnapshot: true }); + if (!inspection.scope.complete) throw evaluationConflict('评测工程来源范围不完整。'); + const paths = new Set(sample.paths.map(value => value.toLowerCase())); + const protectedFiles = sample.files.filter(file => worker.evaluation.role === 'original' || !paths.has(file.path.toLowerCase())); + const current = inspection.files.filter(file => worker.evaluation.role === 'original' || !paths.has(file.path.toLowerCase())); + const view = files => files.map(file => ({ path: file.path, sha256: file.sha256, bytes: file.bytes })).sort((a,b)=>a.path.localeCompare(b.path)); + if (JSON.stringify(view(current)) !== JSON.stringify(view(protectedFiles))) throw evaluationConflict('原测试、运行声明或非候选文件已变化;保留失败,不能据此比较。'); + for (const file of protectedFiles) { + const target = safePath(project,file.path), stat = fs.lstatSync(target); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size !== file.bytes || workerHash(fs.readFileSync(target)) !== file.sha256) throw evaluationConflict('固定评测输入无法核对:'+file.path); + } + await verifyIndexSnapshot(project,inspection); + return { batch, sample, worker, inspection }; +} diff --git a/src/core/tasks/worker-manager.mjs b/src/core/tasks/worker-manager.mjs index 760f80cc..4198cebb 100644 --- a/src/core/tasks/worker-manager.mjs +++ b/src/core/tasks/worker-manager.mjs @@ -17,6 +17,7 @@ import { repairToolPairs } from '../../server/store.mjs'; import { readOwnedProcessTree } from '../owned-process-tree.mjs'; import { assertControlledInstallation } from '../engines/controlled-installation.mjs'; import { verifyEngineInput, originalEngineRun, assertEngineWorkerResult } from '../../shared/contracts/engine-execution.mjs'; +import { verifyEvaluationSources } from '../evaluation/provenance.mjs'; const at = () => new Date().toISOString(); const sourceIdentity = source => Object.fromEntries(['root', 'head', 'reference', 'branch', 'gitDir', 'commonDir', 'indexSha256', @@ -29,7 +30,7 @@ export class WorkerManager { constructor(store, runtime, { faultInjector = null } = {}) { this.store = store; this.runtime = runtime; this.faultInjector = faultInjector; this.active = new Map(); this.preparingGroupId = null; - this.workspaces = new WorkspaceManager(store, { projectBusy: projectId => runtime.projectHasActivity(projectId, null, { excludeGroupId: this.preparingGroupId }) }); + this.workspaces = new WorkspaceManager(store, { projectBusy: projectId => runtime.projectHasActivity(projectId, null, { excludeGroupId: this.preparingGroupId, excludeEvaluationId: this.store.core.collaboration.get(this.preparingGroupId)?.evaluation?.batchId }) }); } groups(session) { return [...this.store.core.collaboration.values()].filter(group => group.sessionId === session.id); } view(session) { @@ -47,8 +48,8 @@ export class WorkerManager { || [...this.store.core.collaboration.values()].some(group => group.id !== excludeGroupId && group.projectId === projectId && !COLLABORATION_CLOSED.has(group.status) && group.children.some(child => this.childActive(child.sessionId))); } - assertIdle(session) { - if (this.runtime.projectHasActivity(session.projectId)) throw workerConflict('项目已有执行或协作操作,请等待结束。'); + assertIdle(session, excludeEvaluationId = null) { + if (this.runtime.projectHasActivity(session.projectId, null, { excludeEvaluationId })) throw workerConflict('项目已有执行或协作操作,请等待结束。'); assertWorkersResolved(this.store, session.projectId); assertMcpResolved(this.store, session.projectId); assertChangeSetsResolved(this.store, session.projectId, { includePrepared: true }); if (session.changes.some(change => ['pending', 'conflict'].includes(change.status))) throw workerConflict('请先审查当前待处理改动,再创建协作目标。'); @@ -76,8 +77,9 @@ export class WorkerManager { if (JSON.stringify(sourceIdentity(source)) !== JSON.stringify(group.source.identity)) throw workerConflict('来源HEAD、索引、配置或已有修改已变化;原工作区保留,请重新审查协作来源。'); return source; } - async plan(session, input) { - this.assertIdle(session); + async plan(session, input, { evaluation = null } = {}) { + this.assertIdle(session, evaluation?.batchId); + if (!evaluation && [...this.store.core.evaluation.values()].some(batch => batch.projectId === session.projectId && !['completed','retained'].includes(batch.status))) throw workerConflict('工程已有评测,请先完成原批或保留现场结束。'); if (workerForProject(this.store, session.projectId)) throw workerConflict('受限子任务不能创建嵌套协作。'); if (this.groups(session).some(group => !COLLABORATION_CLOSED.has(group.status)) || [...this.store.core.collaboration.values()].some(group => group.projectId === session.projectId && !COLLABORATION_CLOSED.has(group.status))) throw workerConflict('项目已有未收尾的父协作,请先完成或明确保留原成果。'); @@ -98,7 +100,10 @@ export class WorkerManager { const group = { schemaVersion: 1, id: randomUUID(), projectId: project.id, sessionId: session.id, taskId: session.taskId, runId: run.id, goal: definition.goal, budget: definition.budget, model: modelSnapshot(config), config: configView(config), source: { identity: sourceIdentity(source), changedPaths: source.changedPaths, dirty: source.dirty, scope: source.scope }, - children: definition.children.map(child => ({ ...child, ...(engine ? { engine: structuredClone(engine) } : {}), id: randomUUID(), workspaceId: randomUUID(), projectId: randomUUID(), sessionId: randomUUID(), taskId: randomUUID() })), + ...(evaluation ? { evaluation } : {}), + children: definition.children.map((child, index) => ({ ...child, ...(engine ? { engine: structuredClone(engine) } : {}), + ...(evaluation ? { evaluation: { ...evaluation, role: index === 0 ? 'original' : 'candidate' } } : {}), + id: randomUUID(), workspaceId: randomUUID(), projectId: randomUUID(), sessionId: randomUUID(), taskId: randomUUID() })), status: 'planned', requiresVerification: false, createdAt: at(), updatedAt: at(), startedAt: null, deadline: null, finishedAt: null, summary: '父子目标、范围、模型和预算已封存,尚未创建工作区或请求模型。', collection: null, merge: null }; group.fingerprint = workerHash(collaborationSeal(group)); @@ -111,7 +116,7 @@ export class WorkerManager { } finally { this.preparingGroupId = null; this.active.delete(temporaryId); active.settle(); } } launch(session, group, status, execute) { - if (this.active.has(group.id) || this.runtime.projectHasActivity(group.projectId)) throw workerConflict('协作操作正在执行,不能重复启动。'); + if (this.active.has(group.id) || this.runtime.projectHasActivity(group.projectId, null, { excludeEvaluationId: group.evaluation?.batchId })) throw workerConflict('协作操作正在执行,不能重复启动。'); const controller = new AbortController(), active = { sessionId: session.id, projectId: group.projectId, group, controller, promise: null }; group.status = status; this.active.set(group.id, active); try { @@ -217,6 +222,8 @@ export class WorkerManager { const chosen = group.children.filter(child => workerId === null || child.id === workerId).map(child => this.store.core.worker.get(child.id)); if (!chosen.length || chosen.some(worker => !worker)) throw workerConflict('子任务不属于当前父目标。'); const candidates = chosen.filter(worker => worker.status !== 'completed'); + if (candidates.some(worker => worker.evaluation?.role === 'original')) throw workerConflict('评测原版只执行固定检查,不请求模型。'); + if (group.evaluation && this.store.core.evaluation.get(group.evaluation.batchId)?.samples.find(sample => sample.id === group.evaluation.sampleId)?.status !== 'candidate_running') throw workerConflict('请在评测样本中显式开始候选执行。'); if (!candidates.length) throw workerConflict('所选子任务已经完成,不会重放。'); for (const worker of candidates) { const childSession = this.store.session(worker.sessionId); workerForSession(this.store, childSession); assertWorkersResolved(this.store, childSession.projectId); @@ -244,6 +251,7 @@ export class WorkerManager { async runChild(group, worker, config, signal) { const session = this.store.session(worker.sessionId); try { + await verifyEvaluationSources(this.store, session); signal.throwIfAborted(); worker.status = 'running'; worker.startedAt ??= at(); worker.ownerPid = process.pid; this.store.putRecord('worker', session, worker, 'worker.starting'); this.runtime.startWorker(session, worker, config); @@ -288,6 +296,8 @@ export class WorkerManager { } async output(session, worker) { workerForSession(this.store, session); + if (worker.evaluation?.role === 'original') throw workerConflict('原版检查不生成候选成果。'); + await verifyEvaluationSources(this.store, session); const baseline = this.baseline(session, worker), project = this.store.project(worker.projectId); await verifyWorkspaceProject(this.store, project); if (this.uncertain(worker).length) throw workerConflict('子任务仍有结果未知的工具、检查或文件批次'); @@ -365,7 +375,7 @@ export class WorkerManager { async collectionFor(session, group) { await this.sourceFor(group); const proofs = []; - for (const child of group.children) proofs.push(await this.verifiedResult(group, child)); + for (const child of group.children.filter(child => !group.evaluation || child.evaluation?.role === 'candidate')) proofs.push(await this.verifiedResult(group, child)); const conflicts = [], targets = [], project = this.store.project(group.projectId); const all = proofs.flatMap(proof => proof.output.files.map(file => ({ ...file, workerId: proof.worker.id, label: proof.worker.label }))); for (let index = 0; index < all.length; index++) { @@ -405,8 +415,9 @@ export class WorkerManager { checkJobId: proof.output.check.id, paths: proof.output.files.map(file => file.path) })), paths: all.map(file => file.path).sort(), conflicts, targets: targets.sort((a, b) => a.path.localeCompare(b.path)) }; } - collect(session, id, fingerprint) { + collect(session, id, fingerprint, { evaluation = false } = {}) { const group = this.own(session, id, fingerprint); + if (group.evaluation && !evaluation) throw workerConflict('评测成果请在原样本对照报告中采纳。'); if (!['ready_to_collect', 'collected', 'conflict'].includes(group.status) || group.requiresVerification || group.merge) throw workerConflict('先完成两个子任务,已有合并审查不能重新归集。'); return this.launch(session, group, 'collecting', async signal => { try { @@ -455,7 +466,7 @@ export class WorkerManager { await this.faultInjector?.('merge_proposal', group, file); } await this.verifyCollection(session, group); signal.throwIfAborted(); - const manager = new ChangeSetManager(this.store, { projectBusy: (projectId, exceptId) => this.runtime.projectHasActivity(projectId, exceptId, { excludeGroupId: group.id }) }); + const manager = new ChangeSetManager(this.store, { projectBusy: (projectId, exceptId) => this.runtime.projectHasActivity(projectId, exceptId, { excludeGroupId: group.id, excludeEvaluationId: group.evaluation?.batchId }) }); const set = await manager.prepare(session, group.merge.changeIds, { collaborationBinding: { id: group.id, fingerprint: group.fingerprint, collectionArtifactId: group.collection.artifactId, collectionSha256: group.collection.sha256 } }); group.merge.changeSetId = set.id; group.merge.fingerprint = set.fingerprint; group.merge.status = 'prepared'; @@ -506,6 +517,7 @@ export class WorkerManager { if (group.status !== 'merge_prepared' || group.requiresVerification || group.merge?.changeSetId !== set.id || group.merge.fingerprint !== set.fingerprint || set.collaborationBinding?.collectionArtifactId !== group.collection?.artifactId || set.collaborationBinding?.collectionSha256 !== group.collection?.sha256 || JSON.stringify(set.files.map(file => file.changeId)) !== JSON.stringify(group.merge.changeIds)) throw workerConflict('父变更集与原协作/归集记录不一致,不能应用。'); + if (group.evaluation) this.runtime.evaluationManager().adoptionFor(session, group); const collection = await this.verifyCollection(session, group); for (const file of set.files) { const target = collection.targets.find(value => value.path === file.path), change = session.changes.find(value => value.id === file.changeId); @@ -527,7 +539,7 @@ export class WorkerManager { && job.status === 'completed' && !job.requiresVerification && job.commit === inspection.git.head && job.sourceFingerprint === inspection.sourceFingerprint); if (!delivery || !delivery.reportArtifactId) throw workerConflict('父工程尚未完成当前成果的本地Git交付。'); const report = this.store.owned('artifact', delivery.reportArtifactId, session.id); this.store.artifactStore.read(report); - for (const child of group.children) await this.verifiedResult(group, child); + for (const child of group.children.filter(child => !group.evaluation || child.evaluation?.role === 'candidate')) await this.verifiedResult(group, child); group.delivery = { jobId: delivery.id, commit: delivery.commit, checkJobId: check.id, reportArtifactId: report.id, reportSha256: report.sha256 }; group.status = 'completed'; group.finishedAt = at(); group.summary = group.children.length + ' 个隔离子成果已审查合并,父工程重新检查通过并完成本地Git交付。'; session.status = 'completed'; this.save(session, group, 'completed'); return group; @@ -709,7 +721,7 @@ export class WorkerManager { const workspace = this.store.core.workspace.get(child.workspaceId); if (workspace?.status === 'prepared' && !workspace.creationAttempted) this.workspaces.discard(session, workspace.id, workspace.fingerprint); } - const manager = new ChangeSetManager(this.store, { projectBusy: (projectId, exceptId) => this.runtime.projectHasActivity(projectId, exceptId, { excludeGroupId: group.id }) }); + const manager = new ChangeSetManager(this.store, { projectBusy: (projectId, exceptId) => this.runtime.projectHasActivity(projectId, exceptId, { excludeGroupId: group.id, excludeEvaluationId: group.evaluation?.batchId }) }); for (const owned of sessions) { for (const set of this.store.core.change_set.values()) if (set.sessionId === owned.id && set.taskId === owned.taskId && !set.attemptedAt && ['prepared', 'stale'].includes(set.status)) { manager.snapshots(owned, set); manager.discard(owned, set.id, set.fingerprint); diff --git a/src/server/app.mjs b/src/server/app.mjs index d3b5cd3a..3264c696 100644 --- a/src/server/app.mjs +++ b/src/server/app.mjs @@ -307,7 +307,31 @@ return async (request, response) => { } } if (method === 'POST' && workerForProject(store, session.projectId) - && /^(?:mcp|engines|research|workspaces|delivery|browser|task-plans|task-proofs|task-checkpoints|collaborations)(?:\/|-|$)/.test(action)) throw workerConflict('受限子任务只开放原目标的读取、范围内改动和已审批检查;此操作请返回父项目。'); + && /^(?:mcp|engines|research|workspaces|delivery|browser|task-plans|task-proofs|task-checkpoints|collaborations|evaluations)(?:\/|-|$)/.test(action)) throw workerConflict('受限子任务只开放原目标的读取、范围内改动和已审批检查;此操作请返回父项目。'); + if (method === 'GET' && action === 'evaluations') return json(response,runtime.evaluationManager().view(session)); + if (method === 'POST' && action === 'evaluations') { + const batch=await runtime.evaluationManager().plan(session,await body(request)); + return json(response,{batch,session:store.publicSession(session)},201); + } + const evaluation=action.match(/^evaluations\/([a-f0-9-]+)\/(prepare|baseline|start|seal|adopt|finalize|retain|stop|report)$/); + if (evaluation) { + const manager=runtime.evaluationManager(); + if (method==='GET' && evaluation[2]==='report') { + const batch=manager.own(session,evaluation[1]); + return json(response,manager.readReport(session,batch,manager.sample(batch,url.searchParams.get('sampleId')))); + } + if (method==='POST' && evaluation[2]!=='report') { + const input=await body(request); + if (!input || typeof input!=='object' || Array.isArray(input) || !/^[a-f0-9]{64}$/.test(input.fingerprint??'') || Object.keys(input).some(key=>!['fingerprint','sampleId'].includes(key))) throw workerConflict('评测操作需要原计划指纹,只接受固定样本身份。'); + const batch=manager.own(session,evaluation[1],input.fingerprint); + let result; + if(evaluation[2]==='stop'){await runtime.stop(session.id);result=batch;} + else if(evaluation[2]==='finalize')result=await manager.finalize(session,batch.id,input.fingerprint); + else if(evaluation[2]==='retain')result=manager.retain(session,batch.id,input.fingerprint); + else result=await manager[evaluation[2]](session,batch.id,input.fingerprint,input.sampleId); + return json(response,{batch:result.batch??result,session:store.publicSession(session)},result.promise?202:200); + } + } if (method === 'GET' && action === 'collaborations') return json(response, runtime.workerManager().view(session)); if (method === 'POST' && action === 'collaborations') { const group = await runtime.workerManager().plan(session, await body(request)); @@ -324,6 +348,7 @@ return async (request, response) => { const input = await body(request); if (Object.keys(input).some(key => !['fingerprint', 'workerId', 'evidenceSha256', 'retain'].includes(key))) throw workerConflict('协作操作字段无效,不能添加命令或模型配置。'); const group = manager.own(session, collaboration[1], input.fingerprint); + if (group.evaluation && ['prepare','start','collect','merge','finalize'].includes(collaboration[2])) throw workerConflict('这个协作属于固定评测,请返回评测样本执行。'); let result; if (collaboration[2] === 'prepare') result = manager.prepare(session, group.id, input.fingerprint); else if (collaboration[2] === 'start') result = manager.start(session, group.id, input.fingerprint, input.workerId ?? null); diff --git a/src/server/runtime.mjs b/src/server/runtime.mjs index 2aa66b6f..903f1ad5 100644 --- a/src/server/runtime.mjs +++ b/src/server/runtime.mjs @@ -36,6 +36,7 @@ import { SkillRegistry, skillToolDefinitions, skillReadInput } from '../core/ski import { McpRegistry } from '../core/tools/mcp-registry.mjs'; import { mcpToolDefinitions, mcpCallInput, assertMcpResolved } from '../core/tools/mcp-contract.mjs'; import { WorkerManager } from '../core/tasks/worker-manager.mjs'; +import { EvaluationManager } from '../core/evaluation/evaluation-manager.mjs'; import { WORKER_TOOLS, COLLABORATION_CLOSED, workerForSession, assertWorkerTool, assertWorkersResolved, workerUsage, workerConflict } from '../shared/contracts/workers.mjs'; const workerAuthority = Symbol('internal-worker-start'); @@ -108,8 +109,9 @@ export class Runtime { this.appDirectory = appDirectory; this.active = new Map(); this.waiters = new Map(); } - isActive(id) { return this.active.has(id) || !!this.workers?.isActive(id) || !!this.controlledEngines?.isActive(id) || !!this.checks?.isActive(id) || !!this.deliveries?.isActive(id) || !!this.changeSets?.isActive(id) || !!this.workspaces?.isActive(id) || !!this.research?.isActive(id) || !!this.engines?.isActive(id) || !!this.browser?.isActive(id) || !!this.taskPlans?.isActive(id); } - hasActivity() { return (this.provider.activeRequests ?? 0) > 0 || (this.provider.hub?.waiting ?? 0) > 0 || this.active.size > 0 || (this.workers?.active.size ?? 0) > 0 || (this.controlledEngines?.drivers.size ?? 0) > 0 || (this.mcp?.active.size ?? 0) > 0 || (this.checks?.active.size ?? 0) > 0 || (this.deliveries?.active.size ?? 0) > 0 || (this.changeSets?.projects.size ?? 0) > 0 || (this.workspaces?.projects.size ?? 0) > 0 || (this.workspaces?.pending.size ?? 0) > 0 || (this.research?.active.size ?? 0) > 0 || (this.engines?.active.size ?? 0) > 0 || (this.browser?.active.size ?? 0) > 0 || (this.taskPlans?.active.size ?? 0) > 0; } + isActive(id) { return !!this.evaluations?.isActive(id) || this.active.has(id) || !!this.workers?.isActive(id) || !!this.controlledEngines?.isActive(id) || !!this.checks?.isActive(id) || !!this.deliveries?.isActive(id) || !!this.changeSets?.isActive(id) || !!this.workspaces?.isActive(id) || !!this.research?.isActive(id) || !!this.engines?.isActive(id) || !!this.browser?.isActive(id) || !!this.taskPlans?.isActive(id); } + hasActivity() { return (this.evaluations?.active.size ?? 0) > 0 || (this.provider.activeRequests ?? 0) > 0 || (this.provider.hub?.waiting ?? 0) > 0 || this.active.size > 0 || (this.workers?.active.size ?? 0) > 0 || (this.controlledEngines?.drivers.size ?? 0) > 0 || (this.mcp?.active.size ?? 0) > 0 || (this.checks?.active.size ?? 0) > 0 || (this.deliveries?.active.size ?? 0) > 0 || (this.changeSets?.projects.size ?? 0) > 0 || (this.workspaces?.projects.size ?? 0) > 0 || (this.workspaces?.pending.size ?? 0) > 0 || (this.research?.active.size ?? 0) > 0 || (this.engines?.active.size ?? 0) > 0 || (this.browser?.active.size ?? 0) > 0 || (this.taskPlans?.active.size ?? 0) > 0; } + evaluationManager() { return this.evaluations ??= new EvaluationManager(this.store, this); } workerManager() { return this.workers ??= new WorkerManager(this.store, this); } projectIndexer() { return this.codeIndex ??= new ProjectIndex({ cacheDirectory: path.join(this.store.directory, 'cache', 'project-index') }); } taskPlanner() { return this.taskPlans ??= new TaskPlanManager(this.store, { redact: value => this.provider.redact(value), @@ -136,28 +138,28 @@ export class Runtime { return this.research ??= new ResearchManager(this.store, { appDirectory: this.appDirectory, redact: value => this.provider.redact(value), projectBusy: (projectId, exceptId) => this.projectHasActivity(projectId, exceptId) }); } - projectHasActivity(projectId, exceptSessionId = null, { excludeGroupId = null } = {}) { - return !!this.workers?.projectActive(projectId, excludeGroupId) || !!this.controlledEngines?.projectActive(projectId, exceptSessionId) || [...this.active.entries()].some(([id, run]) => id !== exceptSessionId && run.project.id === projectId) || !!this.checks?.projectActive(projectId) || !!this.deliveries?.projectActive(projectId) || !!this.changeSets?.projectActive(projectId) || !!this.workspaces?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId); + projectHasActivity(projectId, exceptSessionId = null, { excludeGroupId = null, excludeEvaluationId = null } = {}) { + return !!this.evaluations?.projectActive(projectId, excludeEvaluationId) || !!this.workers?.projectActive(projectId, excludeGroupId) || !!this.controlledEngines?.projectActive(projectId, exceptSessionId) || [...this.active.entries()].some(([id, run]) => id !== exceptSessionId && run.project.id === projectId) || !!this.checks?.projectActive(projectId) || !!this.deliveries?.projectActive(projectId) || !!this.changeSets?.projectActive(projectId) || !!this.workspaces?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId); } workspaceManager() { return this.workspaces ??= new WorkspaceManager(this.store, { initialize: this.store.core.workspace.size === 0, redact: value => this.provider.redact(value), projectBusy: projectId => - !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId) || [...this.active.values()].some(run => run.project.id === projectId) || !!this.checks?.projectActive(projectId) + !!this.evaluations?.projectActive(projectId) || !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId) || [...this.active.values()].some(run => run.project.id === projectId) || !!this.checks?.projectActive(projectId) || !!this.deliveries?.projectActive(projectId) || !!this.changeSets?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId) }); } changeSetManager() { return this.changeSets ??= new ChangeSetManager(this.store, { beforeApply: (session, set) => this.workerManager().verifyMerge(session, set), projectBusy: (projectId, exceptId) => - !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId, exceptId) || [...this.active.entries()].some(([id, run]) => id !== exceptId && run.project.id === projectId) + !!this.evaluations?.projectActive(projectId) || !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId, exceptId) || [...this.active.entries()].some(([id, run]) => id !== exceptId && run.project.id === projectId) || !!this.checks?.projectActive(projectId) || !!this.deliveries?.projectActive(projectId) || !!this.workspaces?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId) }); } checkRunner() { return this.checks ??= new CheckRunner(this.store, { redact: value => this.provider.redact(value), runtimeDirectory: this.appDirectory ? path.join(this.appDirectory, 'runtime') : null, pythonRunnerPath: path.join(this.appDirectory ?? process.cwd(), 'scripts', 'python-check-runner.py'), - projectBusy: (projectId, exceptId) => !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId, exceptId) || [...this.active.entries()].some(([id, run]) => id !== exceptId && run.project.id === projectId) || !!this.deliveries?.projectActive(projectId) || !!this.changeSets?.projectActive(projectId) || !!this.workspaces?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId) }); + projectBusy: (projectId, exceptId) => !!this.evaluations?.projectActive(projectId) || !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId, exceptId) || [...this.active.entries()].some(([id, run]) => id !== exceptId && run.project.id === projectId) || !!this.deliveries?.projectActive(projectId) || !!this.changeSets?.projectActive(projectId) || !!this.workspaces?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId) }); } gitDelivery() { return this.deliveries ??= new GitDelivery(this.store, { redact: value => this.provider.redact(value), - projectBusy: (projectId, exceptId) => !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId, exceptId) || [...this.active.entries()].some(([id, run]) => id !== exceptId && run.project.id === projectId) || !!this.checks?.projectActive(projectId) || !!this.changeSets?.projectActive(projectId) || !!this.workspaces?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId) }); + projectBusy: (projectId, exceptId) => !!this.evaluations?.projectActive(projectId) || !!this.workers?.projectActive(projectId) || !!this.controlledEngines?.projectActive(projectId, exceptId) || [...this.active.entries()].some(([id, run]) => id !== exceptId && run.project.id === projectId) || !!this.checks?.projectActive(projectId) || !!this.changeSets?.projectActive(projectId) || !!this.workspaces?.projectActive(projectId) || !!this.research?.projectActive(projectId) || !!this.engines?.projectActive(projectId) || !!this.browser?.projectActive(projectId) || !!this.taskPlans?.projectActive(projectId) }); } startWorker(session, worker, config) { if (workerForSession(this.store, session) !== worker || !this.workers?.active.has(worker.collaborationId) @@ -170,6 +172,7 @@ export class Runtime { const boundWorker = workerForSession(this.store, session); if (workerAuthorization !== null && workerAuthorization !== workerAuthority || workerConfig && workerAuthorization !== workerAuthority || boundWorker && (workerAuthorization !== workerAuthority || boundWorker !== worker)) throw workerConflict('受限子任务请在父协作中开始或继续,不能替换目标或配置。'); + if (!boundWorker && [...this.store.core.evaluation.values()].some(batch => batch.projectId === session.projectId && !['completed', 'retained'].includes(batch.status))) throw workerConflict('工程已有未收尾评测,请在评测中处理原样本或保留现场结束。'); if (!boundWorker && [...this.store.core.collaboration.values()].some(group => group.projectId === session.projectId && !COLLABORATION_CLOSED.has(group.status))) throw workerConflict('项目已有未收尾父协作,请先处理子任务和合并审查,再发送新目标。'); assertMcpResolved(this.store, session.projectId); if (pendingDeliveryReview(this.store, session)) throw Object.assign(new Error('当前任务已有待确认的Git交付预览。请在任务记录中审查并确认提交,或撤回预览后继续。'), { status: 409 }); @@ -683,6 +686,7 @@ export class Runtime { } }; assertOriginal(); + await this.evaluations?.stopParent(sessionId); await this.workers?.stopParent(sessionId); assertOriginal(); const run = this.active.get(sessionId); @@ -692,5 +696,5 @@ export class Runtime { run.controller.abort(new Error('用户停止任务')); await run.promise; } - async stopAll() { await this.workers?.stopAll(); await Promise.all([...this.active.keys()].map(id => this.stop(id))); await this.controlledEngines?.stopAll(); await this.checks?.stopAll(); await this.deliveries?.stopAll(); await this.changeSets?.stopAll(); await this.workspaces?.stopAll(); await this.research?.stopAll(); await this.engines?.stopAll(); await this.browser?.stopAll(); await this.mcp?.stopAll(); await this.taskPlans?.settle(); } + async stopAll() { await this.evaluations?.stopAll(); await this.workers?.stopAll(); await Promise.all([...this.active.keys()].map(id => this.stop(id))); await this.controlledEngines?.stopAll(); await this.checks?.stopAll(); await this.deliveries?.stopAll(); await this.changeSets?.stopAll(); await this.workspaces?.stopAll(); await this.research?.stopAll(); await this.engines?.stopAll(); await this.browser?.stopAll(); await this.mcp?.stopAll(); await this.taskPlans?.settle(); } } diff --git a/src/server/store.mjs b/src/server/store.mjs index 306170e1..24e29742 100644 --- a/src/server/store.mjs +++ b/src/server/store.mjs @@ -14,6 +14,7 @@ import { ENGINE_RUN_ACTIVE, publicEngineRun, verifyEngineModelInput } from '../s import { BROWSER_ACTIVE } from '../shared/contracts/browser.mjs'; import { publicBrowserPlan } from '../core/browser/browser-manager.mjs'; import { MCP_ACTIVE } from '../core/tools/mcp-contract.mjs'; +import { EVALUATION_ACTIVE } from '../shared/contracts/evaluations.mjs'; import { COLLABORATION_ACTIVE, COLLABORATION_CLOSED } from '../shared/contracts/workers.mjs'; export const now = () => new Date().toISOString(); @@ -253,6 +254,10 @@ export class Store extends EventEmitter { job.status = 'unknown'; job.requiresVerification = true; job.finishedAt = now(); job.error = { code: 'MCP_HOST_INTERRUPTED', message: '宿主中断,外部MCP结果及原进程状态待核对。不按历史PID停止进程,不重放请求。' }; changed = true; } + for (const batch of this.core.evaluation.values()) if (batch.sessionId === session.id && EVALUATION_ACTIVE.has(batch.status)) { + for (const sample of batch.samples) if (EVALUATION_ACTIVE.has(sample.status)) { sample.interruptedStatus = sample.status; sample.status = 'unknown'; sample.error = '宿主中断;原执行结果保持未知,不自动重跑。'; } + batch.status = 'unknown'; batch.message = '宿主中断,评测及所属执行需要核对;不重放已完成工具。'; changed = true; + } for (const group of this.core.collaboration.values()) if (group.sessionId === session.id && COLLABORATION_ACTIVE.has(group.status)) { group.status = 'unknown'; group.requiresVerification = true; group.finishedAt = now(); group.summary = '父协作在宿主中断时没有完整收尾;保留子任务和工作区,核对原输入与现场后显式处理,不自动重放。'; @@ -646,6 +651,7 @@ export class Store extends EventEmitter { ...value, baseline: { head: snapshot.head, branch: snapshot.branch, reference: snapshot.reference }, })), accountedTaskTokens: accountedTokens(task?.usage), + ...([...this.core.evaluation.values()].some(value => value.sessionId === session.id) ? { evaluations: [...this.core.evaluation.values()].filter(value => value.sessionId === session.id) } : {}), ...([...this.core.collaboration.values()].some(value => value.sessionId === session.id) ? { collaborations: [...this.core.collaboration.values()].filter(value => value.sessionId === session.id), } : {}), diff --git a/src/shared/contracts/evaluations.mjs b/src/shared/contracts/evaluations.mjs new file mode 100644 index 00000000..1a8db052 --- /dev/null +++ b/src/shared/contracts/evaluations.mjs @@ -0,0 +1,69 @@ +import { createHash } from 'node:crypto'; +import { workerPath, workerPathKey } from './workers.mjs'; + +export const EVALUATION_ACTIVE = new Set(['preparing', 'baseline_running', 'candidate_running', 'collecting', 'adopting']); +export const evaluationConflict = message => Object.assign(new Error(message), { status: 409 }); +export const evaluationHash = value => createHash('sha256').update(typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value)).digest('hex'); +const object = (value, keys, label) => { + if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(key => !keys.includes(key))) throw evaluationConflict(label + '字段无效。'); +}; +const text = (value, limit, label) => { + if (typeof value !== 'string' || !value.trim() || value.length > limit || !value.isWellFormed() || /[\x00-\x08\x0b\x0c\x0e-\x1f]/.test(value)) throw evaluationConflict(label + '不能为空或超出长度范围。'); + return value.trim(); +}; +const integer = (value, fallback, min, max, label) => { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < min || result > max) throw evaluationConflict(label + '超出允许范围。'); + return result; +}; +export function evaluationSourcePath(value) { + const relative = workerPath(value); + if (!/\.(?:[cm]?[jt]sx?|py|go|rs|cs|vue|css|html)$/.test(relative) + || /(?:^|\/)(?:tests?|__tests__|fixtures?|scripts?|\.github|runtime|releases|vendor|node_modules)(?:\/|$)/i.test(relative) + || /(?:^|[./_-])(?:test|spec|config|setup|conftest)(?:[./_-]|$)/i.test(relative)) throw evaluationConflict('候选范围仅接受明确的源码文件;测试、运行声明和验收脚本保持原输入。'); + return relative; +} +export function evaluationInput(input, config) { + object(input, ['title', 'samples', 'budget', 'evidenceMode'], '评测'); + if (!Array.isArray(input.samples) || !input.samples.length || input.samples.length > 3) throw evaluationConflict('每批评测需要1到3个固定样本。'); + const count = input.samples.length; + const samples = input.samples.map(item => { + object(item, ['label', 'goal', 'criteria', 'paths', 'checks'], '样本'); + if (!Array.isArray(item.paths) || !item.paths.length || item.paths.length > 16) throw evaluationConflict('样本需要1到16个明确源码文件。'); + const paths = item.paths.map(evaluationSourcePath).sort(); + if (new Set(paths.map(workerPathKey)).size !== paths.length) throw evaluationConflict('源码文件不能重复。'); + if (!Array.isArray(item.checks) || !item.checks.length || item.checks.length > 6 || item.checks.some(name => typeof name !== 'string' || !/^(?:test(?::[a-z0-9_-]+)?|typecheck|check|lint|build|python:(?:test|compile))$/.test(name)) + || new Set(item.checks).size !== item.checks.length) throw evaluationConflict('检查需要1到6个不同的已识别名称。'); + return { label: text(item.label, 64, '样本名称'), goal: text(item.goal, 6000, '需求'), criteria: text(item.criteria, 4000, '验收条件'), paths, checks: [...item.checks] }; + }); + if (new Set(samples.map(item => item.label)).size !== count) throw evaluationConflict('样本名称不能重复。'); + const raw = input.budget ?? {}; object(raw, ['maxTokens', 'maxSteps', 'maxToolCalls', 'maxDurationMs'], '总预算'); + const maximumTokens = Math.min(config.maxTaskTokens ?? 1000000, (config.maxRunTokens ?? 200000) * count); + const budget = { + maxTokens: integer(raw.maxTokens, maximumTokens, count * 2000, maximumTokens, '总token预算'), + maxSteps: integer(raw.maxSteps, Math.min(200, (config.maxSteps ?? 12) * count), count * 2, 200, '总请求步数'), + maxToolCalls: integer(raw.maxToolCalls, Math.min(512, count * 64), count * 2, 512, '总工具次数'), + maxDurationMs: integer(raw.maxDurationMs, config.maxRunDurationMs ?? 600000, count * 1000, Math.min(3600000, config.maxRunDurationMs ?? 600000), '总执行时限'), + }; + const evidenceMode = input.evidenceMode ?? 'configured-model'; + if (!['configured-model', 'local-fixture'].includes(evidenceMode)) throw evaluationConflict('评测来源需要明确为实际配置模型或本地固定响应。'); + return { title: text(input.title, 120, '评测名称'), samples, budget, evidenceMode }; +} +export function evaluationSeal(batch) { + return { schemaVersion: 1, id: batch.id, projectId: batch.projectId, sessionId: batch.sessionId, taskId: batch.taskId, runId: batch.runId, + title: batch.title, source: batch.source, budget: batch.budget, evidenceMode: batch.evidenceMode, model: batch.model, + samples: batch.samples.map(({ id, label, goal, criteria, paths, checks }) => ({ id, label, goal, criteria, paths, checks })) }; +} +export function evaluationSampleSeal(batch, sample) { + return { schemaVersion: 1, batchId: batch.id, batchFingerprint: batch.fingerprint, sampleId: sample.id, groupId: sample.groupId, + baseCommit: batch.source.head, definition: evaluationSeal(batch).samples.find(value => value.id === sample.id), + files: sample.files, commands: sample.commands, children: sample.children }; +} +export function evaluationSummary(batch) { + const outcomes = batch.samples.map(sample => sample.report?.summary ?? { original: sample.baselineAttemptedAt ? 'unknown' : 'pending', candidate: ['failed','stopped','unknown'].includes(sample.status) ? sample.status : 'pending' }); + const count = (role, status) => outcomes.filter(item => item[role] === status).length; + return { samples: batch.samples.length, finished: batch.samples.filter(item => !!item.report).length, + original: Object.fromEntries(['passed', 'failed', 'stopped', 'unknown', 'pending'].map(status => [status, count('original', status)])), + candidate: Object.fromEntries(['passed', 'failed', 'stopped', 'unknown', 'pending'].map(status => [status, count('candidate', status)])), + inferenceValidated: false, scope: batch.evidenceMode === 'local-fixture' ? '固定响应仅验证执行链和工程检查,不计模型自主能力。' : '保留当前固定样本的实际结果,不外推任意工程成功率。' }; +} diff --git a/src/shared/contracts/execution.mjs b/src/shared/contracts/execution.mjs index fd6df150..b1d7166e 100644 --- a/src/shared/contracts/execution.mjs +++ b/src/shared/contracts/execution.mjs @@ -4,6 +4,7 @@ export const EXECUTION_SCHEMA = 1; export const ACTIVE_STATES = new Set(['running', 'awaiting_approval', 'stopping']); export const CORE_KINDS = ['task', 'run', 'step', 'tool', 'artifact', 'check_plan', 'check_job', 'delivery_plan', 'delivery_job', 'change_set', 'context_plan', 'workspace', 'research_target', 'research_job', 'engine_job', 'engine_run', 'browser_plan', 'browser_job', 'task_plan', 'task_proof', 'memory', 'task_checkpoint', 'mcp_job', 'collaboration', 'worker', 'input_attachment']; CORE_KINDS.push('resume_plan'); +CORE_KINDS.push('evaluation'); export const timestamp = () => new Date().toISOString(); export const isId = value => typeof value === 'string' && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value); diff --git a/src/shared/contracts/workers.mjs b/src/shared/contracts/workers.mjs index 34fdc607..109842b0 100644 --- a/src/shared/contracts/workers.mjs +++ b/src/shared/contracts/workers.mjs @@ -59,7 +59,8 @@ export function collaborationInput(input, config) { export function collaborationSeal(group) { return { schemaVersion: 1, id: group.id, projectId: group.projectId, sessionId: group.sessionId, taskId: group.taskId, runId: group.runId, - goal: group.goal, budget: group.budget, model: group.model, config: group.config, source: group.source, children: group.children }; + goal: group.goal, budget: group.budget, model: group.model, config: group.config, source: group.source, children: group.children, + ...(group.evaluation ? { evaluation: group.evaluation } : {}) }; } export function workerSeal(worker) { return { schemaVersion: 1, id: worker.id, collaborationId: worker.collaborationId, parentProjectId: worker.parentProjectId, @@ -67,7 +68,7 @@ export function workerSeal(worker) { projectId: worker.projectId, sessionId: worker.sessionId, taskId: worker.taskId, workspaceId: worker.workspaceId, label: worker.label, goal: worker.goal, paths: worker.paths, budget: worker.budget, model: worker.model, baseCommit: worker.baseCommit, baselineArtifactId: worker.baselineArtifactId, baselineSha256: worker.baselineSha256, baselineRunId: worker.baselineRunId, - ...(worker.engine ? { engine: worker.engine } : {}) }; + ...(worker.engine ? { engine: worker.engine } : {}), ...(worker.evaluation ? { evaluation: worker.evaluation } : {}) }; } export function workerForProject(store, projectId) { return [...(store.core.worker?.values() ?? [])].find(value => value.projectId === projectId) ?? null; @@ -83,7 +84,8 @@ export function workerForSession(store, session) { || definition.workspaceId !== worker.workspaceId || JSON.stringify(definition.paths) !== JSON.stringify(worker.paths) || JSON.stringify(definition.budget) !== JSON.stringify(worker.budget) || definition.goal !== worker.goal || JSON.stringify(group.model) !== JSON.stringify(worker.model) - || JSON.stringify(definition.engine) !== JSON.stringify(worker.engine)) throw workerConflict('子任务与父任务的封存分工不一致。'); + || JSON.stringify(definition.engine) !== JSON.stringify(worker.engine) + || JSON.stringify(definition.evaluation) !== JSON.stringify(worker.evaluation)) throw workerConflict('子任务与父任务的封存分工不一致。'); const task = store.core.task.get(worker.taskId); if (!task || task.goal !== worker.goal || task.workerId !== worker.id || task.budget.maxTokens !== worker.budget.maxTokens) throw workerConflict('子任务目标、身份或累计预算已变化。'); const original = store.owned('artifact', worker.contractArtifactId, session.id); @@ -98,6 +100,8 @@ export function assertWorkerWriteScope(store, session, value, { readOnly = false const worker = workerForSession(store, session); if (!worker) return; const portable = workerPath(value); + if (!readOnly && worker.evaluation && store.core.evaluation?.get(worker.evaluation.batchId)?.samples.some(sample => sample.id === worker.evaluation.sampleId && sample.report)) throw workerConflict('评测样本已封存,不能再修改候选。'); + if (worker.evaluation && (worker.evaluation.role === 'original' || !worker.paths.some(path => workerPathKey(path) === workerPathKey(portable)))) throw workerConflict('原版仅供固定检查;候选只能修改封存的明确源码文件。'); if (!pathInWorkerScope(portable, worker.paths)) throw workerConflict('文件超出子任务的固定修改范围:' + portable); if (!readOnly && ['unknown', 'completed', 'resolved_external'].includes(worker.status)) throw workerConflict('子任务已封存或结果待核对,不能修改原成果。'); } diff --git a/task_plan.md b/task_plan.md index 5b3d909d..4d86f0ae 100644 --- a/task_plan.md +++ b/task_plan.md @@ -1,6 +1,17 @@ # Agent Workbench:核心代理能力研究与产品计划 -## 当前任务:1.19 项目上下文缓存与长任务效率 +## 当前任务:1.20 工程质量评测与受控自我迭代 + +接续完整1.19交付790350e,在feat/engineering-evals-1.20实施docs/PLAN-1.20.md。正式1.19/PID36552和Provider保持;回归仅独立数据/端口和自有HTTP响应,不提取Key或主动使用用户真实模型。沿用product-ui-ux/frontend-design及文件计划,每切片验收、回顾、本地Git,不委派代理或新建聊天/自动任务。 + +1. [complete] A:固定样本/基线/需求/允许文件/检查/预算契约;审查原40基准及Node/React/Python,明确夹具与真实推理边界。 +2. [complete] B:复用两个隔离工作区,一个只做原版检查,一个生成候选;有界串行样本账本、原状态与停止/重启保护,不重放完成工具。 +3. [complete] C:原版/候选检查、变更、审批/人工介入、用时和原用量对照;缺失与失败保留原分母,不补造评分。 +4. [in_progress] D:桌面/390px评测入口、完整准备/执行/审查/比较/采纳流程,复用现有审批/差异/检查/Git。 +5. [pending] E:实际Node/React/Python工程和自身模块试点,测试输入固定、来源修改保持,真实模型自主完成率单列待验。 +6. [pending] F:完整回归、独立源码/便携/安装/保留状态升级、原生回看、证据/回顾/本地Git与下一计划。 + +## 1.19 项目上下文缓存与长任务效率(已完成) 用户授权下一阶段全量开发并及时回顾。基线606b5ec,分支feat/project-cache-1.19;按docs/PLAN-1.19.md完成A–F。正式服务及Provider、项目/会话/工件均保留,不提取Key或发起真实模型调用。QA使用独立数据、端口和固定HTTP响应;不新建代理、聊天、goal或automation,不远程push。 diff --git a/tests/evaluations.test.mjs b/tests/evaluations.test.mjs new file mode 100644 index 00000000..9f6ccea1 --- /dev/null +++ b/tests/evaluations.test.mjs @@ -0,0 +1,123 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { fixture, call, until } from './helpers.mjs'; +import { createDeliveryProject, deliveryGit } from './fixtures/delivery-project.mjs'; +import { ownerBytes, applyWorkerProposals, approveWorkerChecks } from './fixtures/workers-project.mjs'; +import { Provider } from '../src/server/provider.mjs'; +import { Runtime } from '../src/server/runtime.mjs'; +import { Store } from '../src/server/store.mjs'; +import { proposeFile } from '../src/server/files.mjs'; +import { evaluationInput, evaluationSourcePath, evaluationSummary } from '../src/shared/contracts/evaluations.mjs'; +import { verifyEvaluationSources } from '../src/core/evaluation/provenance.mjs'; +import { workerHttp } from './fixtures/workers-http.mjs'; + +export const definition = (count=1) => ({title:'修复求和并核对完整原测试',evidenceMode:'local-fixture',samples:Array.from({length:count},(_,index)=>({label:'加法样本'+index,goal:'修复加法运算,保留已有功能',criteria:'原3项加法测试及构建通过;不修改原测试',paths:['src/add.mjs'],checks:['test','build']}))}); +function setup(t,{respond=null,usage=true}={}) { + const f=fixture(t);createDeliveryProject(f);const provider=new Provider(),requests=[]; + provider.complete=async(config,messages,project,signal)=>{ + requests.push(project.id);if(respond)return respond({project,signal,f}); + const owner=[...f.store.sessions.values()].find(session=>session.projectId===project.id),taskId=owner.taskId; + const plan=[...f.store.core.check_plan.values()].find(plan=>plan.taskId===taskId),job=[...f.store.core.check_job.values()].find(job=>job.taskId===taskId); + let calls=[]; + if(!owner.changes.length)calls=[call(randomUUID(),'propose_file',{path:'src/add.mjs',content:'export const add = (a,b) => a + b;\n',reason:'本地固定响应中的加法修复'})]; + else if(owner.changes.some(change=>change.status==='pending'))calls=[]; + else if(!plan)calls=[call(randomUUID(),'plan_checks',{names:['test','build']})]; + else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})]; + return {message:{role:'assistant',content:calls.length?'':'固定响应声称完成,仍需实际证据',...(calls.length?{tool_calls:calls}:{})},usage:usage?{total_tokens:168,prompt_tokens:123,completion_tokens:45}:null}; + }; + const runtime=new Runtime(f.store,provider),manager=runtime.evaluationManager();f.onCleanup(()=>runtime.stopAll());return {...f,runtime,manager,requests}; +} +async function prepared(t,options={}) { + const f=setup(t,options),original=ownerBytes(f),batch=await f.manager.plan(f.session,definition()),sample=batch.samples[0]; + await f.manager.prepare(f.session,batch.id,batch.fingerprint,sample.id).promise; + assert.equal(sample.status,'ready',batch.message);return {...f,batch,sample,original,group:f.store.core.collaboration.get(sample.groupId)}; +} +async function baseline(f){await f.manager.baseline(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.sample.status,'baseline_complete',f.batch.message);} +async function complete(f){await baseline(f);await f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.sample.status,'awaiting_review',f.batch.message);await applyWorkerProposals(f,f.runtime,f.group);await approveWorkerChecks(f,f.runtime,f.group,f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id));assert.equal(f.sample.status,'candidate_complete',f.batch.message);} +async function sealed(f){await complete(f);await f.manager.seal(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.ok(f.sample.report,f.batch.message);return f.manager.readReport(f.session,f.batch,f.sample);} + +test('evaluation contract fixes bounded samples, source paths, checks and aggregate budget',t=>{ + const f=fixture(t),valid=evaluationInput(definition(3),f.store.config);assert.equal(valid.samples.length,3); + for(const name of ['test/a.mjs','src/a.test.ts','src/config.ts','package.json','scripts/check.mjs','src','../evil.mjs'])assert.throws(()=>evaluationSourcePath(name)); + for(const input of [{...definition(),command:'bypass'},{...definition(),samples:[]},{...definition(3),samples:[...definition(3).samples,definition().samples[0]]},{...definition(),budget:{maxSteps:1}},{...definition(),samples:[{...definition().samples[0],checks:['test','test']}]}])assert.throws(()=>evaluationInput(input,f.store.config)); +}); +test('same-source workspaces seal real commands before execution and preserve original staged and unstaged bytes',async t=>{ + const f=await prepared(t);assert.deepEqual(ownerBytes(f),f.original);assert.equal(f.requests.length,0); + assert.equal(f.sample.commands[0].script,'node --test test/*.test.mjs');assert.ok(f.sample.contractSha256); + for(const child of f.group.children){const owner=f.store.session(child.sessionId);assert.throws(()=>proposeFile(f.store,owner,{path:'test/add.test.mjs',content:'pass',reason:'weaken test'}),/原版|范围/);assert.throws(()=>f.runtime.start(owner.id,'replace'),/受限|父协作/);} + const original=f.group.children[0],owner=f.store.session(original.sessionId); + assert.throws(()=>proposeFile(f.store,owner,{path:'src/add.mjs',content:'change',reason:'original'}),/原版/); + assert.throws(()=>f.runtime.workerManager().start(f.session,f.group.id,f.group.fingerprint,original.id),/原版/); + await assert.rejects(f.runtime.checkRunner().plan(f.store.session(f.group.children[1].sessionId),['test']),/完整检查/); +}); +test('actual original failures and candidate success are both preserved, then adoption requires parent review, checks and Git',async t=>{ + const f=await prepared(t),report=await sealed(f);assert.deepEqual(report.summary,{original:'failed',candidate:'passed'}); + assert.equal(report.original.attempts[0].results[0].tests.failed,3);assert.equal(report.candidate.attempts[0].results[0].tests.passed,3); + assert.equal(report.original.usage.requests,0);assert.ok(report.candidate.usage.requests>0);assert.equal(report.inferenceValidated,false);assert.ok(report.candidate.approvals.some(approval=>approval.status==='approved')); + assert.deepEqual(ownerBytes(f),f.original);const source=fs.readFileSync(path.join(f.projectRoot,'src/add.mjs')); + await f.manager.adopt(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.batch.status,'awaiting_delivery',f.batch.message);assert.deepEqual(fs.readFileSync(path.join(f.projectRoot,'src/add.mjs')),source); + await assert.rejects(f.manager.finalize(f.session,f.batch.id,f.batch.fingerprint),/应用|变更集/); + const set=f.store.core.change_set.get(f.group.merge.changeSetId);await (await f.runtime.changeSetManager().start(f.session,set.id,set.fingerprint)).promise;assert.equal(set.status,'completed',set.error?.message); + const plan=await f.runtime.checkRunner().plan(f.session,['test','build']);assert.equal((await (await f.runtime.checkRunner().start(f.session,plan.id)).promise).status,'passed'); + const delivery=await f.runtime.gitDelivery().plan(f.session,{message:'Apply evaluated addition candidate'});const result=await (await f.runtime.gitDelivery().start(f.session,delivery.id)).promise;assert.equal(result.status,'completed',result.error?.message); + await f.manager.finalize(f.session,f.batch.id,f.batch.fingerprint);assert.equal(f.batch.adoption.status,'completed'); + const after=ownerBytes(f);assert.deepEqual(after.files,f.original.files);assert.notEqual(after.head,f.original.head);assert.match(deliveryGit(f,['diff','--cached','--name-only']),/owner-staged/); +}); +test('candidate cannot shrink tests and changing original test bytes rejects execution before any model request',async t=>{ + const f=await prepared(t);await baseline(f);const child=f.group.children[1],owner=f.store.session(child.sessionId); + fs.appendFileSync(path.join(f.store.project(child.projectId).root,'test/add.test.mjs'),'\n// changed\n'); + await assert.rejects(verifyEvaluationSources(f.store,owner),/原测试/); + await f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;assert.equal(f.requests.length,0);assert.equal(f.sample.status,'failed'); + await f.manager.seal(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;const report=f.manager.readReport(f.session,f.batch,f.sample);assert.equal(report.inputVerified,false);assert.equal(report.summary.candidate,'unknown'); + assert.throws(()=>f.manager.adopt(f.session,f.batch.id,f.batch.fingerprint,f.sample.id),/通过/); +}); +test('missing upstream usage remains unknown and does not become a free run or invented cost',async t=>{ + const f=await prepared(t,{usage:false}),report=await sealed(f);assert.ok(report.candidate.usage.unknownRequests>0);assert.equal(report.candidate.usage.reported,false);assert.equal(report.candidate.usage.cost,null);assert.ok(report.candidate.usage.estimatedUnknownTokens>0); +}); +test('missing raw check logs preserve unknown result and original sample denominator',async t=>{ + const f=await prepared(t);await complete(f);const job=f.store.core.check_job.get(f.sample.baselineJobId);f.store.core.artifact.delete(job.results[0].logArtifactId); + await f.manager.seal(f.session,f.batch.id,f.batch.fingerprint,f.sample.id).promise;const report=f.manager.readReport(f.session,f.batch,f.sample);assert.equal(report.summary.original,'unknown');assert.ok(report.original.evidenceError);assert.equal(evaluationSummary(f.batch).samples,1); + assert.throws(()=>f.manager.adopt(f.session,f.batch.id,f.batch.fingerprint,f.sample.id),/原版结果完整/); +}); +test('source and model changes never silently replace an existing batch input',async t=>{ + const f=setup(t),batch=await f.manager.plan(f.session,definition());fs.appendFileSync(path.join(f.projectRoot,'README.md'),'owner later'); + await f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[0].id).promise;assert.equal(batch.samples[0].status,'failed');assert.equal(f.store.core.worker.size,0);assert.equal(f.requests.length,0); +}); +test('model revision change before preparing a fixed sample cannot silently use the replacement connection',async t=>{ + const f=setup(t),batch=await f.manager.plan(f.session,definition());f.store.saveConfig({model:'replacement-connection'}); + await f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[0].id).promise;assert.equal(batch.samples[0].status,'failed');assert.equal(f.requests.length,0);assert.equal(f.store.core.worker.size,0); +}); +test('HTTP evaluations require host token, original fingerprint and session ownership; worker API cannot bypass evaluation flow',async t=>{ + const f=fixture(t),host=await workerHttp(f);assert.equal((await host.raw('/sessions/'+f.session.id+'/evaluations',definition(),'wrong-token')).status,403); + const {batch}=await host.api('/sessions/'+f.session.id+'/evaluations',definition()),sample=batch.samples[0]; + assert.equal((await host.raw('/sessions/'+f.session.id+'/evaluations/'+batch.id+'/prepare',{sampleId:sample.id})).status,409); + assert.equal((await host.raw('/sessions/'+f.session.id+'/evaluations/'+batch.id+'/prepare',{fingerprint:batch.fingerprint,sampleId:sample.id,command:'extra'})).status,409); + const other=f.store.createSession(f.project.id);assert.equal((await host.raw('/sessions/'+other.id+'/evaluations/'+batch.id+'/prepare',{fingerprint:batch.fingerprint,sampleId:sample.id})).status,404); + await host.api('/sessions/'+f.session.id+'/evaluations/'+batch.id+'/prepare',{fingerprint:batch.fingerprint,sampleId:sample.id});await until(()=>!host.runtime.evaluationManager().isActive(f.session.id),20000); + const saved=f.store.core.evaluation.get(batch.id);assert.equal(saved.samples[0].status,'ready',saved.message);const group=f.store.core.collaboration.get(saved.samples[0].groupId); + assert.equal((await host.raw('/sessions/'+group.children[1].sessionId+'/evaluations',definition())).status,409); + assert.equal((await host.raw('/sessions/'+f.session.id+'/collaborations/'+group.id+'/start',{fingerprint:group.fingerprint,workerId:group.children[1].id})).status,409); + assert.equal(host.requests.length,0); +}); +test('sample queue cannot skip or reset attempted samples, and reports freeze further writes',async t=>{ + const f=setup(t),batch=await f.manager.plan(f.session,definition(2));assert.throws(()=>f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[1].id),/顺序/); + const sample=batch.samples[0];await f.manager.prepare(f.session,batch.id,batch.fingerprint,sample.id).promise;assert.equal(sample.status,'ready',batch.message); + assert.throws(()=>f.manager.prepare(f.session,batch.id,batch.fingerprint,sample.id),/尝试/); + await f.manager.seal(f.session,batch.id,batch.fingerprint,sample.id).promise;const child=sample.children[1],owner=f.store.session(child.sessionId); + assert.throws(()=>proposeFile(f.store,owner,{path:'src/add.mjs',content:'change',reason:'sealed'}),/封存/); + await f.manager.prepare(f.session,batch.id,batch.fingerprint,batch.samples[1].id).promise;assert.equal(batch.samples[1].status,'ready',batch.message); + f.manager.retain(f.session,batch.id,batch.fingerprint);assert.equal(evaluationSummary(batch).samples,2);assert.equal(evaluationSummary(batch).finished,1); +}); +test('stop cancels only current candidate and preserves partial results instead of replaying a request',async t=>{ + const f=await prepared(t,{respond:({signal})=>new Promise((resolve,reject)=>signal.addEventListener('abort',()=>reject(signal.reason),{once:true}))});await baseline(f); + const started=f.manager.start(f.session,f.batch.id,f.batch.fingerprint,f.sample.id);await until(()=>f.requests.length===1);await f.runtime.stop(f.session.id);await started.promise; + assert.equal(f.requests.length,1);assert.equal(f.runtime.hasActivity(),false);assert.notEqual(f.sample.status,'candidate_complete');assert.deepEqual(ownerBytes(f),f.original); +}); +test('restart recovers interrupted evaluations as unknown and keeps completed checks and plan identity',async t=>{ + const f=await prepared(t);await baseline(f);const fingerprint=f.batch.fingerprint,jobId=f.sample.baselineJobId; + f.batch.status='candidate_running';f.sample.status='candidate_running';f.store.save(f.session);f.store.close(); + const reopened=new Store(f.data);f.onCleanup(()=>reopened.close());const saved=reopened.core.evaluation.get(f.batch.id);assert.equal(saved.status,'unknown');assert.equal(saved.samples[0].status,'unknown');assert.equal(saved.fingerprint,fingerprint);assert.equal(saved.samples[0].baselineJobId,jobId);assert.equal(reopened.core.check_job.get(jobId).status,'failed'); +}); diff --git a/tests/fixtures/workers-project.mjs b/tests/fixtures/workers-project.mjs index d45ffaee..583e99d0 100644 --- a/tests/fixtures/workers-project.mjs +++ b/tests/fixtures/workers-project.mjs @@ -8,8 +8,8 @@ import { call, until } from '../helpers.mjs'; export const workerDefinition = (paths = ['src/a.mjs', 'src/b.mjs']) => ({ goal: '交付两个独立改进并核对父工程', children: paths.map((relative, index) => ({ label: 'worker-' + (index + 1), goal: '实现子任务' + (index + 1), paths: [relative] })) }); -export function prepareWorkerFixture(f, { ownerChanges = true, respond = null } = {}) { - createDeliveryProject(f, { broken: false, ownerChanges }); +export function prepareWorkerFixture(f, { ownerChanges = true, respond = null, broken = false } = {}) { + createDeliveryProject(f, { broken, ownerChanges }); const provider = new Provider(), requests = []; provider.complete = async (config, messages, project, signal, progress, planned, tools) => { requests.push({ config, messages, projectId: project.id, tools: tools.map(value => value.function.name) });