test: define bounded 1.24.1 live delivery and original Purge retest

This commit is contained in:
Agent Workbench Dev
2026-11-03 09:29:10 +08:00
parent 7313fd2588
commit 9ca3ba497b
5 changed files with 136 additions and 0 deletions
+20
View File
@@ -0,0 +1,20 @@
# 1.24.1 真实交付与原版 Purge:H 批次
用户已授权重新实测。本批使用正式1.24.1、保留原PID36552及Provider;不重启、不抽取或复制Key,不续跑旧失败Task。G修复是本批执行基线,E结果保留,不合并分母。本批不扩展1.25功能或重新打包发行。
| 新样本 | 实际验收 | 上限 |
| --- | --- | --- |
| React | 模型自行修复四模块,原9项测试、类型检查、构建、真实桌面/窄屏交互及审查Git | 24实际请求、140000报告tokens、10分钟 |
| Python | 模型自行修复三个模块,原6项测试、编译及审查Git | 同上 |
| Node AB | 同一三模块收支报告,先关闭再启用原版受限Purge | 每例24请求、105000报告tokens、10分钟 |
| Node BA | 相同需求、基线、工具、检查和驱动,先开启再关闭 | 同上 |
整批六例串行,最多100实际请求/700000报告tokens。当前已配置Claude连接固定Chat协议、32K容量、2048输出上限;无自动切池。容量经正规API临时调整并保存修订,结束恢复公开参数和原池。原生及宿主按真实已知用量和未知消费预留拒绝超额派发;在途超额、未知usage和网络失败照实记录,不自动重放或重置账本。服务公有模型中心只返回最近100尝试,预算和最终分母改用只读完整modelAttempts,不读取凭据字段。
基线沿用E自有工程的业务需求与原测试,创建全新项目、工作区和Task。模型只收一次任务、允许路径和流程合同,自己读源码并提出实现;验收程序仅批准匹配的原检查、核对模型封存批次和确认Git。审批、应用、显式续接、父工程核对分别计数;实现提示和人工代写均为零。旧已修改工程不会作为新基线。
原版开启为Purge1.1.78、`purge-original-restricted-v1`、Harness0.2.1-rc.2;关闭为同Harness但不加载原版入口或独立提示。原版自动更新、宿主修改、原操作工具、自动重试及继续关闭,保留真实提示/内部钩子。固定来源、每请求实际输入、原生模块加载和进程树退出逐例核对。这个实验只评价此受限工程配置,不是通用模型“破甲”或权重提升的证明。
完整批次交接和源码保留由真实请求与工具账本观察,不能把G离线回放当实测成功。来源漂移、未知操作、预算耗尽或不能确认进程退出即停止本例;独立后续组仅在剩余额度、无活动和来源一致时开始。不得以追加提示、扩大预算、改测试或同失败Task重跑制造成功。
交付结果与Purge收益单独报告:同成功标准下才比较消耗,更早失败不算节省;四例只能给描述性结论,不编统计显著性或收费节省。封存原请求/上游usage、源码基线、原失败、检查/页面/Git证据及监督次数。所有原会话导出、表行、工件原字节和模型参数恢复逐项核对,新追加归属明确;私有原用户快照不入Git,按范围本地提交、不推远端。
+4
View File
@@ -1,5 +1,9 @@
# 汇总发现
## H 实测设计依据
G已修完整批次同轮交接和已读小源码稳定保留,需由新真实样本验证实际工程收益。E驱动可复用工程/工具审查和AB/BA监督合同,但其prepare包含临时处理器激活、零窗口断言和旧源码身份,不能直接运行;H应只读确认正式1.24.1并冻结当前运行载荷。旧E52请求/621486报告tokens和全部失败保持,不改分母;原版开启11份输入和关闭17份输入的来源核对不是收益证明。
## F 固定安装与后续边界
新候选独立源码重建22/22、三归档、实际便携和12组隔离安装通过后,正式已从临时QA根切换至固定versions目录;原PID/Provider与当前42项目/58会话/2850工件及全部原表/导出/模型中心精确保持。原操作675.500秒单列,不沿用旧速度。69份原件离线通过,正式原生窗口回看因物理Escape停止仍未验收。
+4
View File
@@ -1,5 +1,9 @@
# 统一执行进度
## H 新真实模型实测启动
用户要求重新实测真实交付与Purge收益。起点7313fd2、工作区干净;已回顾G修复和E原结果(Python通过,React与四对照未交付),定位原驱动。采用六个新有界样本,公开连接经正规API临时配置,原Provider/内存Key与旧数据保持;尚无真实请求。旧E激活临时处理器的步骤本轮不复用,直接核对已正式固定的1.24.1。本轮使用planning-with-files,不开启子代理。
## G 已确认问题修复启动
用户明确优先修完已总结的问题。基线876df5f干净;先复现完整审查批次仍running和源码在16K软目标下逐份挤出。沿用既有规划与产品流程验收技能,不开子代理,不读取Key或重放旧实测Task;F正式已固定安装,旧Esc停止不作为当前源码修复的等待条件。首批读取超出输出上限,截断内容不作完整审查,后续缩小范围。
+97
View File
@@ -0,0 +1,97 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {createHash} from 'node:crypto';
import {requestMetrics} from '../src/shared/contracts/model-request.mjs';
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
import {assertPurgeModelInput,nativeSystemTexts,ORIGINAL_PURGE_KIND} from '../src/core/engines/purge-strategy.mjs';
// Offline evidence only. No Runtime/Provider, model requests or plugin loading.
const root=path.resolve('docs/live-original-purge-h-evidence'),sha=raw=>createHash('sha256').update(raw).digest('hex');
const file=relative=>{const full=path.resolve(root,relative);assert.ok(full.startsWith(root+path.sep));return full;};
const read=relative=>JSON.parse(fs.readFileSync(file(relative))),manifest=read('manifest.json');
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.slice,'live-original-purge-h');
const paths=new Set();let bytes=0;
for(const e of manifest.entries){assert.ok(!paths.has(e.path));paths.add(e.path);const raw=fs.readFileSync(file(e.path));assert.equal(raw.length,e.bytes,e.path);assert.equal(sha(raw),e.sha256,e.path);bytes+=raw.length;}
assert.equal(paths.size,manifest.count);assert.equal(bytes,manifest.bytes);
assert.equal(sha(fs.readFileSync(manifest.dManifest)),manifest.dManifestSha256);
const d=JSON.parse(fs.readFileSync(manifest.dManifest)),identity=read('receipts/runtime-identity.json');
const release=read('receipts/release-identity.json');
assert.equal(sha(fs.readFileSync(file('receipts/release-identity.json'))),identity.releaseIdentitySha256);
assert.equal(identity.manifestSha256,release.manifestSha256);assert.equal(identity.runtime,release.installedRoot);
assert.equal(identity.sourceCommit,release.sourceCommit);assert.equal(identity.sourceCommit,manifest.sourceCommit);
const installed=read('receipts/release-manifest.json');assert.equal(sha(fs.readFileSync(file('receipts/release-manifest.json'))),release.manifestSha256);
for(const input of identity.files){assert.deepEqual(input,installed.files.find(f=>f.path===input.path),input.path);}
assert.equal(identity.files.filter(f=>f.path.startsWith('dist-')).length,22);
const freeze=read('receipts/driver-freeze.json');
for(const input of freeze.files){const sealed=input.path.startsWith('.qa/')?'drivers/'+path.basename(input.path):null;
if(sealed)assert.equal(sha(fs.readFileSync(file(sealed))),input.sha256,input.path);}
const records=read('receipts/owned-records.json'),owners=new Map(read('receipts/owned-sessions.json').map(s=>[s.id,s])),core={};
for(const row of records){const value=JSON.parse(row.payload);assert.ok(owners.has(value.sessionId),row.id);(core[row.kind]??=new Map()).set(value.id,value);}
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,core.artifact.size);
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId,kind+'/'+id);return value;},
artifactStore:{read(a){return this.readBytes(a).toString('utf8');},readBytes(a){const input=artifacts.get(a.id),raw=fs.readFileSync(file(input.file));assert.equal(sha(raw),a.sha256);assert.equal(raw.length,a.bytes);return raw;}},
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(t=>t.sessionId===sessionId&&t.callId===callId);assert.ok(tool);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
const attempts=read('receipts/all-model-attempts.json'),attemptIds=new Set(attempts.map(a=>a.id));assert.equal(attemptIds.size,attempts.length);
let reportedTokens=0,unknownRequests=0;
for(const attempt of attempts){
const request=core.artifact.get(attempt.requestArtifactId);assert.ok(request);assert.equal(request.sha256,attempt.requestSha256);
const input=JSON.parse(store.artifactStore.read(request));assert.equal(requestMetrics(input).sha256,attempt.requestSha256);assert.equal(input.model,attempt.model);
if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknownRequests++;continue;}
const response=core.artifact.get(attempt.responseArtifactId);assert.equal(response?.sha256,attempt.responseSha256);
const raw=store.artifactStore.read(response);let usage;
try{const value=JSON.parse(raw);usage=value.usage??value.response?.usage;}catch{
for(const line of raw.split(/\r?\n/)){if(!line.startsWith('data:'))continue;const value=line.slice(5).trim();if(!value||value==='[DONE]')continue;
const event=JSON.parse(value);if(event.usage??event.response?.usage)usage=event.usage??event.response.usage;}
}
assert.deepEqual(usage??null,attempt.usage??null,attempt.id);
if(usage)reportedTokens+=usage.total_tokens;else unknownRequests++;
}
assert.equal(attempts.length,manifest.requests);assert.equal(reportedTokens,manifest.reportedTokens);assert.equal(unknownRequests,manifest.unknownRequests);
const names=['react','python','r1-off','r1-on','r2-on','r2-off'];assert.deepEqual(manifest.samples.map(s=>s.name),names);
let reference,onInputs=0,offInputs=0,originalProcesses=0;
const onStrategy=[...core.engine_run.values()].find(j=>j.installation.strategy?.kind===ORIGINAL_PURGE_KIND)?.installation.strategy;assert.ok(onStrategy);
for(const sample of manifest.samples){
const result=read(sample.result),sampleAttempts=attempts.filter(a=>a.sessionId===sample.sessionId),session=owners.get(sample.sessionId);
const baseline=new Map(sample.baseline.map(input=>[input.path,input]));assert.equal(baseline.size,sample.baseline.length);
for(const input of baseline.values()){const raw=fs.readFileSync(file(input.file));assert.equal(raw.length,input.bytes);assert.equal(sha(raw),input.sha256);}
assert.equal(result.success,sample.success);assert.equal(sampleAttempts.length,sample.requests);assert.equal(sample.requests,result.usage.requests);
assert.equal(sampleAttempts.reduce((n,a)=>n+(a.usage?.total_tokens??0),0),sample.reportedTokens);
assert.deepEqual([...new Set(sampleAttempts.map(a=>a.model))],sample.actualModels);assert.deepEqual(sample.actualModels,['claude-opus-5-5']);
const budget=read('receipts/'+sample.name+'-budget.json');assert.equal(budget.contextLimit,32000);assert.equal(budget.requests,24);
assert.equal(budget.tokens,['react','python'].includes(sample.name)?140000:105000);
const task=[...core.task.values()].find(t=>t.id===(result.childId??result.taskId));assert.ok(task);assert.equal(task.usage.totalTokens,sample.reportedTokens);
const consumption=[...core.run.values()].filter(r=>r.taskId===task.id).flatMap(r=>r.consumption??[]);
assert.deepEqual(new Set(consumption.map(c=>c.attemptId)),new Set(sampleAttempts.map(a=>a.id)));
if(sample.contract){
const c=read(sample.contract),comparison={goal:c.goal,sourceTree:c.sourceTree,writable:c.writable,checks:c.checks,budget:c.budget,
driverSha256:c.driverSha256,connectionId:c.model.id,revision:c.model.revision,model:c.model.model,protocol:c.model.protocol,contextLimit:c.model.contextLimit,maxOutputTokens:c.model.maxOutputTokens};
if(reference)assert.deepEqual(comparison,reference,'Same controlled AB/BA baseline and conditions');else reference=comparison;
assert.equal(sample.sourceTree,c.sourceTree);
// The contract records the intentional working copies after the original
// owner staged/unstaged edits; the sealed Git tree is their prior baseline.
const ownerCopies={'owner-staged.txt':'Owner working copy after stage\n','owner-unstaged.txt':'Owner unstaged original\n'};
for(const [name,sha256]of Object.entries(c.sourceFiles))assert.equal(ownerCopies[name]?sha(Buffer.from(ownerCopies[name])):baseline.get(name)?.sha256,sha256,name);
const enabled=sample.name.endsWith('-on');assert.equal(c.enabled,enabled);assert.equal(c.upstreamPluginLoaded,enabled);
for(const job of core.engine_run.values())if(job.sessionId===sample.sessionId){
originalEngineRun(store,session,job);assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true);
if(enabled){assert.equal(job.installation.strategy.kind,ORIGINAL_PURGE_KIND);assert.equal(job.native.originalPurge.readiness.originalApplyCalled,true);assert.equal(job.native.originalPurge.nativeLoads.length,2);originalProcesses++;}
else {assert.equal(job.installation.strategy??null,null);assert.equal(job.native.originalPurge??null,null);}
for(const request of job.requests){const input=verifyEngineModelInput(store,session,job,request),texts=nativeSystemTexts(input.native);
assert.ok(texts.every(t=>!t.includes('[Agent Workbench engineering prompt experiment]')));
if(enabled){assertPurgeModelInput(input.native,job.installation.strategy);onInputs++;}
else {assert.ok(texts.every(t=>!t.includes(onStrategy.originalConfig.postPrompt)));offInputs++;}
}
}
}
if(sample.success){assert.equal(result.reportStatus??result.outcome,['react','python'].includes(sample.name)?'verified_committed':'reviewed_parent_delivered');assert.ok(result.commit);}
}
const restoration=read('receipts/restoration.json');assert.equal(restoration.passed,true);assert.equal(restoration.originalConfigRestored,true);
assert.equal(read('receipts/complete-state-preservation.json').passed,true);
assert.equal(restoration.originalConnectionParametersRestored,true);assert.deepEqual(restoration.actualSelection,restoration.originalSelection);
for(const value of Object.values(restoration.originalRows))assert.equal(value.changed,0);
const audit=read('receipts/process-audit.json');assert.equal(audit.passed,true);assert.equal(audit.inspectorClosed,true);assert.equal(audit.hostActive,false);
assert.ok(audit.processes.every(p=>p.rootExited&&p.treeConfirmed&&p.currentlyExists===false));
console.log(JSON.stringify({passed:true,files:paths.size,bytes,requests:attempts.length,reportedTokens,unknownRequests,onInputs,offInputs,originalProcesses,
engineering:manifest.samples.slice(0,2).map(s=>({name:s.name,success:s.success})),purge:manifest.samples.slice(2).map(s=>({name:s.name,success:s.success,reportedTokens:s.reportedTokens})),
manifestSha256:sha(fs.readFileSync(file('manifest.json'))),scope:'Offline exact source/request/upstream usage/original plugin/cleanup evidence. Six descriptive samples, no general success-rate or unverified fee claim.'}));
+11
View File
@@ -1,5 +1,16 @@
# Agent Workbench:核心代理能力研究与产品计划
## 当前优先 H:1.24.1 真实交付与原版 Purge 重新实测
用户已明确授权新一轮真实模型实验。基线7313fd2,复用已验驱动并冻结新样本;不重跑旧失败,不重启正式宿主,不读/复制Key,不委派子代理。
1. [in_progress] 核对正式版本/原PID/空闲/连接存在位,保存私有原状态;固定新批次来源、驱动、六样本和预算合同。
2. [pending] 新React、Python实际模型交付,逐例最多24请求/140000报告tokens/10分钟,模型自行实现,原测试和人工审批分别计数。
3. [pending] 新同源Node原版Purge对照,顺序off/on、on/off;每例24请求/105000报告tokens/10分钟。整批最多100请求/700000报告tokens,未知消费单列且不自动重放。
4. [pending] 恢复原公开连接参数/池,核对全部原表/导出/工件字节及新增归属;封存请求/用量/来源/进程退出,离线复核和本地Git。
交付闭环与Purge收益分开判定;离线复现不算真实成功,失败停止不能解释成节省;本轮不做新发行或1.25功能扩展。
## 当前优先 G:修完原件已确认问题
用户要求先把已总结问题修完;原生窗口回看不阻塞本次源码修复。沿用planning-with-files及product-ui-ux的真实流程验收,不启动子代理。