diff --git a/docs/LIVE-RETEST-H-PLAN.md b/docs/LIVE-RETEST-H-PLAN.md new file mode 100644 index 00000000..8adc1d5c --- /dev/null +++ b/docs/LIVE-RETEST-H-PLAN.md @@ -0,0 +1,20 @@ +# 1.24.1 真实交付与原版 Purge:H 批次 + +用户已授权重新实测。本批使用正式1.24.1、保留原PID36552及Provider;不重启、不抽取或复制Key,不续跑旧失败Task。G修复是本批执行基线,E结果保留,不合并分母。本批不扩展1.25功能或重新打包发行。 + +| 新样本 | 实际验收 | 上限 | +| --- | --- | --- | +| React | 模型自行修复四模块,原9项测试、类型检查、构建、真实桌面/窄屏交互及审查Git | 24实际请求、140000报告tokens、10分钟 | +| Python | 模型自行修复三个模块,原6项测试、编译及审查Git | 同上 | +| Node AB | 同一三模块收支报告,先关闭再启用原版受限Purge | 每例24请求、105000报告tokens、10分钟 | +| Node BA | 相同需求、基线、工具、检查和驱动,先开启再关闭 | 同上 | + +整批六例串行,最多100实际请求/700000报告tokens。当前已配置Claude连接固定Chat协议、32K容量、2048输出上限;无自动切池。容量经正规API临时调整并保存修订,结束恢复公开参数和原池。原生及宿主按真实已知用量和未知消费预留拒绝超额派发;在途超额、未知usage和网络失败照实记录,不自动重放或重置账本。服务公有模型中心只返回最近100尝试,预算和最终分母改用只读完整modelAttempts,不读取凭据字段。 + +基线沿用E自有工程的业务需求与原测试,创建全新项目、工作区和Task。模型只收一次任务、允许路径和流程合同,自己读源码并提出实现;验收程序仅批准匹配的原检查、核对模型封存批次和确认Git。审批、应用、显式续接、父工程核对分别计数;实现提示和人工代写均为零。旧已修改工程不会作为新基线。 + +原版开启为Purge1.1.78、`purge-original-restricted-v1`、Harness0.2.1-rc.2;关闭为同Harness但不加载原版入口或独立提示。原版自动更新、宿主修改、原操作工具、自动重试及继续关闭,保留真实提示/内部钩子。固定来源、每请求实际输入、原生模块加载和进程树退出逐例核对。这个实验只评价此受限工程配置,不是通用模型“破甲”或权重提升的证明。 + +完整批次交接和源码保留由真实请求与工具账本观察,不能把G离线回放当实测成功。来源漂移、未知操作、预算耗尽或不能确认进程退出即停止本例;独立后续组仅在剩余额度、无活动和来源一致时开始。不得以追加提示、扩大预算、改测试或同失败Task重跑制造成功。 + +交付结果与Purge收益单独报告:同成功标准下才比较消耗,更早失败不算节省;四例只能给描述性结论,不编统计显著性或收费节省。封存原请求/上游usage、源码基线、原失败、检查/页面/Git证据及监督次数。所有原会话导出、表行、工件原字节和模型参数恢复逐项核对,新追加归属明确;私有原用户快照不入Git,按范围本地提交、不推远端。 diff --git a/findings.md b/findings.md index a4fc0f0e..8cc0bdb9 100644 --- a/findings.md +++ b/findings.md @@ -1,5 +1,9 @@ # 汇总发现 +## H 实测设计依据 + +G已修完整批次同轮交接和已读小源码稳定保留,需由新真实样本验证实际工程收益。E驱动可复用工程/工具审查和AB/BA监督合同,但其prepare包含临时处理器激活、零窗口断言和旧源码身份,不能直接运行;H应只读确认正式1.24.1并冻结当前运行载荷。旧E52请求/621486报告tokens和全部失败保持,不改分母;原版开启11份输入和关闭17份输入的来源核对不是收益证明。 + ## F 固定安装与后续边界 新候选独立源码重建22/22、三归档、实际便携和12组隔离安装通过后,正式已从临时QA根切换至固定versions目录;原PID/Provider与当前42项目/58会话/2850工件及全部原表/导出/模型中心精确保持。原操作675.500秒单列,不沿用旧速度。69份原件离线通过,正式原生窗口回看因物理Escape停止仍未验收。 diff --git a/progress.md b/progress.md index 9c557ffe..5ae559c8 100644 --- a/progress.md +++ b/progress.md @@ -1,5 +1,9 @@ # 统一执行进度 +## H 新真实模型实测启动 + +用户要求重新实测真实交付与Purge收益。起点7313fd2、工作区干净;已回顾G修复和E原结果(Python通过,React与四对照未交付),定位原驱动。采用六个新有界样本,公开连接经正规API临时配置,原Provider/内存Key与旧数据保持;尚无真实请求。旧E激活临时处理器的步骤本轮不复用,直接核对已正式固定的1.24.1。本轮使用planning-with-files,不开启子代理。 + ## G 已确认问题修复启动 用户明确优先修完已总结的问题。基线876df5f干净;先复现完整审查批次仍running和源码在16K软目标下逐份挤出。沿用既有规划与产品流程验收技能,不开子代理,不读取Key或重放旧实测Task;F正式已固定安装,旧Esc停止不作为当前源码修复的等待条件。首批读取超出输出上限,截断内容不作完整审查,后续缩小范围。 diff --git a/scripts/verify-live-retest-h-evidence.mjs b/scripts/verify-live-retest-h-evidence.mjs new file mode 100644 index 00000000..7a9d3d42 --- /dev/null +++ b/scripts/verify-live-retest-h-evidence.mjs @@ -0,0 +1,97 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import {createHash} from 'node:crypto'; +import {requestMetrics} from '../src/shared/contracts/model-request.mjs'; +import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs'; +import {assertPurgeModelInput,nativeSystemTexts,ORIGINAL_PURGE_KIND} from '../src/core/engines/purge-strategy.mjs'; + +// Offline evidence only. No Runtime/Provider, model requests or plugin loading. +const root=path.resolve('docs/live-original-purge-h-evidence'),sha=raw=>createHash('sha256').update(raw).digest('hex'); +const file=relative=>{const full=path.resolve(root,relative);assert.ok(full.startsWith(root+path.sep));return full;}; +const read=relative=>JSON.parse(fs.readFileSync(file(relative))),manifest=read('manifest.json'); +assert.equal(manifest.schemaVersion,1);assert.equal(manifest.slice,'live-original-purge-h'); +const paths=new Set();let bytes=0; +for(const e of manifest.entries){assert.ok(!paths.has(e.path));paths.add(e.path);const raw=fs.readFileSync(file(e.path));assert.equal(raw.length,e.bytes,e.path);assert.equal(sha(raw),e.sha256,e.path);bytes+=raw.length;} +assert.equal(paths.size,manifest.count);assert.equal(bytes,manifest.bytes); +assert.equal(sha(fs.readFileSync(manifest.dManifest)),manifest.dManifestSha256); +const d=JSON.parse(fs.readFileSync(manifest.dManifest)),identity=read('receipts/runtime-identity.json'); +const release=read('receipts/release-identity.json'); +assert.equal(sha(fs.readFileSync(file('receipts/release-identity.json'))),identity.releaseIdentitySha256); +assert.equal(identity.manifestSha256,release.manifestSha256);assert.equal(identity.runtime,release.installedRoot); +assert.equal(identity.sourceCommit,release.sourceCommit);assert.equal(identity.sourceCommit,manifest.sourceCommit); +const installed=read('receipts/release-manifest.json');assert.equal(sha(fs.readFileSync(file('receipts/release-manifest.json'))),release.manifestSha256); +for(const input of identity.files){assert.deepEqual(input,installed.files.find(f=>f.path===input.path),input.path);} +assert.equal(identity.files.filter(f=>f.path.startsWith('dist-')).length,22); +const freeze=read('receipts/driver-freeze.json'); +for(const input of freeze.files){const sealed=input.path.startsWith('.qa/')?'drivers/'+path.basename(input.path):null; + if(sealed)assert.equal(sha(fs.readFileSync(file(sealed))),input.sha256,input.path);} +const records=read('receipts/owned-records.json'),owners=new Map(read('receipts/owned-sessions.json').map(s=>[s.id,s])),core={}; +for(const row of records){const value=JSON.parse(row.payload);assert.ok(owners.has(value.sessionId),row.id);(core[row.kind]??=new Map()).set(value.id,value);} +const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,core.artifact.size); +const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId,kind+'/'+id);return value;}, + artifactStore:{read(a){return this.readBytes(a).toString('utf8');},readBytes(a){const input=artifacts.get(a.id),raw=fs.readFileSync(file(input.file));assert.equal(sha(raw),a.sha256);assert.equal(raw.length,a.bytes);return raw;}}, + savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(t=>t.sessionId===sessionId&&t.callId===callId);assert.ok(tool);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}}; +const attempts=read('receipts/all-model-attempts.json'),attemptIds=new Set(attempts.map(a=>a.id));assert.equal(attemptIds.size,attempts.length); +let reportedTokens=0,unknownRequests=0; +for(const attempt of attempts){ + const request=core.artifact.get(attempt.requestArtifactId);assert.ok(request);assert.equal(request.sha256,attempt.requestSha256); + const input=JSON.parse(store.artifactStore.read(request));assert.equal(requestMetrics(input).sha256,attempt.requestSha256);assert.equal(input.model,attempt.model); + if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknownRequests++;continue;} + const response=core.artifact.get(attempt.responseArtifactId);assert.equal(response?.sha256,attempt.responseSha256); + const raw=store.artifactStore.read(response);let usage; + try{const value=JSON.parse(raw);usage=value.usage??value.response?.usage;}catch{ + for(const line of raw.split(/\r?\n/)){if(!line.startsWith('data:'))continue;const value=line.slice(5).trim();if(!value||value==='[DONE]')continue; + const event=JSON.parse(value);if(event.usage??event.response?.usage)usage=event.usage??event.response.usage;} + } + assert.deepEqual(usage??null,attempt.usage??null,attempt.id); + if(usage)reportedTokens+=usage.total_tokens;else unknownRequests++; +} +assert.equal(attempts.length,manifest.requests);assert.equal(reportedTokens,manifest.reportedTokens);assert.equal(unknownRequests,manifest.unknownRequests); +const names=['react','python','r1-off','r1-on','r2-on','r2-off'];assert.deepEqual(manifest.samples.map(s=>s.name),names); +let reference,onInputs=0,offInputs=0,originalProcesses=0; +const onStrategy=[...core.engine_run.values()].find(j=>j.installation.strategy?.kind===ORIGINAL_PURGE_KIND)?.installation.strategy;assert.ok(onStrategy); +for(const sample of manifest.samples){ + const result=read(sample.result),sampleAttempts=attempts.filter(a=>a.sessionId===sample.sessionId),session=owners.get(sample.sessionId); + const baseline=new Map(sample.baseline.map(input=>[input.path,input]));assert.equal(baseline.size,sample.baseline.length); + for(const input of baseline.values()){const raw=fs.readFileSync(file(input.file));assert.equal(raw.length,input.bytes);assert.equal(sha(raw),input.sha256);} + assert.equal(result.success,sample.success);assert.equal(sampleAttempts.length,sample.requests);assert.equal(sample.requests,result.usage.requests); + assert.equal(sampleAttempts.reduce((n,a)=>n+(a.usage?.total_tokens??0),0),sample.reportedTokens); + assert.deepEqual([...new Set(sampleAttempts.map(a=>a.model))],sample.actualModels);assert.deepEqual(sample.actualModels,['claude-opus-5-5']); + const budget=read('receipts/'+sample.name+'-budget.json');assert.equal(budget.contextLimit,32000);assert.equal(budget.requests,24); + assert.equal(budget.tokens,['react','python'].includes(sample.name)?140000:105000); + const task=[...core.task.values()].find(t=>t.id===(result.childId??result.taskId));assert.ok(task);assert.equal(task.usage.totalTokens,sample.reportedTokens); + const consumption=[...core.run.values()].filter(r=>r.taskId===task.id).flatMap(r=>r.consumption??[]); + assert.deepEqual(new Set(consumption.map(c=>c.attemptId)),new Set(sampleAttempts.map(a=>a.id))); + if(sample.contract){ + const c=read(sample.contract),comparison={goal:c.goal,sourceTree:c.sourceTree,writable:c.writable,checks:c.checks,budget:c.budget, + driverSha256:c.driverSha256,connectionId:c.model.id,revision:c.model.revision,model:c.model.model,protocol:c.model.protocol,contextLimit:c.model.contextLimit,maxOutputTokens:c.model.maxOutputTokens}; + if(reference)assert.deepEqual(comparison,reference,'Same controlled AB/BA baseline and conditions');else reference=comparison; + assert.equal(sample.sourceTree,c.sourceTree); + // The contract records the intentional working copies after the original + // owner staged/unstaged edits; the sealed Git tree is their prior baseline. + const ownerCopies={'owner-staged.txt':'Owner working copy after stage\n','owner-unstaged.txt':'Owner unstaged original\n'}; + for(const [name,sha256]of Object.entries(c.sourceFiles))assert.equal(ownerCopies[name]?sha(Buffer.from(ownerCopies[name])):baseline.get(name)?.sha256,sha256,name); + const enabled=sample.name.endsWith('-on');assert.equal(c.enabled,enabled);assert.equal(c.upstreamPluginLoaded,enabled); + for(const job of core.engine_run.values())if(job.sessionId===sample.sessionId){ + originalEngineRun(store,session,job);assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true); + if(enabled){assert.equal(job.installation.strategy.kind,ORIGINAL_PURGE_KIND);assert.equal(job.native.originalPurge.readiness.originalApplyCalled,true);assert.equal(job.native.originalPurge.nativeLoads.length,2);originalProcesses++;} + else {assert.equal(job.installation.strategy??null,null);assert.equal(job.native.originalPurge??null,null);} + for(const request of job.requests){const input=verifyEngineModelInput(store,session,job,request),texts=nativeSystemTexts(input.native); + assert.ok(texts.every(t=>!t.includes('[Agent Workbench engineering prompt experiment]'))); + if(enabled){assertPurgeModelInput(input.native,job.installation.strategy);onInputs++;} + else {assert.ok(texts.every(t=>!t.includes(onStrategy.originalConfig.postPrompt)));offInputs++;} + } + } + } + if(sample.success){assert.equal(result.reportStatus??result.outcome,['react','python'].includes(sample.name)?'verified_committed':'reviewed_parent_delivered');assert.ok(result.commit);} +} +const restoration=read('receipts/restoration.json');assert.equal(restoration.passed,true);assert.equal(restoration.originalConfigRestored,true); +assert.equal(read('receipts/complete-state-preservation.json').passed,true); +assert.equal(restoration.originalConnectionParametersRestored,true);assert.deepEqual(restoration.actualSelection,restoration.originalSelection); +for(const value of Object.values(restoration.originalRows))assert.equal(value.changed,0); +const audit=read('receipts/process-audit.json');assert.equal(audit.passed,true);assert.equal(audit.inspectorClosed,true);assert.equal(audit.hostActive,false); +assert.ok(audit.processes.every(p=>p.rootExited&&p.treeConfirmed&&p.currentlyExists===false)); +console.log(JSON.stringify({passed:true,files:paths.size,bytes,requests:attempts.length,reportedTokens,unknownRequests,onInputs,offInputs,originalProcesses, + engineering:manifest.samples.slice(0,2).map(s=>({name:s.name,success:s.success})),purge:manifest.samples.slice(2).map(s=>({name:s.name,success:s.success,reportedTokens:s.reportedTokens})), + manifestSha256:sha(fs.readFileSync(file('manifest.json'))),scope:'Offline exact source/request/upstream usage/original plugin/cleanup evidence. Six descriptive samples, no general success-rate or unverified fee claim.'})); diff --git a/task_plan.md b/task_plan.md index 5e33c0e6..98733b67 100644 --- a/task_plan.md +++ b/task_plan.md @@ -1,5 +1,16 @@ # Agent Workbench:核心代理能力研究与产品计划 +## 当前优先 H:1.24.1 真实交付与原版 Purge 重新实测 + +用户已明确授权新一轮真实模型实验。基线7313fd2,复用已验驱动并冻结新样本;不重跑旧失败,不重启正式宿主,不读/复制Key,不委派子代理。 + +1. [in_progress] 核对正式版本/原PID/空闲/连接存在位,保存私有原状态;固定新批次来源、驱动、六样本和预算合同。 +2. [pending] 新React、Python实际模型交付,逐例最多24请求/140000报告tokens/10分钟,模型自行实现,原测试和人工审批分别计数。 +3. [pending] 新同源Node原版Purge对照,顺序off/on、on/off;每例24请求/105000报告tokens/10分钟。整批最多100请求/700000报告tokens,未知消费单列且不自动重放。 +4. [pending] 恢复原公开连接参数/池,核对全部原表/导出/工件字节及新增归属;封存请求/用量/来源/进程退出,离线复核和本地Git。 + +交付闭环与Purge收益分开判定;离线复现不算真实成功,失败停止不能解释成节省;本轮不做新发行或1.25功能扩展。 + ## 当前优先 G:修完原件已确认问题 用户要求先把已总结问题修完;原生窗口回看不阻塞本次源码修复。沿用planning-with-files及product-ui-ux的真实流程验收,不启动子代理。