Files
Agent-Workbench/scripts/verify-live-retest-e-evidence.mjs
2026-11-02 23:13:19 +08:00

93 lines
9.6 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {createHash} from 'node:crypto';
import {requestMetrics} from '../src/shared/contracts/model-request.mjs';
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
import {assertPurgeModelInput,nativeSystemTexts,ORIGINAL_PURGE_KIND} from '../src/core/engines/purge-strategy.mjs';
// Offline evidence only. No Runtime/Provider, model requests or plugin loading.
const root=path.resolve('docs/live-original-purge-e-evidence'),sha=raw=>createHash('sha256').update(raw).digest('hex');
const file=relative=>{const full=path.resolve(root,relative);assert.ok(full.startsWith(root+path.sep));return full;};
const read=relative=>JSON.parse(fs.readFileSync(file(relative))),manifest=read('manifest.json');
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.slice,'live-original-purge-e');
const paths=new Set();let bytes=0;
for(const e of manifest.entries){assert.ok(!paths.has(e.path));paths.add(e.path);const raw=fs.readFileSync(file(e.path));assert.equal(raw.length,e.bytes,e.path);assert.equal(sha(raw),e.sha256,e.path);bytes+=raw.length;}
assert.equal(paths.size,manifest.count);assert.equal(bytes,manifest.bytes);
assert.equal(sha(fs.readFileSync(manifest.dManifest)),manifest.dManifestSha256);
const d=JSON.parse(fs.readFileSync(manifest.dManifest)),identity=read('receipts/runtime-identity.json');
for(const build of d.buildInputs){const input=identity.files.find(f=>f.path===build.path);assert.equal(input?.sha256,build.sha256,build.path);}
assert.equal(identity.dSourceCommit,d.sourceCommit);assert.equal(identity.sourceCommit,manifest.sourceCommit);
const freeze=read('receipts/driver-freeze.json');
for(const input of freeze.files){const sealed=input.path.startsWith('.qa/')?'drivers/'+path.basename(input.path):null;
if(sealed)assert.equal(sha(fs.readFileSync(file(sealed))),input.sha256,input.path);}
const records=read('receipts/owned-records.json'),owners=new Map(read('receipts/owned-sessions.json').map(s=>[s.id,s])),core={};
for(const row of records){const value=JSON.parse(row.payload);assert.ok(owners.has(value.sessionId),row.id);(core[row.kind]??=new Map()).set(value.id,value);}
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,core.artifact.size);
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId,kind+'/'+id);return value;},
artifactStore:{read(a){return this.readBytes(a).toString('utf8');},readBytes(a){const input=artifacts.get(a.id),raw=fs.readFileSync(file(input.file));assert.equal(sha(raw),a.sha256);assert.equal(raw.length,a.bytes);return raw;}},
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(t=>t.sessionId===sessionId&&t.callId===callId);assert.ok(tool);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
const attempts=read('receipts/all-model-attempts.json'),attemptIds=new Set(attempts.map(a=>a.id));assert.equal(attemptIds.size,attempts.length);
let reportedTokens=0,unknownRequests=0;
for(const attempt of attempts){
const request=core.artifact.get(attempt.requestArtifactId);assert.ok(request);assert.equal(request.sha256,attempt.requestSha256);
const input=JSON.parse(store.artifactStore.read(request));assert.equal(requestMetrics(input).sha256,attempt.requestSha256);assert.equal(input.model,attempt.model);
if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknownRequests++;continue;}
const response=core.artifact.get(attempt.responseArtifactId);assert.equal(response?.sha256,attempt.responseSha256);
const raw=store.artifactStore.read(response);let usage;
try{const value=JSON.parse(raw);usage=value.usage??value.response?.usage;}catch{
for(const line of raw.split(/\r?\n/)){if(!line.startsWith('data:'))continue;const value=line.slice(5).trim();if(!value||value==='[DONE]')continue;
const event=JSON.parse(value);if(event.usage??event.response?.usage)usage=event.usage??event.response.usage;}
}
assert.deepEqual(usage??null,attempt.usage??null,attempt.id);
if(usage)reportedTokens+=usage.total_tokens;else unknownRequests++;
}
assert.equal(attempts.length,manifest.requests);assert.equal(reportedTokens,manifest.reportedTokens);assert.equal(unknownRequests,manifest.unknownRequests);
const names=['react','python','r1-off','r1-on','r2-on','r2-off'];assert.deepEqual(manifest.samples.map(s=>s.name),names);
let reference,onInputs=0,offInputs=0,originalProcesses=0;
const onStrategy=[...core.engine_run.values()].find(j=>j.installation.strategy?.kind===ORIGINAL_PURGE_KIND)?.installation.strategy;assert.ok(onStrategy);
for(const sample of manifest.samples){
const result=read(sample.result),sampleAttempts=attempts.filter(a=>a.sessionId===sample.sessionId),session=owners.get(sample.sessionId);
const baseline=new Map(sample.baseline.map(input=>[input.path,input]));assert.equal(baseline.size,sample.baseline.length);
for(const input of baseline.values()){const raw=fs.readFileSync(file(input.file));assert.equal(raw.length,input.bytes);assert.equal(sha(raw),input.sha256);}
assert.equal(result.success,sample.success);assert.equal(sampleAttempts.length,sample.requests);assert.equal(sample.requests,result.usage.requests);
assert.equal(sampleAttempts.reduce((n,a)=>n+(a.usage?.total_tokens??0),0),sample.reportedTokens);
assert.deepEqual([...new Set(sampleAttempts.map(a=>a.model))],sample.actualModels);assert.deepEqual(sample.actualModels,['claude-opus-5-5']);
const budget=read('receipts/'+sample.name+'-budget.json');assert.equal(budget.contextLimit,32000);assert.equal(budget.requests,24);
assert.equal(budget.tokens,['react','python'].includes(sample.name)?140000:105000);
const task=[...core.task.values()].find(t=>t.id===(result.childId??result.taskId));assert.ok(task);assert.equal(task.usage.totalTokens,sample.reportedTokens);
const consumption=[...core.run.values()].filter(r=>r.taskId===task.id).flatMap(r=>r.consumption??[]);
assert.deepEqual(new Set(consumption.map(c=>c.attemptId)),new Set(sampleAttempts.map(a=>a.id)));
if(sample.contract){
const c=read(sample.contract),comparison={goal:c.goal,sourceTree:c.sourceTree,writable:c.writable,checks:c.checks,budget:c.budget,
driverSha256:c.driverSha256,connectionId:c.model.id,revision:c.model.revision,model:c.model.model,protocol:c.model.protocol,contextLimit:c.model.contextLimit,maxOutputTokens:c.model.maxOutputTokens};
if(reference)assert.deepEqual(comparison,reference,'Same controlled AB/BA baseline and conditions');else reference=comparison;
assert.equal(sample.sourceTree,c.sourceTree);
// The contract records the intentional working copies after the original
// owner staged/unstaged edits; the sealed Git tree is their prior baseline.
const ownerCopies={'owner-staged.txt':'Owner working copy after stage\n','owner-unstaged.txt':'Owner unstaged original\n'};
for(const [name,sha256]of Object.entries(c.sourceFiles))assert.equal(ownerCopies[name]?sha(Buffer.from(ownerCopies[name])):baseline.get(name)?.sha256,sha256,name);
const enabled=sample.name.endsWith('-on');assert.equal(c.enabled,enabled);assert.equal(c.upstreamPluginLoaded,enabled);
for(const job of core.engine_run.values())if(job.sessionId===sample.sessionId){
originalEngineRun(store,session,job);assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true);
if(enabled){assert.equal(job.installation.strategy.kind,ORIGINAL_PURGE_KIND);assert.equal(job.native.originalPurge.readiness.originalApplyCalled,true);assert.equal(job.native.originalPurge.nativeLoads.length,2);originalProcesses++;}
else {assert.equal(job.installation.strategy??null,null);assert.equal(job.native.originalPurge??null,null);}
for(const request of job.requests){const input=verifyEngineModelInput(store,session,job,request),texts=nativeSystemTexts(input.native);
assert.ok(texts.every(t=>!t.includes('[Agent Workbench engineering prompt experiment]')));
if(enabled){assertPurgeModelInput(input.native,job.installation.strategy);onInputs++;}
else {assert.ok(texts.every(t=>!t.includes(onStrategy.originalConfig.postPrompt)));offInputs++;}
}
}
}
if(sample.success){assert.equal(result.reportStatus??result.outcome,['react','python'].includes(sample.name)?'verified_committed':'reviewed_parent_delivered');assert.ok(result.commit);}
}
const restoration=read('receipts/restoration.json');assert.equal(restoration.passed,true);assert.equal(restoration.originalConfigRestored,true);
assert.equal(read('receipts/complete-state-preservation.json').passed,true);
assert.equal(restoration.originalConnectionParametersRestored,true);assert.deepEqual(restoration.actualSelection,restoration.originalSelection);
for(const value of Object.values(restoration.originalRows))assert.equal(value.changed,0);
const audit=read('receipts/process-audit.json');assert.equal(audit.passed,true);assert.equal(audit.inspectorClosed,true);assert.equal(audit.hostActive,false);
assert.ok(audit.processes.every(p=>p.rootExited&&p.treeConfirmed&&p.currentlyExists===false));
console.log(JSON.stringify({passed:true,files:paths.size,bytes,requests:attempts.length,reportedTokens,unknownRequests,onInputs,offInputs,originalProcesses,
engineering:manifest.samples.slice(0,2).map(s=>({name:s.name,success:s.success})),purge:manifest.samples.slice(2).map(s=>({name:s.name,success:s.success,reportedTokens:s.reportedTokens})),
manifestSha256:sha(fs.readFileSync(file('manifest.json'))),scope:'Offline exact source/request/upstream usage/original plugin/cleanup evidence. Six descriptive samples, no general success-rate or unverified fee claim.'}));