Files
Agent-Workbench/docs/model-selection-k-evidence/source-bytes/scripts/verify-original-purge-production-evidence.mjs
T
2026-11-03 15:37:04 +08:00

88 lines
8.3 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {createHash} from 'node:crypto';
import {execFileSync} from 'node:child_process';
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
import {assertPurgeModelInput} from '../src/core/engines/purge-strategy.mjs';
// Offline only: inspect original bytes and pure contracts, never instantiate
// a Provider/Runtime, load an upstream plugin or send a model request.
const root=path.resolve('docs/original-purge-production-evidence'),sha=raw=>createHash('sha256').update(raw).digest('hex');
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
const read=relative=>JSON.parse(fs.readFileSync(inside(relative))),raw=fs.readFileSync(inside('manifest.json')),manifest=JSON.parse(raw);
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.equal(manifest.formalServiceChanged,false);
let bytes=0;const paths=new Set();
for(const entry of manifest.entries){assert.ok(!paths.has(entry.path));paths.add(entry.path);const data=fs.readFileSync(inside(entry.path));
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;}
assert.equal(paths.size,manifest.count);assert.equal(bytes,manifest.bytes);
for(const input of manifest.sourceInputs){
const git=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
assert.equal(sha(git),input.gitBlobSha256,input.path);
if(input.snapshot){const bytes=fs.readFileSync(inside(input.snapshot));assert.equal(sha(bytes),input.sha256);assert.equal(bytes.length,input.bytes);
assert.equal(bytes.toString('utf8').replaceAll('\r\n','\n'),git.toString('utf8').replaceAll('\r\n','\n'));}
else assert.equal(sha(git),input.sha256,input.path);
}
const receipt=name=>fs.readFileSync(inside(manifest.receipts.find(value=>value.name===name).file),'utf8');
assert.match(receipt('full'),new RegExp('tests '+manifest.fullSummary.tests+'\\b'));
assert.match(receipt('full'),new RegExp('pass '+manifest.fullSummary.passed+'\\b'));
assert.match(receipt('full'),/fail 5\b/);assert.equal(manifest.fullSummary.failed,5);
assert.equal(manifest.fullSummary.tests,manifest.fullSummary.passed+manifest.fullSummary.failed);
// Preserve the original failed whole run. Subsequent evidence is the same
// Python environment, two repaired migrations and legacy fixture corrections.
assert.deepEqual(manifest.fullSummary.failedNames.slice().sort(),[
'real pytest pass, failure, skip, collection errors and teardown errors use collected case outcomes',
'a missing interpreter and repeated pytest item objects cannot create a false verification pass',
'a directly started compiled host upgrades twice with the same PID, sessions, applied bytes and volatile provider',
'the running 0.1.0 process upgrades in place and retains its project, session, files and volatile model connection',
'a 1.6.1 full-provenance sent view remains verifiable after introducing the compact view'
].sort());
assert.deepEqual(manifest.correctedSuites.map(value=>value.name),['python-final','upgrade-final','context-final']);
for(const {name,tests:count,passed,failed} of manifest.correctedSuites){
assert.ok(count>0);assert.equal(passed,count);assert.equal(failed,0);
assert.match(receipt(name),new RegExp('tests '+count+'\\b'));assert.match(receipt(name),new RegExp('pass '+count+'\\b'));assert.match(receipt(name),/fail 0\b/);
}
assert.equal(manifest.correctedSuites[0].tests,15);assert.equal(manifest.correctedSuites[1].tests,2);
const frozen=read(manifest.fullRunInputs),amended=[];
for(const input of frozen.files){const current=manifest.sourceInputs.find(value=>value.path===input.path)??manifest.buildInputs.find(value=>value.path===input.path);
assert.ok(current,input.path);if(current.sha256!==input.sha256)amended.push(input.path);}
assert.deepEqual(amended,manifest.amendedAcceptanceOrUpgradeInputs);
for(const file of amended)assert.ok(['scripts/upgrade-running.mjs','scripts/verify-controlled-projects.mjs','scripts/verify-original-purge-host.mjs','tests/working-set.test.mjs'].includes(file),file);
const preservation=JSON.parse(receipt('formal-preservation'));
assert.equal(preservation.passed,true);assert.equal(preservation.qaRealModelRequests,0);assert.equal(preservation.selectedPoolPreserved,true);
for(const table of Object.values(preservation.originalRows))assert.equal(table.changed,0);
assert.match(receipt('build'),/Server bundle built\./);
const runtime=read(manifest.runtime);assert.equal(runtime.passed,true);assert.equal(runtime.productionAdapter,true);
assert.deepEqual(runtime.cases.map(value=>value.scenario),['completed','failed-no-retry','host-denied-tool']);
assert.equal(runtime.cases[0].exit.code,0);assert.deepEqual(runtime.cases.map(value=>value.bridgeCalls),[1,1,2]);
const host=read(manifest.host.result);assert.equal(host.passed,true);assert.equal(host.results.length,4);
for(const sample of manifest.host.samples){const value=read(sample.result);assert.equal(value.passed,true);assert.equal(value.job.native.cleanup.treeConfirmed,true);
if(value.scenario==='reported-budget-overrun'){assert.equal(value.task.usage.totalTokens,55017);assert.equal(value.run.stopReason,'token_limit');assert.equal(value.modelRequests.length,1);}
if(value.scenario==='stop-in-flight'){assert.equal(value.run.usage.unknownRequests,1);assert.equal(value.task.usage.unknownRequests,1);
assert.equal(value.modelRequests.length,2);assert.equal(value.resumedJob.taskId,value.job.taskId);assert.notEqual(value.resumedJob.runId,value.job.runId);
assert.equal(value.resumedJob.native.cleanup.treeConfirmed,true);assert.equal(value.task.usage.totalTokens,168);}
if(value.scenario==='source-changed-at-approval')assert.match(value.job.error,/已变化/);
}
const ui=read(manifest.ui.result);assert.equal(ui.passed,true);assert.equal(ui.cases.length,4);assert.deepEqual(ui.pageErrors,[]);assert.deepEqual(ui.unexpectedFailures,[]);
assert.equal(manifest.ui.screenshots.length,4);
let requests=0,artifacts=0;
for(const sample of manifest.engineering){
const result=read(sample.result),core={},owners=new Map(read(sample.sessions).map(value=>[value.id,value]));
for(const row of read(sample.records)){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
const inputs=new Map(sample.artifacts.map(value=>[value.id,value]));artifacts+=inputs.size;
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId);return value;},
artifactStore:{read(artifact){const data=fs.readFileSync(inside(inputs.get(artifact.id).file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
assert.equal(result.success,true);assert.equal(result.metrics.contextLimit,32000);assert.equal(result.parentCheck.results[0].tests.passed,9);
assert.ok(result.checks.some(job=>job.status==='failed'&&job.results[0].tests.failed===9));
for(const job of core.engine_run.values()){
const session=owners.get(job.sessionId);originalEngineRun(store,session,job);assert.equal(job.status,'completed');assert.equal(job.native.cleanup.treeConfirmed,true);
const original=job.native.originalPurge;assert.equal(original.readiness.originalApplyCalled,true);assert.equal(original.nativeLoads.length,2);assert.ok(original.durabilityReceipts.length);
assert.equal(job.installation.strategy.kind,'purge-original-restricted-v1');
for(const request of job.requests){const input=verifyEngineModelInput(store,session,job,request);assertPurgeModelInput(input.native,job.installation.strategy);requests++;}
}
}
assert.deepEqual(manifest.engineering.map(value=>value.name),['chat','responses']);
console.log(JSON.stringify({passed:true,sourceCommit:manifest.sourceCommit,files:paths.size,bytes,fullSummary:manifest.fullSummary,artifacts,requests,
manifestSha256:sha(raw),realModelRequests:0,scope:'Offline original source/receipt/request/usage/cleanup checks. Actual reasoning and Purge benefit remain unmeasured.'}));