Files
Agent-Workbench/scripts/verify-live-pilot-evidence.mjs

102 lines
9.5 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { completionBody, requestMetrics } from '../src/shared/contracts/model-request.mjs';
import { nativeMessagesToChat } from '../src/shared/contracts/engine-execution.mjs';
import { nativeSystemTexts } from '../src/core/engines/purge-strategy.mjs';
if (process.argv.includes('--help')) {
console.log('node scripts/verify-live-pilot-evidence.mjs — verify sealed 1.24 pilot bytes, source identities, original request/SSE usage and native input rejection; offline, no model calls.');
process.exit(0);
}
const root=path.resolve('docs/live-model-purge-1.24-evidence');
const read=file=>JSON.parse(fs.readFileSync(file,'utf8').replace(/^\uFEFF/,''));
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
const manifest=read(path.join(root,'manifest.json')),entries=new Map(manifest.entries.map(x=>[x.path,x]));
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.formalVersion,'1.24.0');assert.equal(manifest.sampleCount,6);
assert.equal(manifest.upstreamPluginLoaded,false);assert.equal(manifest.originalPluginBenefit,'not_tested');
assert.equal(entries.size,manifest.count);
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
const from=relative=>{assert.ok(entries.has(relative),'File belongs to manifest: '+relative);return read(inside(relative));};
let bytes=0;
for(const entry of entries.values()){
const file=inside(entry.path);assert.equal(fs.lstatSync(file).isFile(),true,entry.path);
const original=fs.readFileSync(file);assert.equal(original.length,entry.bytes,entry.path);assert.equal(sha(original),entry.sha256,entry.path);bytes+=original.length;
}
assert.equal(bytes,manifest.bytes);
const files=[];
function visit(relative='') {
for(const entry of fs.readdirSync(path.join(root,relative),{withFileTypes:true})){
const next=path.posix.join(relative,entry.name);assert.equal(entry.isSymbolicLink(),false,next);
if(entry.isDirectory())visit(next);else{assert.equal(entry.isFile(),true,next);files.push(next);}
}
}
visit();assert.deepEqual(files.sort(),[...entries.keys(),'manifest.json'].sort(),'Exact sealed file scope');
assert.match(manifest.sourceCommit,/^[a-f0-9]{40}$/);
for(const input of manifest.sourceInputs){
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$/);assert.ok(!input.path.split('/').includes('..'));
const data=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:8*1024*1024});
assert.equal(data.length,input.bytes,input.path);assert.equal(sha(data),input.sha256,'Historical source used by the sealed pilot: '+input.path);
}
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,381);
for(const a of artifacts.values()){assert.equal(entries.get(a.file)?.sha256,a.sha256,a.id);assert.equal(entries.get(a.file)?.bytes,a.bytes,a.id);}
const setup=from('receipts/setup-public-receipt.json'),analysis=from('receipts/analysis.json'),preservation=from('receipts/state-preservation.json');
const complete=from('receipts/complete-owned-artifact-receipts.json'),reconciliation=from('receipts/request-and-source-reconciliation.json');
assert.equal(complete.newUniqueFiles,342);assert.equal(complete.artifactRecords,381);assert.equal(complete.totalArtifactBytes,6426284);
assert.equal(preservation.passed,true);assert.equal(preservation.original.exportDigestsUnchanged,33);assert.equal(preservation.original.artifactFilesUnchanged,1780);assert.equal(preservation.original.modelAttemptsUnchanged,70);
assert.deepEqual(preservation.formal.selected,setup.originalSelection);assert.deepEqual(preservation.formal.active,[]);assert.equal(preservation.formal.pid,manifest.formalPid);
const audit=from('receipts/process-audit.json');assert.ok(audit.processes.every(x=>x.rootExited&&x.treeConfirmed&&!x.currentlyExists));assert.deepEqual(audit.debug9229,[]);
let requests=0,reportedTokens=0,unknown=0,sseResponses=0,approvals=0,applications=0,continuations=0;let contractReference;
for(const sample of manifest.samples){
const result=from(sample.result),attempts=from(sample.attempts);assert.equal(result.success,false,'Original sample must retain failed acceptance: '+sample.name);
approvals+=(result.decisions??[]).length;applications+=(result.applications??[]).length;continuations+=(result.continuations??result.restarts??[]).length;
assert.equal(attempts.length,result.usage.requests);
for(const attempt of attempts){
requests++;assert.equal(attempt.sessionId,sample.sessionId);const request=from(artifacts.get(attempt.requestArtifactId).file),metrics=requestMetrics(request);
assert.equal(metrics.sha256,attempt.requestSha256);assert.equal(request.model,attempt.model);
if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknown++;continue;}
const response=fs.readFileSync(inside(artifacts.get(attempt.responseArtifactId).file));assert.equal(sha(response),attempt.responseSha256);
let usage;
for(const line of response.toString('utf8').split(/\r?\n/)){
if(!line.startsWith('data:'))continue;const text=line.slice(5).trim();if(!text||text==='[DONE]')continue;
const chunk=JSON.parse(text);if(chunk.usage)usage=chunk.usage;
}
assert.deepEqual(usage,attempt.usage,'Stored upstream usage: '+attempt.id);reportedTokens+=usage.total_tokens;sseResponses++;
}
if(['react','python'].includes(sample.name)){
const models=[...new Set(attempts.map(x=>x.model))];assert.deepEqual(models,[sample.name==='react'?'gpt-6.1-sol':'claude-opus-5-5']);
if(sample.name==='python')assert.equal(result.model,'gpt-6.1-sol','Preserve the documented original verifier metadata error.');
continue;
}
const base='samples/'+sample.name+'/',contract=from(base+'contract.json'),exp=from(base+'original-child-session.json');
const native=from(base+'engine-'+result.engineRuns[0].id+'.json');
const pinned={sourceTree:contract.sourceTree,sourceFiles:contract.sourceFiles,goal:contract.goal,writable:contract.writable,checks:contract.checks,budget:contract.budget,
connectionId:contract.model.id,revision:contract.model.revision,model:contract.model.model,protocol:contract.model.protocol,streaming:contract.model.streaming,
contextLimit:contract.model.contextLimit,maxOutputTokens:contract.model.maxOutputTokens,driverSha256:contract.driverSha256,installation:native.installationSha256};
if(!contractReference)contractReference=pinned;else assert.deepEqual(pinned,contractReference,'Paired source/model/budget/driver identity: '+sample.name);
assert.equal(native.native.cleanup.rootExited,true);assert.equal(native.native.cleanup.treeConfirmed,true);assert.deepEqual(native.native.cleanup.remainingProcesses,[]);
const inputs=native.native.bridgeRequests.filter(x=>x.route==='/model'),recorded=analysis.analyses.find(x=>x.name===sample.name);
assert.equal(inputs.length,attempts.length+1);assert.equal(result.stopReason,'engine_input_budget');assert.equal(result.applications.length,0);assert.equal(exp.execution.changeSets.length,0);
for(let i=0;i<inputs.length;i++){
const input=inputs[i].data,config={...exp.execution.run.model,modelRouting:exp.execution.run.model.routing,maxOutputTokens:input.maxTokens};
const metrics=requestMetrics(completionBody(config,nativeMessagesToChat(input),input.tools.map(x=>({type:'function',function:x}))));
assert.equal(metrics.sha256,recorded.modelInputs[i].requestSha256);assert.equal(metrics.estimatedInputTokens,recorded.modelInputs[i].estimatedInputTokens);
assert.equal(nativeSystemTexts(input).join('\n').split(setup.purge.defaultPrompt).length-1,result.enabled?1:0);
if(i<native.requests.length)assert.equal(metrics.sha256,native.requests[i].requestSha256);
else{assert.equal(inputs[i].status,'failed');assert.ok(metrics.estimatedInputTokens+input.maxTokens+128>config.contextLimit);assert.equal(recorded.modelInputs[i].dispatched,false);}
}
assert.equal(result.upstreamPluginLoaded,false);
}
assert.equal(requests,20);assert.equal(sseResponses,19);assert.equal(reportedTokens,419597);assert.equal(unknown,1);
assert.equal(approvals,4);assert.equal(applications,0);assert.equal(continuations,0);
assert.equal(manifest.requests,requests);assert.equal(analysis.requests,requests);assert.equal(manifest.reportedTokens,reportedTokens);assert.equal(reconciliation.reportedTokens,reportedTokens);
assert.equal(analysis.aggregate.enabled.reportedTokens,138758);assert.equal(analysis.aggregate.disabled.reportedTokens,78875);
for(const source of reconciliation.sourceReceipts)for(const [file,identity]of Object.entries(source.files))assert.equal(entries.get('source-baselines/'+source.sample+'/'+file)?.sha256,identity.sha256);
console.log(JSON.stringify({passed:true,manifestSha256:sha(fs.readFileSync(path.join(root,'manifest.json'))),evidenceFiles:manifest.count,evidenceBytes:manifest.bytes,
sourceInputs:manifest.sourceInputs.length,sourceCommit:manifest.sourceCommit,samples:manifest.sampleCount,requests,sseResponses,reportedTokens,unknownUsageRequests:unknown,
supervision:{fixedCheckApprovals:approvals,appliedBatches:applications,continuations,gitConfirmations:0,targetedImplementationHints:0},
promptComparison:{delivered:0,enabledReportedTokens:138758,disabledReportedTokens:78875,nativeInputRejections:4},originalPluginBenefit:'not_tested',
scope:'Offline byte/source/request/usage/input reconstruction only; no model request, no original failure rewrite, no claim that autonomous delivery passed.'}));