Files
Agent-Workbench/scripts/verify-engine-context-evidence.mjs

72 lines
5.8 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { originalEngineRun,verifyEngineModelInput } from '../src/shared/contracts/engine-execution.mjs';
import { requestMetrics } from '../src/shared/contracts/model-request.mjs';
// Offline verification of the A repair. It never creates a Runtime or Provider.
const root=path.resolve('docs/live-context-a-evidence'),sha=bytes=>createHash('sha256').update(bytes).digest('hex');
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
const read=relative=>JSON.parse(fs.readFileSync(inside(relative),'utf8'));
const manifest=read('manifest.json'),entries=new Map(manifest.entries.map(entry=>[entry.path,entry]));
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.match(manifest.sourceCommit,/^[a-f0-9]{40}$/);
let bytes=0;
for(const entry of entries.values()){
assert.ok(fs.lstatSync(inside(entry.path)).isFile());const data=fs.readFileSync(inside(entry.path));
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;
}
assert.equal(entries.size,manifest.count);assert.equal(bytes,manifest.bytes);
for(const input of manifest.sourceInputs){
assert.match(input.path,/^(?:src|scripts|tests|benchmarks)\/[A-Za-z0-9_./-]+$|^package(?:-lock)?\.json$/);
assert.ok(!input.path.split('/').includes('..'));
const blob=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
assert.equal(sha(blob),input.gitBlobSha256,input.path);
const raw=input.snapshot?fs.readFileSync(inside(input.snapshot)):blob;
assert.equal(sha(raw),input.sha256,input.path);assert.equal(raw.length,input.bytes,input.path);
assert.equal(raw.toString('utf8').replace(/\r\n/g,'\n'),blob.toString('utf8').replace(/\r\n/g,'\n'),input.path);
}
let originalRequests=0,bounded=0,artifactRecords=0;
for(const sample of manifest.samples){
const result=read(sample.result),records=read(sample.records),sessions=new Map(read(sample.sessions).map(value=>[value.id,value]));
const core={};for(const row of records){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
artifactRecords+=core.artifact.size;
const blobs=new Map(sample.artifacts.map(value=>[value.id,value]));
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId,'Original owner: '+id);return value;},
artifactStore:{read(artifact){const entry=blobs.get(artifact.id),data=fs.readFileSync(inside(entry.file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
if(!sample.passed){
assert.equal(result.success,false);assert.ok(result.engineRuns.some(job=>job.inputRejections?.length));
for(const job of core.engine_run.values())for(const rejected of job.inputRejections??[]){
const artifact=store.owned('artifact',rejected.artifactId,job.sessionId),receipt=JSON.parse(store.artifactStore.read(artifact));
assert.equal(artifact.sha256,rejected.sha256);assert.equal(receipt.dispatched,false);assert.equal(receipt.budget.contextLimit,16000);
assert.ok(receipt.budget.requiredInputTokens>receipt.budget.inputBudget);
}
continue; // Earlier development projections are not reinterpreted as v1.
}
assert.equal(result.success,true);assert.equal(result.metrics.contextLimit,16000);assert.equal(result.metrics.tests,9);assert.equal(result.parentCheck.results[0].tests.passed,9);
assert.equal(result.writable.length,3);assert.equal(result.realModelRequests,0);assert.match(result.commit,/^[a-f0-9]{40}$/);
assert.ok(result.checks.some(check=>check.status==='failed'&&check.results[0].tests.failed===9));
assert.ok([...core.tool.values()].some(tool=>tool.tool==='prepare_change_set'));
const sampleBodies=[];
for(const job of core.engine_run.values()){
const session=sessions.get(job.sessionId);originalEngineRun(store,session,job);
assert.equal(job.status,'completed');assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true);
for(const request of job.requests){
const input=verifyEngineModelInput(store,session,job,request),metrics=requestMetrics(input.body);originalRequests++;
assert.ok(metrics.estimatedInputTokens+input.native.maxTokens+input.view.safetyTokens<=16000);sampleBodies.push(input.body);
if(input.view.mode==='bounded'){bounded++;assert.ok(input.view.beforeInputTokens>input.view.inputBudget);}
}
assert.equal(job.inputRejections?.length??0,0);
}
const text=JSON.stringify(sampleBodies);assert.ok(text.includes('logArtifactId'));assert.ok(text.includes('failed'));
if(sample.name==='responses'){
const attempts=read(sample.attempts);assert.equal(attempts.length,result.metrics.requests);
assert.ok(attempts.every(attempt=>attempt.protocol==='responses'&&attempt.stream.terminal&&attempt.responseArtifactId));
}
}
assert.ok(bounded>0);assert.equal(manifest.samples.filter(sample=>sample.passed).length,2);
console.log(JSON.stringify({passed:true,manifestSha256:sha(fs.readFileSync(inside('manifest.json'))),files:entries.size,bytes,artifactRecords,sourceCommit:manifest.sourceCommit,
originalRequests,bounded,realModelRequests:0,scope:'Offline original bytes, source, native view/provenance, failed development receipts and two owned-model engineering chains; real-model/Purge benefit not measured.'}));