Files
Agent-Workbench/scripts/verify-model-budget-evidence.mjs

62 lines
5.4 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {createHash} from 'node:crypto';
import {execFileSync} from 'node:child_process';
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
import {observedTaskModels} from './live-model-observation.mjs';
// Offline only: no Provider, Runtime, credentials, model request or formal data.
const root=path.resolve('docs/live-budget-c-evidence'),sha=value=>createHash('sha256').update(value).digest('hex');
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
const raw=fs.readFileSync(inside('manifest.json')),manifest=JSON.parse(raw),read=relative=>JSON.parse(fs.readFileSync(inside(relative),'utf8'));
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.equal(manifest.formalServiceChanged,false);
const entries=new Map();let bytes=0;
for(const entry of manifest.entries){assert.ok(!entries.has(entry.path));entries.set(entry.path,entry);const data=fs.readFileSync(inside(entry.path));
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;}
assert.equal(entries.size,manifest.count);assert.equal(bytes,manifest.bytes);
for(const input of manifest.sourceInputs){
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$|^package(?:-lock)?\.json$/);assert.ok(!input.path.split('/').includes('..'));
const blob=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
const data=input.snapshot?fs.readFileSync(inside(input.snapshot)):blob;
assert.equal(sha(blob),input.gitBlobSha256);assert.equal(sha(data),input.sha256);assert.equal(data.length,input.bytes);
assert.equal(data.toString('utf8').replaceAll('\r\n','\n'),blob.toString('utf8').replaceAll('\r\n','\n'));
}
const receipt=name=>fs.readFileSync(inside(manifest.receipts.find(value=>value.name===name).file),'utf8');
assert.match(receipt('final-related'),/tests 149\b/);assert.match(receipt('final-related'),/pass 149\b/);assert.match(receipt('final-related'),/fail 0\b/);
assert.match(receipt('final-build'),/Server bundle built\./);
const ui=read(manifest.ui.result),session=read(manifest.ui.session);
assert.equal(ui.passed,true);assert.equal(ui.cases.length,2);assert.equal(ui.ownedHttpRequests,1);assert.deepEqual(ui.errors,[]);
assert.equal(session.status,'paused');assert.equal(session.execution.run.stopReason,'token_limit');
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.reportedTokens,55017);
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.exceededTokens,20017);
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.unknownRequests,0);
assert.equal(session.execution.run.budgetPause.modelFailure.httpStatus,503);
assert.equal(observedTaskModels(session).model,'owned-budget-ui');
let originalRequests=0,artifactRecords=0;
for(const sample of manifest.samples){
const result=read(sample.result),records=read(sample.records),sessions=new Map(read(sample.sessions).map(value=>[value.id,value])),core={};
for(const row of records){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
artifactRecords+=core.artifact.size;const artifacts=new Map(sample.artifacts.map(value=>[value.id,value]));
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId);return value;},
artifactStore:{read(artifact){const data=fs.readFileSync(inside(artifacts.get(artifact.id).file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
assert.equal(result.success,sample.passed);assert.equal(result.realModelRequests,0);
if(!sample.passed){assert.ok(result.error);continue;}
assert.equal(result.metrics.contextLimit,16000);assert.equal(result.parentCheck.results[0].tests.passed,9);
assert.ok(result.checks.some(job=>job.status==='failed'&&job.results[0].tests.failed===9));assert.equal(result.writable.length,3);
for(const job of core.engine_run.values()){
const owner=sessions.get(job.sessionId);originalEngineRun(store,owner,job);
assert.equal(job.status,'completed');assert.equal(job.native.cleanup.treeConfirmed,true);assert.equal(job.native.cleanup.rootExited,true);
for(const request of job.requests){
verifyEngineModelInput(store,owner,job,request);assert.equal(request.reservationVersion,1);originalRequests++;
assert.ok(request.tokenReservation.routes.every(route=>route.inputRatio>=1));
}
}
}
assert.deepEqual(manifest.samples.filter(value=>value.passed).map(value=>value.name).sort(),['chat','responses']);
for(const name of ['a-offline','b-offline','pilot-offline'])assert.equal(JSON.parse(receipt(name)).passed,true);
console.log(JSON.stringify({passed:true,sourceCommit:manifest.sourceCommit,manifestSha256:sha(raw),files:entries.size,bytes,
sourceInputs:manifest.sourceInputs.length,artifactRecords,originalRequests,realModelRequests:0,
scope:'Offline raw bytes/source, known usage reservation, actual Task model identity and owned native engineering evidence; original failures retained, real model and original Purge not measured.'}));