62 lines
5.4 KiB
JavaScript
62 lines
5.4 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import {createHash} from 'node:crypto';
|
|
import {execFileSync} from 'node:child_process';
|
|
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
|
|
import {observedTaskModels} from './live-model-observation.mjs';
|
|
|
|
// Offline only: no Provider, Runtime, credentials, model request or formal data.
|
|
const root=path.resolve('docs/live-budget-c-evidence'),sha=value=>createHash('sha256').update(value).digest('hex');
|
|
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
|
|
const raw=fs.readFileSync(inside('manifest.json')),manifest=JSON.parse(raw),read=relative=>JSON.parse(fs.readFileSync(inside(relative),'utf8'));
|
|
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.equal(manifest.formalServiceChanged,false);
|
|
const entries=new Map();let bytes=0;
|
|
for(const entry of manifest.entries){assert.ok(!entries.has(entry.path));entries.set(entry.path,entry);const data=fs.readFileSync(inside(entry.path));
|
|
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;}
|
|
assert.equal(entries.size,manifest.count);assert.equal(bytes,manifest.bytes);
|
|
for(const input of manifest.sourceInputs){
|
|
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$|^package(?:-lock)?\.json$/);assert.ok(!input.path.split('/').includes('..'));
|
|
const blob=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
|
|
const data=input.snapshot?fs.readFileSync(inside(input.snapshot)):blob;
|
|
assert.equal(sha(blob),input.gitBlobSha256);assert.equal(sha(data),input.sha256);assert.equal(data.length,input.bytes);
|
|
assert.equal(data.toString('utf8').replaceAll('\r\n','\n'),blob.toString('utf8').replaceAll('\r\n','\n'));
|
|
}
|
|
const receipt=name=>fs.readFileSync(inside(manifest.receipts.find(value=>value.name===name).file),'utf8');
|
|
assert.match(receipt('final-related'),/tests 149\b/);assert.match(receipt('final-related'),/pass 149\b/);assert.match(receipt('final-related'),/fail 0\b/);
|
|
assert.match(receipt('final-build'),/Server bundle built\./);
|
|
const ui=read(manifest.ui.result),session=read(manifest.ui.session);
|
|
assert.equal(ui.passed,true);assert.equal(ui.cases.length,2);assert.equal(ui.ownedHttpRequests,1);assert.deepEqual(ui.errors,[]);
|
|
assert.equal(session.status,'paused');assert.equal(session.execution.run.stopReason,'token_limit');
|
|
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.reportedTokens,55017);
|
|
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.exceededTokens,20017);
|
|
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.unknownRequests,0);
|
|
assert.equal(session.execution.run.budgetPause.modelFailure.httpStatus,503);
|
|
assert.equal(observedTaskModels(session).model,'owned-budget-ui');
|
|
let originalRequests=0,artifactRecords=0;
|
|
for(const sample of manifest.samples){
|
|
const result=read(sample.result),records=read(sample.records),sessions=new Map(read(sample.sessions).map(value=>[value.id,value])),core={};
|
|
for(const row of records){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
|
|
artifactRecords+=core.artifact.size;const artifacts=new Map(sample.artifacts.map(value=>[value.id,value]));
|
|
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId);return value;},
|
|
artifactStore:{read(artifact){const data=fs.readFileSync(inside(artifacts.get(artifact.id).file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
|
|
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
|
|
assert.equal(result.success,sample.passed);assert.equal(result.realModelRequests,0);
|
|
if(!sample.passed){assert.ok(result.error);continue;}
|
|
assert.equal(result.metrics.contextLimit,16000);assert.equal(result.parentCheck.results[0].tests.passed,9);
|
|
assert.ok(result.checks.some(job=>job.status==='failed'&&job.results[0].tests.failed===9));assert.equal(result.writable.length,3);
|
|
for(const job of core.engine_run.values()){
|
|
const owner=sessions.get(job.sessionId);originalEngineRun(store,owner,job);
|
|
assert.equal(job.status,'completed');assert.equal(job.native.cleanup.treeConfirmed,true);assert.equal(job.native.cleanup.rootExited,true);
|
|
for(const request of job.requests){
|
|
verifyEngineModelInput(store,owner,job,request);assert.equal(request.reservationVersion,1);originalRequests++;
|
|
assert.ok(request.tokenReservation.routes.every(route=>route.inputRatio>=1));
|
|
}
|
|
}
|
|
}
|
|
assert.deepEqual(manifest.samples.filter(value=>value.passed).map(value=>value.name).sort(),['chat','responses']);
|
|
for(const name of ['a-offline','b-offline','pilot-offline'])assert.equal(JSON.parse(receipt(name)).passed,true);
|
|
console.log(JSON.stringify({passed:true,sourceCommit:manifest.sourceCommit,manifestSha256:sha(raw),files:entries.size,bytes,
|
|
sourceInputs:manifest.sourceInputs.length,artifactRecords,originalRequests,realModelRequests:0,
|
|
scope:'Offline raw bytes/source, known usage reservation, actual Task model identity and owned native engineering evidence; original failures retained, real model and original Purge not measured.'}));
|