102 lines
9.5 KiB
JavaScript
102 lines
9.5 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { completionBody, requestMetrics } from '../src/shared/contracts/model-request.mjs';
|
|
import { nativeMessagesToChat } from '../src/shared/contracts/engine-execution.mjs';
|
|
import { nativeSystemTexts } from '../src/core/engines/purge-strategy.mjs';
|
|
|
|
if (process.argv.includes('--help')) {
|
|
console.log('node scripts/verify-live-pilot-evidence.mjs — verify sealed 1.24 pilot bytes, source identities, original request/SSE usage and native input rejection; offline, no model calls.');
|
|
process.exit(0);
|
|
}
|
|
const root=path.resolve('docs/live-model-purge-1.24-evidence');
|
|
const read=file=>JSON.parse(fs.readFileSync(file,'utf8').replace(/^\uFEFF/,''));
|
|
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
|
const manifest=read(path.join(root,'manifest.json')),entries=new Map(manifest.entries.map(x=>[x.path,x]));
|
|
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.formalVersion,'1.24.0');assert.equal(manifest.sampleCount,6);
|
|
assert.equal(manifest.upstreamPluginLoaded,false);assert.equal(manifest.originalPluginBenefit,'not_tested');
|
|
assert.equal(entries.size,manifest.count);
|
|
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
|
|
const from=relative=>{assert.ok(entries.has(relative),'File belongs to manifest: '+relative);return read(inside(relative));};
|
|
let bytes=0;
|
|
for(const entry of entries.values()){
|
|
const file=inside(entry.path);assert.equal(fs.lstatSync(file).isFile(),true,entry.path);
|
|
const original=fs.readFileSync(file);assert.equal(original.length,entry.bytes,entry.path);assert.equal(sha(original),entry.sha256,entry.path);bytes+=original.length;
|
|
}
|
|
assert.equal(bytes,manifest.bytes);
|
|
const files=[];
|
|
function visit(relative='') {
|
|
for(const entry of fs.readdirSync(path.join(root,relative),{withFileTypes:true})){
|
|
const next=path.posix.join(relative,entry.name);assert.equal(entry.isSymbolicLink(),false,next);
|
|
if(entry.isDirectory())visit(next);else{assert.equal(entry.isFile(),true,next);files.push(next);}
|
|
}
|
|
}
|
|
visit();assert.deepEqual(files.sort(),[...entries.keys(),'manifest.json'].sort(),'Exact sealed file scope');
|
|
assert.match(manifest.sourceCommit,/^[a-f0-9]{40}$/);
|
|
for(const input of manifest.sourceInputs){
|
|
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$/);assert.ok(!input.path.split('/').includes('..'));
|
|
const data=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:8*1024*1024});
|
|
assert.equal(data.length,input.bytes,input.path);assert.equal(sha(data),input.sha256,'Historical source used by the sealed pilot: '+input.path);
|
|
}
|
|
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,381);
|
|
for(const a of artifacts.values()){assert.equal(entries.get(a.file)?.sha256,a.sha256,a.id);assert.equal(entries.get(a.file)?.bytes,a.bytes,a.id);}
|
|
const setup=from('receipts/setup-public-receipt.json'),analysis=from('receipts/analysis.json'),preservation=from('receipts/state-preservation.json');
|
|
const complete=from('receipts/complete-owned-artifact-receipts.json'),reconciliation=from('receipts/request-and-source-reconciliation.json');
|
|
assert.equal(complete.newUniqueFiles,342);assert.equal(complete.artifactRecords,381);assert.equal(complete.totalArtifactBytes,6426284);
|
|
assert.equal(preservation.passed,true);assert.equal(preservation.original.exportDigestsUnchanged,33);assert.equal(preservation.original.artifactFilesUnchanged,1780);assert.equal(preservation.original.modelAttemptsUnchanged,70);
|
|
assert.deepEqual(preservation.formal.selected,setup.originalSelection);assert.deepEqual(preservation.formal.active,[]);assert.equal(preservation.formal.pid,manifest.formalPid);
|
|
const audit=from('receipts/process-audit.json');assert.ok(audit.processes.every(x=>x.rootExited&&x.treeConfirmed&&!x.currentlyExists));assert.deepEqual(audit.debug9229,[]);
|
|
let requests=0,reportedTokens=0,unknown=0,sseResponses=0,approvals=0,applications=0,continuations=0;let contractReference;
|
|
for(const sample of manifest.samples){
|
|
const result=from(sample.result),attempts=from(sample.attempts);assert.equal(result.success,false,'Original sample must retain failed acceptance: '+sample.name);
|
|
approvals+=(result.decisions??[]).length;applications+=(result.applications??[]).length;continuations+=(result.continuations??result.restarts??[]).length;
|
|
assert.equal(attempts.length,result.usage.requests);
|
|
for(const attempt of attempts){
|
|
requests++;assert.equal(attempt.sessionId,sample.sessionId);const request=from(artifacts.get(attempt.requestArtifactId).file),metrics=requestMetrics(request);
|
|
assert.equal(metrics.sha256,attempt.requestSha256);assert.equal(request.model,attempt.model);
|
|
if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknown++;continue;}
|
|
const response=fs.readFileSync(inside(artifacts.get(attempt.responseArtifactId).file));assert.equal(sha(response),attempt.responseSha256);
|
|
let usage;
|
|
for(const line of response.toString('utf8').split(/\r?\n/)){
|
|
if(!line.startsWith('data:'))continue;const text=line.slice(5).trim();if(!text||text==='[DONE]')continue;
|
|
const chunk=JSON.parse(text);if(chunk.usage)usage=chunk.usage;
|
|
}
|
|
assert.deepEqual(usage,attempt.usage,'Stored upstream usage: '+attempt.id);reportedTokens+=usage.total_tokens;sseResponses++;
|
|
}
|
|
if(['react','python'].includes(sample.name)){
|
|
const models=[...new Set(attempts.map(x=>x.model))];assert.deepEqual(models,[sample.name==='react'?'gpt-6.1-sol':'claude-opus-5-5']);
|
|
if(sample.name==='python')assert.equal(result.model,'gpt-6.1-sol','Preserve the documented original verifier metadata error.');
|
|
continue;
|
|
}
|
|
const base='samples/'+sample.name+'/',contract=from(base+'contract.json'),exp=from(base+'original-child-session.json');
|
|
const native=from(base+'engine-'+result.engineRuns[0].id+'.json');
|
|
const pinned={sourceTree:contract.sourceTree,sourceFiles:contract.sourceFiles,goal:contract.goal,writable:contract.writable,checks:contract.checks,budget:contract.budget,
|
|
connectionId:contract.model.id,revision:contract.model.revision,model:contract.model.model,protocol:contract.model.protocol,streaming:contract.model.streaming,
|
|
contextLimit:contract.model.contextLimit,maxOutputTokens:contract.model.maxOutputTokens,driverSha256:contract.driverSha256,installation:native.installationSha256};
|
|
if(!contractReference)contractReference=pinned;else assert.deepEqual(pinned,contractReference,'Paired source/model/budget/driver identity: '+sample.name);
|
|
assert.equal(native.native.cleanup.rootExited,true);assert.equal(native.native.cleanup.treeConfirmed,true);assert.deepEqual(native.native.cleanup.remainingProcesses,[]);
|
|
const inputs=native.native.bridgeRequests.filter(x=>x.route==='/model'),recorded=analysis.analyses.find(x=>x.name===sample.name);
|
|
assert.equal(inputs.length,attempts.length+1);assert.equal(result.stopReason,'engine_input_budget');assert.equal(result.applications.length,0);assert.equal(exp.execution.changeSets.length,0);
|
|
for(let i=0;i<inputs.length;i++){
|
|
const input=inputs[i].data,config={...exp.execution.run.model,modelRouting:exp.execution.run.model.routing,maxOutputTokens:input.maxTokens};
|
|
const metrics=requestMetrics(completionBody(config,nativeMessagesToChat(input),input.tools.map(x=>({type:'function',function:x}))));
|
|
assert.equal(metrics.sha256,recorded.modelInputs[i].requestSha256);assert.equal(metrics.estimatedInputTokens,recorded.modelInputs[i].estimatedInputTokens);
|
|
assert.equal(nativeSystemTexts(input).join('\n').split(setup.purge.defaultPrompt).length-1,result.enabled?1:0);
|
|
if(i<native.requests.length)assert.equal(metrics.sha256,native.requests[i].requestSha256);
|
|
else{assert.equal(inputs[i].status,'failed');assert.ok(metrics.estimatedInputTokens+input.maxTokens+128>config.contextLimit);assert.equal(recorded.modelInputs[i].dispatched,false);}
|
|
}
|
|
assert.equal(result.upstreamPluginLoaded,false);
|
|
}
|
|
assert.equal(requests,20);assert.equal(sseResponses,19);assert.equal(reportedTokens,419597);assert.equal(unknown,1);
|
|
assert.equal(approvals,4);assert.equal(applications,0);assert.equal(continuations,0);
|
|
assert.equal(manifest.requests,requests);assert.equal(analysis.requests,requests);assert.equal(manifest.reportedTokens,reportedTokens);assert.equal(reconciliation.reportedTokens,reportedTokens);
|
|
assert.equal(analysis.aggregate.enabled.reportedTokens,138758);assert.equal(analysis.aggregate.disabled.reportedTokens,78875);
|
|
for(const source of reconciliation.sourceReceipts)for(const [file,identity]of Object.entries(source.files))assert.equal(entries.get('source-baselines/'+source.sample+'/'+file)?.sha256,identity.sha256);
|
|
console.log(JSON.stringify({passed:true,manifestSha256:sha(fs.readFileSync(path.join(root,'manifest.json'))),evidenceFiles:manifest.count,evidenceBytes:manifest.bytes,
|
|
sourceInputs:manifest.sourceInputs.length,sourceCommit:manifest.sourceCommit,samples:manifest.sampleCount,requests,sseResponses,reportedTokens,unknownUsageRequests:unknown,
|
|
supervision:{fixedCheckApprovals:approvals,appliedBatches:applications,continuations,gitConfirmations:0,targetedImplementationHints:0},
|
|
promptComparison:{delivered:0,enabledReportedTokens:138758,disabledReportedTokens:78875,nativeInputRejections:4},originalPluginBenefit:'not_tested',
|
|
scope:'Offline byte/source/request/usage/input reconstruction only; no model request, no original failure rewrite, no claim that autonomous delivery passed.'}));
|