47 lines
3.9 KiB
JavaScript
47 lines
3.9 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import {createHash} from 'node:crypto';
|
|
|
|
// Read saved evidence only. No Provider, network request or credential access.
|
|
if(process.argv.includes('--help')){
|
|
console.log('node scripts/verify-upstream-model-identity.mjs --evidence DIR --output NEW_JSON — offline request/response model identifier check; exit 2 for mismatch or missing identity');process.exit(0);
|
|
}
|
|
const arg=name=>{const i=process.argv.indexOf(name);assert.ok(i>=0&&process.argv[i+1],name+' required');return process.argv[i+1];};
|
|
const root=path.resolve(arg('--evidence')),output=path.resolve(arg('--output'));
|
|
assert.equal(fs.existsSync(output),false,'Preserve earlier audit outputs.');
|
|
const sha=raw=>createHash('sha256').update(raw).digest('hex');
|
|
const manifestRaw=fs.readFileSync(path.join(root,'manifest.json')),manifest=JSON.parse(manifestRaw);
|
|
const entries=new Map(manifest.entries.map(e=>[e.path,e]));
|
|
function read(relative){const full=path.resolve(root,relative);assert.ok(full.startsWith(root+path.sep));
|
|
const raw=fs.readFileSync(full),entry=entries.get(relative);assert.ok(entry,relative);assert.equal(raw.length,entry.bytes);assert.equal(sha(raw),entry.sha256);return raw;}
|
|
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));
|
|
function artifact(id){const a=artifacts.get(id);assert.ok(a,id);const raw=read(a.file);assert.equal(sha(raw),a.sha256);return raw;}
|
|
const attempts=JSON.parse(read('receipts/all-model-attempts.json'));
|
|
const rows=[];
|
|
for(const attempt of attempts){
|
|
const request=JSON.parse(artifact(attempt.requestArtifactId));assert.equal(request.model,attempt.model);
|
|
const models=new Set();
|
|
const use=value=>{for(const model of [value.model,value.response?.model])if(typeof model==='string'&&model.trim())models.add(model);};
|
|
if(attempt.responseArtifactId){
|
|
const raw=artifact(attempt.responseArtifactId);assert.equal(sha(raw),attempt.responseSha256);
|
|
const text=raw.toString('utf8');
|
|
if(text.trimStart().startsWith('{'))use(JSON.parse(text));
|
|
else for(const line of text.split(/\r?\n/)){if(!line.startsWith('data:'))continue;const data=line.slice(5).trim();if(data&&data!=='[DONE]')use(JSON.parse(data));}
|
|
}
|
|
const reported=[...models];
|
|
rows.push({attemptId:attempt.id,sessionId:attempt.sessionId,requestArtifactId:attempt.requestArtifactId,responseArtifactId:attempt.responseArtifactId??null,
|
|
requestedModel:request.model,upstreamReportedModels:reported,status:reported.length===0?'missing':reported.length===1&&reported[0]===request.model?'matched':'mismatch'});
|
|
}
|
|
assert.equal(rows.length,manifest.requests);
|
|
const mismatchRequests=rows.filter(r=>r.status==='mismatch').length,missingModelIdentifiers=rows.filter(r=>r.status==='missing').length;
|
|
const result={schemaVersion:1,passed:mismatchRequests===0&&missingModelIdentifiers===0,status:mismatchRequests?'mismatch':missingModelIdentifiers?'missing':'matched',
|
|
sourceManifest:path.relative(process.cwd(),path.join(root,'manifest.json')).replaceAll('\\','/'),sourceManifestSha256:sha(manifestRaw),
|
|
requests:rows.length,requestedModels:[...new Set(rows.map(r=>r.requestedModel))],upstreamReportedModels:[...new Set(rows.flatMap(r=>r.upstreamReportedModels))],
|
|
matchedRequests:rows.filter(r=>r.status==='matched').length,mismatchRequests,missingModelIdentifiers,attempts:rows,
|
|
scope:'Exact identifiers in saved outgoing requests and upstream raw JSON/SSE. No documented alias mapping assumed; returned identifiers do not independently authenticate vendor model weights. Missing identity is separate from unknown token usage.'};
|
|
fs.writeFileSync(output,JSON.stringify(result,null,2)+'\n',{flag:'wx'});
|
|
console.log(JSON.stringify({passed:result.passed,status:result.status,requests:result.requests,requestedModels:result.requestedModels,
|
|
upstreamReportedModels:result.upstreamReportedModels,mismatchRequests,missingModelIdentifiers,output}));
|
|
process.exitCode=result.passed?0:2;
|