188 lines
18 KiB
JavaScript
188 lines
18 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
const root = path.resolve('docs/live-delivery-r-evidence');
|
|
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
|
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
|
const digest = value => sha(JSON.stringify(canonical(value)));
|
|
const file = name => {
|
|
assert.equal(typeof name, 'string'); assert.ok(!/[\\\x00\r\n]/.test(name) && !name.split('/').includes('..'));
|
|
const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target;
|
|
};
|
|
const read = name => JSON.parse(fs.readFileSync(file(name)));
|
|
const manifest = read('manifest.json'); assert.equal(manifest.kind, 'owned-live-delivery-r-evidence');
|
|
const names = new Set(); let totalBytes = 0;
|
|
for (const row of manifest.files) {
|
|
assert.ok(!names.has(row.path)); names.add(row.path);
|
|
const bytes = fs.readFileSync(file(row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256, row.path); totalBytes += bytes.length;
|
|
}
|
|
const list = (directory = root) => fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
|
|
assert.equal(entry.isSymbolicLink(), false); const target = path.join(directory, entry.name);
|
|
return entry.isDirectory() ? list(target) : [path.relative(root, target).replaceAll('\\','/')];
|
|
});
|
|
assert.deepEqual(list().sort(), ['manifest.json', ...names].sort());
|
|
const owned = name => { assert.ok(names.has(name)); return read(name); };
|
|
const initial = owned(manifest.references.initial), preflight = owned(manifest.references.preflight), node = owned(manifest.references.node);
|
|
const final = owned(manifest.references.final), network = owned(manifest.references.network), proof = owned(manifest.references.ownership);
|
|
const reading = owned(manifest.references.reading), budget = owned(manifest.references.budget), artifacts = owned(manifest.references.artifacts).artifacts;
|
|
const frozen = owned('contract/manifest.json'), connectionId = '5b4daed9-b2f7-4de0-b949-573eba70c7b3';
|
|
assert.equal(frozen.modelRequestsAtFreeze, 0); assert.equal(frozen.maxNodeRequests, 8); assert.equal(frozen.maxNodeTokens, 90000);
|
|
assert.equal(frozen.contextLimit, 16000); assert.equal(frozen.maxOutputTokens, 2048);
|
|
assert.equal(sha(fs.readFileSync(file('run/current.json'))), frozen.originalBaselineSha256);
|
|
assert.equal(sha(fs.readFileSync(file('provenance/frozen-plan.md'))), frozen.planSha256);
|
|
const gitBytes = (commit, relative) => execFileSync('git', ['show', commit + ':' + relative], { maxBuffer: 2 * 1024 * 1024, windowsHide: true });
|
|
for (const row of frozen.files) {
|
|
const bytes = fs.readFileSync(file('contract/' + row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
|
|
assert.deepEqual(bytes, gitBytes(manifest.experimentPlanCommit, 'docs/live-delivery-r-contract/' + row.path));
|
|
}
|
|
assert.deepEqual(fs.readFileSync(file('contract/manifest.json')), gitBytes(manifest.experimentPlanCommit, 'docs/live-delivery-r-contract/manifest.json'));
|
|
for (const row of frozen.dependencySources) assert.equal(sha(gitBytes(manifest.experimentBaseCommit, row.path)), row.sha256);
|
|
for (const row of budget.sourceInputs) {
|
|
const bytes = fs.readFileSync(file('run/' + row.capturedPath)); assert.equal(sha(bytes), row.sha256);
|
|
assert.deepEqual(bytes, gitBytes(manifest.releaseSourceCommit, row.path));
|
|
}
|
|
assert.equal(manifest.releaseSourceCommit, owned('provenance/release-1.24.6.json').sourceCommit);
|
|
assert.equal(network.connectionId, connectionId); assert.equal(network.connectionRevision, 5);
|
|
assert.equal(network.status, 'passed'); assert.equal(network.stages.dns.status, 'passed');
|
|
assert.deepEqual(network.trials.map(row => row.mode), ['auto','ipv4','ipv6']);
|
|
assert.ok(network.trials.every(row => row.status === 'passed' && row.certificateAuthorized === true));
|
|
assert.equal(network.httpRequests, 0); assert.equal(network.modelRequests, 0); assert.equal(network.credentialAccess, false);
|
|
assert.equal(network.route.kind, 'direct-endpoint-only'); assert.equal(network.route.modelApiRouteVerified, false);
|
|
assert.equal(owned('run/network/restoration.json').originalStateExactExceptNewDiagnosticHistory, true);
|
|
let totalRequests = 0, reportedTokens = 0;
|
|
const allAttempts = [];
|
|
for (const [folder, outcome, expectedCount] of [['preflight',preflight,1],['node',node,6]]) {
|
|
const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, expectedCount);
|
|
allAttempts.push(...attempts); assert.equal(outcome.captures.length, expectedCount * 2);
|
|
for (const attempt of attempts) {
|
|
assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.connectionRevision, folder === 'preflight' ? 5 : 6);
|
|
assert.equal(attempt.credentialRevision, 1); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
|
|
assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions'); assert.equal(attempt.streaming, false);
|
|
assert.equal(attempt.requestUrl, 'https://jojocode.com/v1/chat/completions'); assert.equal(attempt.status, 'completed');
|
|
assert.equal(attempt.modelIdentity.policy, 'require-match'); assert.equal(attempt.modelIdentity.status, 'matched');
|
|
assert.equal(attempt.modelIdentity.complete, true); assert.deepEqual(attempt.modelIdentity.reportedModels, ['gpt-6.1-sol']);
|
|
assert.equal(attempt.usageStatus, 'reported'); assert.equal(attempt.responseRedacted, false);
|
|
const captures = Object.fromEntries(outcome.captures.filter(row => row.attemptId === attempt.id).map(row => [row.kind,row]));
|
|
const requestBytes = fs.readFileSync(file('run/' + captures.request.path)), responseBytes = fs.readFileSync(file('run/' + captures.response.path));
|
|
assert.equal(sha(requestBytes), attempt.requestSha256); assert.equal(sha(requestBytes), captures.request.sha256);
|
|
assert.equal(sha(responseBytes), attempt.responseSha256); assert.equal(sha(responseBytes), captures.response.sha256);
|
|
assert.equal(attempt.modelIdentity.responseSource.sha256, attempt.responseSha256); assert.equal(requestBytes.length, attempt.requestBytes);
|
|
const body = JSON.parse(requestBytes), raw = JSON.parse(responseBytes); assert.equal(body.model, 'gpt-6.1-sol'); assert.notEqual(body.stream,true);
|
|
assert.equal(raw.model, 'gpt-6.1-sol'); assert.deepEqual(raw.usage, attempt.usage);
|
|
assert.ok(raw.usage.completion_tokens <= body.max_tokens); reportedTokens += raw.usage.total_tokens; totalRequests++;
|
|
if (folder === 'preflight') {
|
|
assert.equal(body.tools?.length ?? 0, 0); assert.equal(body.max_tokens, 256); assert.equal(preflight.actualTools, 0); assert.equal(preflight.passed,true);
|
|
assert.equal(raw.choices[0].message.tool_calls?.length ?? 0, 0); assert.ok(raw.choices[0].message.content.includes(owned('run/preflight/contract.json').nonce));
|
|
assert.deepEqual(raw.usage, preflight.usage); assert.equal(raw.usage.total_tokens, 4492);
|
|
} else {
|
|
assert.equal(body.max_tokens,2048); assert.ok(body.tools.length > 0);
|
|
assert.deepEqual(body.messages.filter(row => row.role === 'user').map(row => row.content), [fs.readFileSync(file('run/node/goal.txt'),'utf8')]);
|
|
}
|
|
}
|
|
assert.equal(owned('run/' + folder + '/restoration.json').passed,true);
|
|
}
|
|
assert.equal(totalRequests,7); assert.equal(reportedTokens,77063); assert.equal(node.success,false); assert.equal(node.stopReason,'token_limit');
|
|
assert.equal(node.usage.requests,6); assert.equal(node.usage.totalTokens,72571); assert.equal(node.usage.unknownRequests,0);
|
|
assert.equal(node.usage.estimatedUnknownTokens,0); assert.deepEqual(node.applications,[]); assert.equal(node.modelImplementationSuppliedBySupervisor,false);
|
|
const session = owned('run/node/session.json'); assert.deepEqual(session.changes,[]); assert.equal(session.execution.run.toolCalls,12);
|
|
assert.equal(session.execution.checks.length,1); const checks=session.execution.checks[0]; assert.equal(checks.status,'failed');
|
|
assert.deepEqual(checks.results.find(row => row.name==='test').tests,{total:9,passed:0,failed:9,cancelled:0,skipped:0});
|
|
assert.equal(checks.results.find(row => row.name==='build').status,'passed'); assert.equal(node.approvals.length,1); assert.equal(node.approvals[0].allow,true);
|
|
assert.equal(node.approvals[0].command,'npm run test\nnpm run build');
|
|
assert.deepEqual(session.execution.run.budgetPause,budget.pause); assert.equal(budget.pause.remainingTokens,17429);
|
|
assert.equal(budget.pause.tokenReservation.reservedTokens,29623); assert.equal(budget.pause.requiredInputTokens,12352);
|
|
assert.ok(budget.pause.requiredInputTokens<=budget.pause.inputBudget); assert.ok(budget.pause.tokenReservation.reservedTokens>budget.pause.remainingTokens);
|
|
assert.equal(budget.pause.runBudget.maxTokens,90000); assert.equal(budget.pause.runBudget.maxSteps,8);
|
|
assert.equal(budget.pause.contextLimit,16000); assert.equal(budget.pause.safetyTokens,1600); assert.equal(budget.pause.outputReservation,2048);
|
|
for (const plan of session.execution.contextPlans) {
|
|
assert.ok(plan.estimatedInputTokens<=plan.budget.inputBudget);
|
|
assert.ok(plan.tokenReservation.reservedTokens<=Math.min(plan.budget.runRemainingTokens,plan.budget.taskRemainingTokens));
|
|
assert.equal(plan.requestAttempts,1);
|
|
}
|
|
const rawFourth=JSON.parse(fs.readFileSync(file('run/'+node.captures.filter(row=>row.kind==='response')[3].path)));
|
|
assert.equal(rawFourth.usage.prompt_tokens,27164); assert.equal(rawFourth.usage.prompt_tokens_details.cached_tokens,24192);
|
|
const testSource=fs.readFileSync(file('run/node/source-before/test/expense.test.mjs'),'utf8').replaceAll('\r\n','\n');
|
|
const filePages=reading.pages.filter(row=>row.path==='test/expense.test.mjs');
|
|
assert.deepEqual(filePages.map(row=>[row.offset,row.next_offset]),[[0,473],[473,875],[875,1271]]);
|
|
for(const row of filePages){assert.equal(row.content,testSource.slice(row.offset,row.next_offset));assert.equal(row.total_characters,2584);}
|
|
assert.equal(budget.fileCoverage.missingCharacters,1313); assert.equal(budget.pageSizing[2].readMessageBytes,1024);
|
|
const wrapper=reading.pages.find(row=>row.kind==='tool-result'&&row.artifact_id===filePages[0].artifactId);
|
|
assert.ok(wrapper); assert.equal(wrapper.next_offset,495); assert.equal(wrapper.total_characters,891);
|
|
const artifactMap=new Map(artifacts.map(row=>[row.id,row])); assert.equal(artifacts.length,61);
|
|
for(const artifact of artifacts){
|
|
assert.ok(final.ownedSessions.includes(artifact.sessionId)); assert.ok(final.ownedProjects.includes(artifact.projectId));
|
|
const bytes=fs.readFileSync(file('run/'+artifact.capturedPath)); assert.equal(sha(bytes),artifact.sha256); assert.equal(bytes.length,artifact.bytes);
|
|
}
|
|
for(const result of checks.results){const artifact=artifactMap.get(result.logArtifactId);assert.ok(artifact);assert.equal(artifact.kind,'check-log');}
|
|
const snapshot=owned('run/final/snapshot.json');
|
|
for(const item of initial.baseline.exports)assert.deepEqual(snapshot.exports.find(row=>row.id===item.id),item);
|
|
for(const table of ['sessions','records','events','outbox']){
|
|
const before=initial.baseline.database.tables[table],after=snapshot.database.tables[table];assert.equal(after.schemaSha256,before.schemaSha256);
|
|
const rows=new Map(after.rows.map(row=>[row.id,row]));for(const row of before.rows)assert.deepEqual(rows.get(row.id),row);
|
|
assert.equal(proof.additions[table].length,after.count-before.count);
|
|
for(const row of proof.additions[table]){
|
|
assert.ok(final.ownedSessions.includes(row.sessionId));if(row.projectId)assert.ok(final.ownedProjects.includes(row.projectId));
|
|
assert.equal(rows.get(row.id).sha256,row.sha256);
|
|
}
|
|
}
|
|
assert.equal(proof.additions.records.filter(row=>row.kind==='artifact').length,artifacts.length);
|
|
for(const artifact of artifacts)assert.ok(proof.additions.records.some(row=>row.kind==='artifact'&&row.id===artifact.id));
|
|
const originalProjectRows=new Map(initial.baseline.database.tables.projects.rows.map(row=>[row.id,row]));
|
|
const finalProjectRows=new Map(snapshot.database.tables.projects.rows.map(row=>[row.id,row]));
|
|
for(const row of proof.projects){
|
|
assert.equal(row.actualSha256,finalProjectRows.get(row.id).sha256);
|
|
if(row.originalSha256){assert.equal(row.originalSha256,originalProjectRows.get(row.id).sha256);assert.equal(row.normalizedSha256,row.originalSha256);}
|
|
else assert.ok(final.ownedProjects.includes(row.id));
|
|
}
|
|
assert.equal(proof.projects.filter(row=>row.originalSha256).length,61);
|
|
assert.equal(proof.projects.filter(row=>row.owned).length,2); assert.equal(proof.projectInsertionOffset,2);
|
|
const oldArtifactMap=new Map(initial.baseline.persistent.artifacts.map(row=>[row.path,row]));
|
|
const nextArtifactMap=new Map(snapshot.persistent.artifacts.map(row=>[row.path,row]));
|
|
for(const [name,row]of oldArtifactMap)assert.deepEqual(nextArtifactMap.get(name),row);
|
|
const newFiles=snapshot.persistent.artifacts.filter(row=>!oldArtifactMap.has(row.path));assert.equal(newFiles.length,50);
|
|
assert.ok(newFiles.every(row=>artifacts.some(artifact=>artifact.sha256===row.sha256&&artifact.bytes===row.bytes)));
|
|
assert.deepEqual(snapshot.modelCenter.selected,initial.center.selected);assert.deepEqual(snapshot.modelCenter.pools,initial.center.pools);
|
|
const beforeMeta=owned('run/node/original-public-metadata.json'),afterMeta=owned('run/node/final-public-metadata.json');
|
|
const finalMetadataRows=new Map(snapshot.database.tables.metadata.rows.map(row=>[row.id,row]));
|
|
const oldAttempts=owned('run/preflight/original-attempts.json');assert.equal(oldAttempts.length,209);
|
|
assert.deepEqual(owned('run/node/original-attempts.json'),[...oldAttempts,...owned('run/preflight/attempts.json')]);
|
|
assert.equal(finalMetadataRows.get('modelAttempts').sha256,digest({key:'modelAttempts',value:JSON.stringify([...oldAttempts,...allAttempts])}));
|
|
for(const [key,value]of [['config',afterMeta.config],['modelRegistry',afterMeta.registry]])assert.equal(finalMetadataRows.get(key).sha256,digest({key,value:JSON.stringify(value)}));
|
|
const rawNetwork={...network};delete rawNetwork.stale;delete rawNetwork.currentConnectionRevision;
|
|
assert.equal(finalMetadataRows.get('modelNetworkDiagnostics').sha256,digest({key:'modelNetworkDiagnostics',value:JSON.stringify([rawNetwork])}));
|
|
for(const row of initial.baseline.database.tables.metadata.rows)if(!['config','modelRegistry','modelAttempts'].includes(row.id))assert.deepEqual(finalMetadataRows.get(row.id),row);
|
|
assert.deepEqual([...finalMetadataRows.keys()].filter(key=>!initial.baseline.database.tables.metadata.rows.some(row=>row.id===key)),['modelNetworkDiagnostics']);
|
|
assert.deepEqual({...afterMeta.config,configRevision:beforeMeta.config.configRevision},beforeMeta.config);
|
|
const registry=structuredClone(afterMeta.registry);registry.revision=beforeMeta.registry.revision;
|
|
const connection=registry.connections.find(row=>row.id===connectionId),oldConnection=beforeMeta.registry.connections.find(row=>row.id===connectionId);
|
|
assert.equal(connection.revision,oldConnection.revision+2);connection.revision=oldConnection.revision;connection.updatedAt=oldConnection.updatedAt;
|
|
const expected=structuredClone(beforeMeta.registry);for(const value of [expected,registry])value.connections.sort((a,b)=>a.id.localeCompare(b.id));
|
|
assert.deepEqual(canonical(registry),canonical(expected));assert.deepEqual(owned('run/node/restoration.json').identityDefaultNormalization,{});
|
|
for(const name of ['src/amount.mjs','src/summary.mjs','src/report.mjs'])assert.deepEqual(fs.readFileSync(file('run/node/source-before/'+name)),fs.readFileSync(file('run/node/source-after/'+name)));
|
|
const git=owned('run/node/git-final.json'),baseline=owned('run/node/source-identity.json');
|
|
assert.equal(git.head,baseline.head);assert.equal(git.status,baseline.status);assert.equal(git.stagedAfter,baseline.stagedBefore);assert.equal(git.diff,'');
|
|
assert.equal(final.passed,true);assert.equal(final.version,'1.24.6');assert.equal(final.pid,36552);assert.equal(final.inspectorClosed,true);
|
|
assert.equal(final.originalProjects,61);assert.equal(final.originalSessions,77);assert.equal(final.originalArtifacts,3610);assert.equal(final.originalAttempts,209);
|
|
assert.equal(final.finalProjects,63);assert.equal(final.finalSessions,79);assert.equal(final.finalArtifacts,3660);assert.equal(final.addedAttempts,7);
|
|
assert.deepEqual(final.connectionRevision,[5,7]);assert.equal(final.credentialRevision,1);assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision,[65,69]);
|
|
assert.equal(final.originalBudgetsSelectionAndPoolRestored,true);assert.equal(final.reportedTokens,reportedTokens);assert.equal(final.unknownRequests,0);
|
|
assert.equal(final.nodeDelivered,false);assert.equal(final.purgeBenefitTested,false);assert.equal(final.productSourceChanged,false);assert.equal(final.keyReadOrResaved,false);
|
|
let gitIndex;
|
|
if(process.argv.includes('--index')){
|
|
const prefix=path.relative(process.cwd(),root).replaceAll('\\','/'),manifestBytes=fs.readFileSync(file('manifest.json'));
|
|
const entries=[...manifest.files,{path:'manifest.json',bytes:manifestBytes.length,sha256:sha(manifestBytes)}];
|
|
const output=execFileSync('git',['cat-file','--batch'],{input:entries.map(row=>':'+prefix+'/'+row.path).join('\n')+'\n',maxBuffer:256*1024*1024,windowsHide:true});
|
|
let offset=0,indexBytes=0;
|
|
for(const entry of entries){const end=output.indexOf(10,offset),header=output.subarray(offset,end).toString('utf8'),match=header.match(/^[a-f0-9]{40} blob (\d+)$/);assert.ok(match,header);
|
|
const size=Number(match[1]);offset=end+1;const bytes=output.subarray(offset,offset+size);offset+=size;assert.equal(output[offset++],10);
|
|
assert.equal(size,entry.bytes,entry.path);assert.equal(sha(bytes),entry.sha256,entry.path);indexBytes+=size;}
|
|
assert.equal(offset,output.length);gitIndex={passed:true,files:entries.length,bytes:indexBytes,exactOriginalBytes:true};
|
|
}
|
|
console.log(JSON.stringify({success:true,files:manifest.files.length,bytes:totalBytes,manifestSha256:sha(fs.readFileSync(file('manifest.json'))),
|
|
modelRequests:totalRequests,reportedTokens,unknownRequests:0,preflightPassed:true,nodeDelivered:false,originalPurgeBenefitTested:false,
|
|
ownedArtifactRecords:artifacts.length,originalDataAndDefaultsVerified:true,readingPaginationAndBudgetPauseVerified:true,productVersion:'1.24.6',
|
|
...(gitIndex?{gitIndex}:{}),scope:'Offline original bytes, frozen plan/source, actual reports, paging, ownership and declared restoration. No network, host mutation, model or credential access.'}));
|