import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; const root=path.resolve('docs/live-delivery-t-evidence'); const sha=bytes=>createHash('sha256').update(bytes).digest('hex'); const canonical=value=>Array.isArray(value)?value.map(canonical):value&&typeof value==='object' ?Object.fromEntries(Object.keys(value).sort().map(key=>[key,canonical(value[key])])):value; const digest=value=>sha(JSON.stringify(canonical(value))); const file=name=>{assert.equal(typeof name,'string');assert.ok(!/[\\\x00\r\n]/.test(name)&&!name.split('/').includes('..')); const target=path.resolve(root,name);assert.ok(target.startsWith(root+path.sep));return target;}; const bytes=name=>fs.readFileSync(file(name)),read=name=>JSON.parse(bytes(name)); const manifest=read('manifest.json');assert.equal(manifest.kind,'owned-live-delivery-t-evidence'); const names=new Set();let totalBytes=0; for(const row of manifest.files){assert.ok(!names.has(row.path));names.add(row.path);const value=bytes(row.path); assert.equal(value.length,row.bytes);assert.equal(sha(value),row.sha256,row.path);totalBytes+=value.length;} const list=(directory=root)=>fs.readdirSync(directory,{withFileTypes:true}).flatMap(entry=>{assert.equal(entry.isSymbolicLink(),false); const target=path.join(directory,entry.name);return entry.isDirectory()?list(target):[path.relative(root,target).replaceAll('\\','/')];}); assert.deepEqual(list().sort(),['manifest.json',...names].sort()); const owned=name=>{assert.ok(names.has(name),name);return read(name);}; const initial=owned(manifest.references.initial),preflight=owned(manifest.references.preflight),node=owned(manifest.references.node); const final=owned(manifest.references.final),network=owned(manifest.references.network),proof=owned(manifest.references.ownership); const context=owned(manifest.references.context),replay=owned(manifest.references.candidates),artifacts=owned(manifest.references.artifacts).artifacts; const frozen=owned('contract/manifest.json'),connectionId='5b4daed9-b2f7-4de0-b949-573eba70c7b3'; assert.equal(frozen.modelRequestsAtFreeze,0);assert.equal(frozen.maxNetworkJobs,1);assert.equal(frozen.maxPreflightRequests,1); assert.equal(frozen.maxNodeRequests,8);assert.equal(frozen.maxNodeTokens,90000);assert.equal(frozen.maxNodeDurationMs,600000); assert.equal(frozen.contextLimit,16000);assert.equal(frozen.maxOutputTokens,2048);assert.equal(frozen.safetyReserveTokens,1600); assert.equal(sha(bytes('run/current.json')),frozen.originalBaselineSha256); assert.equal(sha(bytes('provenance/frozen-plan.md')),frozen.planSha256); const gitBytes=(commit,relative)=>execFileSync('git',['show',commit+':'+relative],{maxBuffer:2*1024*1024,windowsHide:true}); for(const row of frozen.files){assert.equal(bytes('contract/'+row.path).length,row.bytes);assert.equal(sha(bytes('contract/'+row.path)),row.sha256); assert.deepEqual(bytes('contract/'+row.path),gitBytes(manifest.experimentPlanCommit,'docs/live-delivery-t-contract/'+row.path));} assert.deepEqual(bytes('contract/manifest.json'),gitBytes(manifest.experimentPlanCommit,'docs/live-delivery-t-contract/manifest.json')); for(const row of frozen.dependencySources)assert.equal(sha(gitBytes(manifest.experimentBaseCommit,row.path)),row.sha256); for(const row of context.sourceInputs){assert.equal(sha(bytes('run/'+row.capturedPath)),row.sha256);assert.equal(bytes('run/'+row.capturedPath).length,row.bytes); assert.deepEqual(bytes('run/'+row.capturedPath),gitBytes(manifest.releaseSourceCommit,row.path));} assert.equal(context.sourceInputs.length,9);assert.equal(context.sourceCommit,manifest.releaseSourceCommit); assert.equal(manifest.releaseSourceCommit,owned('provenance/release-1.24.7.json').sourceCommit); assert.equal(sha(bytes('provenance/release-1.24.7.json')),frozen.releaseIdentitySha256); assert.deepEqual(bytes('run/node/goal.txt'),gitBytes('fbedb738','docs/live-delivery-r-evidence/run/node/goal.txt')); const stripDerived=row=>{const {stale,currentConnectionRevision,...raw}=row;return raw;}; assert.equal(network.connectionId,connectionId);assert.equal(network.connectionRevision,7);assert.equal(network.status,'passed'); assert.equal(network.stages.dns.status,'passed');assert.deepEqual(network.trials.map(row=>row.mode),['auto','ipv4','ipv6']); assert.ok(network.trials.every(row=>row.status==='passed'&&row.certificateAuthorized===true)); assert.equal(network.httpRequests,0);assert.equal(network.modelRequests,0);assert.equal(network.credentialAccess,false); assert.equal(network.route.kind,'direct-endpoint-only');assert.equal(network.route.modelApiRouteVerified,false); assert.equal(owned('run/network/restoration.json').originalStateExactExceptOneAppendedDiagnostic,true); assert.equal(owned('run/network/driver-error.json').jobId,network.id); assert.equal(sha(bytes('run/network/audit-first-driver.mjs.txt')),owned('run/network/audit-first-error.json').driverSha256); const allAttempts=[];let totalRequests=0,reportedTokens=0; const assertPairs=messages=>{const pending=new Set();for(const message of messages){ if(message.role==='assistant'){assert.equal(pending.size,0);for(const call of message.tool_calls??[]){assert.ok(!pending.has(call.id));pending.add(call.id);}} else if(message.role==='tool'){assert.ok(pending.has(message.tool_call_id));pending.delete(message.tool_call_id);} else assert.equal(pending.size,0); }assert.equal(pending.size,0);}; for(const [folder,outcome,expectedCount]of[['preflight',preflight,1],['node',node,8]]){ const attempts=owned('run/'+folder+'/attempts.json');assert.equal(attempts.length,expectedCount);allAttempts.push(...attempts); assert.equal(outcome.captures.length,expectedCount*2); for(const attempt of attempts){ assert.equal(attempt.connectionId,connectionId);assert.equal(attempt.connectionRevision,folder==='preflight'?7:8); assert.equal(attempt.credentialRevision,1);assert.equal(attempt.poolId,null);assert.equal(attempt.ordinal,1); assert.equal(attempt.model,'gpt-6.1-sol');assert.equal(attempt.protocol,'chat-completions');assert.equal(attempt.streaming,false); assert.equal(attempt.requestUrl,'https://jojocode.com/v1/chat/completions');assert.equal(attempt.status,'completed'); assert.equal(attempt.modelIdentity.policy,'require-match');assert.equal(attempt.modelIdentity.status,'matched'); assert.equal(attempt.modelIdentity.complete,true);assert.deepEqual(attempt.modelIdentity.reportedModels,['gpt-6.1-sol']); assert.equal(attempt.usageStatus,'reported');assert.equal(attempt.responseRedacted,false); const captures=Object.fromEntries(outcome.captures.filter(row=>row.attemptId===attempt.id).map(row=>[row.kind,row])); const request=bytes('run/'+captures.request.path),response=bytes('run/'+captures.response.path); assert.equal(sha(request),attempt.requestSha256);assert.equal(sha(request),captures.request.sha256); assert.equal(sha(response),attempt.responseSha256);assert.equal(sha(response),captures.response.sha256); assert.equal(attempt.modelIdentity.responseSource.sha256,attempt.responseSha256);assert.equal(request.length,attempt.requestBytes); const body=JSON.parse(request),raw=JSON.parse(response);assertPairs(body.messages); assert.equal(body.model,'gpt-6.1-sol');assert.notEqual(body.stream,true);assert.equal(raw.model,'gpt-6.1-sol');assert.deepEqual(raw.usage,attempt.usage); assert.ok(raw.usage.completion_tokens<=body.max_tokens);reportedTokens+=raw.usage.total_tokens;totalRequests++; if(folder==='preflight'){ assert.equal(body.tools?.length??0,0);assert.equal(body.max_tokens,256);assert.equal(preflight.actualTools,0);assert.equal(preflight.passed,true); assert.equal(raw.choices[0].message.tool_calls?.length??0,0);assert.ok(raw.choices[0].message.content.includes(owned('run/preflight/contract.json').nonce)); assert.deepEqual(raw.usage,preflight.usage);assert.equal(raw.usage.total_tokens,112); }else{assert.equal(body.max_tokens,2048);assert.ok(body.tools.length>0); assert.deepEqual(body.messages.filter(row=>row.role==='user').map(row=>row.content),[bytes('run/node/goal.txt').toString('utf8')]);} }assert.equal(owned('run/'+folder+'/restoration.json').passed,true); } assert.equal(totalRequests,9);assert.equal(reportedTokens,76174);assert.equal(node.success,false);assert.equal(node.stopReason,'step_limit'); assert.equal(node.usage.requests,8);assert.equal(node.usage.totalTokens,76062);assert.equal(node.usage.unknownRequests,0);assert.equal(node.usage.estimatedUnknownTokens,0); assert.deepEqual(node.applications,[]);assert.equal(node.modelImplementationSuppliedBySupervisor,false); const session=owned('run/node/session.json');assert.deepEqual(session.changes,[]);assert.equal(session.execution.run.toolCalls,16); assert.deepEqual(session.execution.run.budget,{maxSteps:8,maxDurationMs:600000,maxTokens:90000,maxTaskTokens:90000}); assert.equal(session.execution.checks.length,1);const checks=session.execution.checks[0];assert.equal(checks.status,'failed'); assert.deepEqual(checks.results.find(row=>row.name==='test').tests,{total:9,passed:0,failed:9,cancelled:0,skipped:0}); assert.equal(checks.results.find(row=>row.name==='build').status,'passed');assert.equal(node.approvals.length,1);assert.equal(node.approvals[0].allow,true); assert.equal(node.approvals[0].command,'npm run test\nnpm run build'); assert.equal(session.execution.contextPlans.length,8); for(const plan of session.execution.contextPlans){ assert.ok(plan.estimatedInputTokens<=plan.budget.inputBudget); assert.ok(plan.tokenReservation.reservedTokens<=Math.min(plan.budget.runRemainingTokens,plan.budget.taskRemainingTokens)); assert.equal(plan.budget.contextLimit,16000);assert.equal(plan.budget.safetyTokens,1600);assert.equal(plan.budget.outputReservation,2048);assert.equal(plan.requestAttempts,1); } const testPath='test/expense.test.mjs',testSource=bytes('run/node/source-before/'+testPath).toString('utf8');assert.equal(testSource.length,2584); const filePages=context.sourceResults.filter(row=>row.path===testPath); assert.deepEqual(filePages.map(row=>[row.offset,row.next_offset]),[[0,473],[473,2089],[0,472],[2089,2557],[2557,2584]]); for(const row of context.sourceResults){const source=bytes('run/node/source-before/'+row.path).toString('utf8'); assert.equal(row.content,source.slice(row.offset,row.next_offset));assert.equal(row.sha256,sha(source));} assert.deepEqual(context.returnedTestRanges,[[0,2584]]);assert.equal(context.returnedTestCharacters,2584); const union=ranges=>ranges.slice().sort((a,b)=>a[0]-b[0]).reduce((result,[a,b])=>{const previous=result.at(-1); if(previous&&a<=previous[1])previous[1]=Math.max(previous[1],b);else result.push([a,b]);return result;},[]); const actualCoverage=[]; for(let index=0;index<8;index++){ const capture=node.captures.filter(row=>row.kind==='request')[index],body=JSON.parse(bytes('run/'+capture.path)),segments=[]; const add=(value,content,offset=0)=>{if(value?.path!==testPath||typeof content!=='string')return; assert.equal(value.sha256,sha(testSource));const start=value.offset+offset;assert.equal(content,testSource.slice(start,start+content.length));segments.push([start,start+content.length]);}; for(const message of body.messages){ if(message.role==='tool'){const value=JSON.parse(message.content);add(value,value.content);} if(message.role!=='system')continue; const start=message.content.indexOf('\n{');if(start<0)continue;let value;try{value=JSON.parse(message.content.slice(start+1));}catch{continue;} if(value.data?.kind==='task-working-set')for(const entry of value.data.entries)add(entry.observation,entry.content,entry.content_offset??0); if(value.data?.kind==='latest-tool-feedback')for(const entry of value.data.entries)if(entry.tool==='read_file')add(entry.result,entry.result?.content_excerpt); } const ranges=union(segments),count=ranges.reduce((sum,[a,b])=>sum+b-a,0);actualCoverage.push(count); const expected=context.perRequest[index];assert.equal(expected.requestSha256,capture.sha256);assert.deepEqual(expected.files[0].ranges,ranges);assert.equal(expected.files[0].characters,count); assert.equal(expected.originalCheckJobIdMentionedAnywhere,bytes('run/'+capture.path).includes(checks.id)); } assert.deepEqual(actualCoverage,[0,0,0,400,1616,1060,27,421]); assert.ok(context.perRequest.every(row=>!row.files[0].fullSourcePresentInStructuredFileViews)); assert.deepEqual(context.perRequest.slice(5).map(row=>row.originalCheckJobIdMentionedAnywhere),[false,false,false]); assert.equal(replay.passed,true);assert.equal(replay.totalExternalModelRequests,0);assert.equal(replay.hostMutations,0); assert.deepEqual(replay.result[2].testCandidateRanges,[[473,2089],[2557,2584]]);assert.deepEqual(replay.result[2].missingTestRangesBeforeBudgetSelection,[[0,473],[2089,2557]]); assert.equal(replay.result[2].checkCandidatePresent,true); const ownedToolRows=owned('run/analysis/owned-tool-rows.json');assert.equal(ownedToolRows.length,16); for(const row of ownedToolRows)assert.equal(digest(row),proof.additions.records.find(item=>item.id===row.id)?.sha256); const artifactMap=new Map(artifacts.map(row=>[row.id,row]));assert.equal(artifacts.length,74); for(const artifact of artifacts){assert.ok(final.ownedSessions.includes(artifact.sessionId));assert.ok(final.ownedProjects.includes(artifact.projectId)); assert.equal(sha(bytes('run/'+artifact.capturedPath)),artifact.sha256);assert.equal(bytes('run/'+artifact.capturedPath).length,artifact.bytes);} for(const row of context.sourceResults){const artifact=artifactMap.get(row.artifactId);assert.ok(artifact);assert.equal(artifact.sha256,row.artifactSha256); const value=owned('run/'+artifact.capturedPath);assert.equal(value.content,row.content);assert.equal(value.offset,row.offset);} for(const result of checks.results){assert.equal(artifactMap.get(result.logArtifactId)?.kind,'check-log');} const snapshot=owned('run/final/snapshot.json'); for(const item of initial.baseline.exports)assert.deepEqual(snapshot.exports.find(row=>row.id===item.id),item); for(const table of ['sessions','records','events','outbox']){ const before=initial.baseline.database.tables[table],after=snapshot.database.tables[table];assert.equal(after.schemaSha256,before.schemaSha256); const rows=new Map(after.rows.map(row=>[row.id,row]));for(const row of before.rows)assert.deepEqual(rows.get(row.id),row); assert.equal(proof.additions[table].length,after.count-before.count); for(const row of proof.additions[table]){assert.ok(final.ownedSessions.includes(row.sessionId));if(row.projectId)assert.ok(final.ownedProjects.includes(row.projectId));assert.equal(rows.get(row.id).sha256,row.sha256);} } assert.equal(proof.additions.records.filter(row=>row.kind==='artifact').length,artifacts.length); for(const artifact of artifacts)assert.ok(proof.additions.records.some(row=>row.kind==='artifact'&&row.id===artifact.id)); const originalProjectRows=new Map(initial.baseline.database.tables.projects.rows.map(row=>[row.id,row])); const finalProjectRows=new Map(snapshot.database.tables.projects.rows.map(row=>[row.id,row])); for(const row of proof.projects){assert.equal(row.actualSha256,finalProjectRows.get(row.id).sha256); if(row.originalSha256){assert.equal(row.originalSha256,originalProjectRows.get(row.id).sha256);assert.equal(row.normalizedSha256,row.originalSha256);}else assert.ok(final.ownedProjects.includes(row.id));} assert.equal(proof.projects.filter(row=>row.originalSha256).length,63);assert.equal(proof.projects.filter(row=>row.owned).length,2);assert.equal(proof.projectInsertionOffset,2); const oldArtifactMap=new Map(initial.baseline.persistent.artifacts.map(row=>[row.path,row])),nextArtifactMap=new Map(snapshot.persistent.artifacts.map(row=>[row.path,row])); for(const [name,row]of oldArtifactMap)assert.deepEqual(nextArtifactMap.get(name),row); const addedFiles=snapshot.persistent.artifacts.filter(row=>!oldArtifactMap.has(row.path));assert.equal(addedFiles.length,60); assert.ok(addedFiles.every(row=>artifacts.some(artifact=>artifact.sha256===row.sha256&&artifact.bytes===row.bytes))); assert.deepEqual(snapshot.modelCenter.selected,initial.center.selected);assert.deepEqual(snapshot.modelCenter.pools,initial.center.pools); const beforeMeta=owned('run/node/original-public-metadata.json'),afterMeta=owned('run/node/final-public-metadata.json'); const metadataRows=new Map(snapshot.database.tables.metadata.rows.map(row=>[row.id,row])); const oldAttempts=owned('run/preflight/original-attempts.json');assert.equal(oldAttempts.length,216); assert.deepEqual(owned('run/node/original-attempts.json'),[...oldAttempts,...owned('run/preflight/attempts.json')]); assert.equal(metadataRows.get('modelAttempts').sha256,digest({key:'modelAttempts',value:JSON.stringify([...oldAttempts,...allAttempts])})); for(const [key,value]of[['config',afterMeta.config],['modelRegistry',afterMeta.registry]])assert.equal(metadataRows.get(key).sha256,digest({key,value:JSON.stringify(value)})); const oldNetwork=owned('run/network/history-before.json').jobs.map(stripDerived).reverse(),nextNetwork=owned('run/network/history-after.json').jobs.map(stripDerived).reverse(); assert.equal(oldNetwork.length,1);assert.deepEqual(nextNetwork,[...oldNetwork,stripDerived(network)]); assert.equal(initial.baseline.database.tables.metadata.rows.find(row=>row.id==='modelNetworkDiagnostics').sha256,digest({key:'modelNetworkDiagnostics',value:JSON.stringify(oldNetwork)})); assert.equal(metadataRows.get('modelNetworkDiagnostics').sha256,digest({key:'modelNetworkDiagnostics',value:JSON.stringify(nextNetwork)})); for(const row of initial.baseline.database.tables.metadata.rows)if(!['config','modelRegistry','modelAttempts','modelNetworkDiagnostics'].includes(row.id))assert.deepEqual(metadataRows.get(row.id),row); assert.deepEqual([...metadataRows.keys()].filter(key=>!initial.baseline.database.tables.metadata.rows.some(row=>row.id===key)),[]); assert.deepEqual({...afterMeta.config,configRevision:beforeMeta.config.configRevision},beforeMeta.config); const registry=structuredClone(afterMeta.registry);registry.revision=beforeMeta.registry.revision; const connection=registry.connections.find(row=>row.id===connectionId),oldConnection=beforeMeta.registry.connections.find(row=>row.id===connectionId); assert.equal(connection.revision,oldConnection.revision+2);connection.revision=oldConnection.revision;connection.updatedAt=oldConnection.updatedAt; const expectedRegistry=structuredClone(beforeMeta.registry);for(const value of[expectedRegistry,registry])value.connections.sort((a,b)=>a.id.localeCompare(b.id)); assert.deepEqual(canonical(registry),canonical(expectedRegistry));assert.deepEqual(owned('run/node/restoration.json').identityDefaultNormalization,{}); for(const name of ['src/amount.mjs','src/summary.mjs','src/report.mjs'])assert.deepEqual(bytes('run/node/source-before/'+name),bytes('run/node/source-after/'+name)); const git=owned('run/node/git-final.json'),baseline=owned('run/node/source-identity.json'); assert.equal(git.head,baseline.head);assert.equal(git.status,baseline.status);assert.equal(git.stagedAfter,baseline.stagedBefore);assert.equal(git.diff,''); assert.equal(final.passed,true);assert.equal(final.version,'1.24.7');assert.equal(final.pid,36552);assert.equal(final.inspectorClosed,true); assert.equal(final.originalProjects,63);assert.equal(final.originalSessions,79);assert.equal(final.originalArtifacts,3660);assert.equal(final.originalAttempts,216); assert.equal(final.finalProjects,65);assert.equal(final.finalSessions,81);assert.equal(final.finalArtifacts,3720);assert.equal(final.addedAttempts,9); assert.deepEqual(final.connectionRevision,[7,9]);assert.equal(final.credentialRevision,1);assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision,[69,73]); assert.equal(final.originalBudgetsSelectionAndPoolRestored,true);assert.equal(final.reportedTokens,reportedTokens);assert.equal(final.unknownRequests,0); assert.equal(final.nodeDelivered,false);assert.equal(final.purgeBenefitTested,false);assert.equal(final.productSourceChanged,false);assert.equal(final.keyReadOrResaved,false); let gitIndex; if(process.argv.includes('--index')){ const prefix=path.relative(process.cwd(),root).replaceAll('\\','/'),manifestBytes=bytes('manifest.json'); const entries=[...manifest.files,{path:'manifest.json',bytes:manifestBytes.length,sha256:sha(manifestBytes)}]; const output=execFileSync('git',['cat-file','--batch'],{input:entries.map(row=>':'+prefix+'/'+row.path).join('\n')+'\n',maxBuffer:256*1024*1024,windowsHide:true}); let offset=0,indexBytes=0;for(const entry of entries){const end=output.indexOf(10,offset),header=output.subarray(offset,end).toString('utf8'),match=header.match(/^[a-f0-9]{40} blob (\d+)$/);assert.ok(match,header); const size=Number(match[1]);offset=end+1;const value=output.subarray(offset,offset+size);offset+=size;assert.equal(output[offset++],10);assert.equal(size,entry.bytes,entry.path);assert.equal(sha(value),entry.sha256,entry.path);indexBytes+=size;} assert.equal(offset,output.length);gitIndex={passed:true,files:entries.length,bytes:indexBytes,exactOriginalBytes:true}; } console.log(JSON.stringify({success:true,files:manifest.files.length,bytes:totalBytes,manifestSha256:sha(bytes('manifest.json')), modelRequests:totalRequests,reportedTokens,unknownRequests:0,preflightPassed:true,nodeDelivered:false,originalPurgeBenefitTested:false, ownedArtifactRecords:artifacts.length,originalDataAndDefaultsVerified:true,actualRequestRetentionAndCandidateReplayVerified:true,productVersion:'1.24.7', ...(gitIndex?{gitIndex}:{}),scope:'Offline original bytes, frozen plan/source, original usage, file coverage, owned records and declared restoration. No network, host mutation, model or credential access.'}));