import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { fileURLToPath } from 'node:url'; // Offline verification only. This script never opens a model connection or reads credentials. const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const root = path.join(repo, 'docs/live-model-identity-j-evidence'); const sha = value => createHash('sha256').update(value).digest('hex'); const read = name => JSON.parse(fs.readFileSync(path.join(root, name))); const manifest = read('manifest.json'); const listed = new Set(); for (const file of manifest.entries) { assert.equal(path.posix.isAbsolute(file.path), false); assert.equal(file.path.includes('\\'), false); assert.equal(file.path.split('/').includes('..'), false); assert.equal(listed.has(file.path), false); listed.add(file.path); const bytes = fs.readFileSync(path.join(root, file.path)); assert.equal(bytes.length, file.bytes, file.path + ' length'); assert.equal(sha(bytes), file.sha256, file.path + ' original bytes'); } const actual = []; function walk(relative = '') { for (const entry of fs.readdirSync(path.join(root, relative), { withFileTypes: true })) { const name = path.posix.join(relative, entry.name); if (entry.isDirectory()) walk(name); else { assert.ok(entry.isFile(), 'Only regular evidence files'); if (name !== 'manifest.json') actual.push(name); } } } walk(); assert.deepEqual(actual.sort(), [...listed].sort()); assert.equal(manifest.count, listed.size); assert.equal(manifest.bytes, manifest.entries.reduce((n, f) => n + f.bytes, 0)); const contract = read('contract.json'), result = read('result.json'), attempts = read('attempts.json'); const state = read('session.json'), exported = read('session-export.json'); const requestBytes = fs.readFileSync(path.join(root, 'request.json')); const request = JSON.parse(requestBytes); const response = fs.readFileSync(path.join(root, 'response.sse')); assert.equal(contract.maxRequests, 1); assert.equal(contract.maxAttempts, 1); assert.equal(contract.maxOutputTokens, 256); assert.equal(contract.maxTaskTokens, 16000); assert.equal(contract.modelIdentityPolicy, 'require-match'); assert.deepEqual(contract.aliases, []); assert.equal(contract.poolSwitching, false); assert.equal(contract.automaticReplay, false); assert.equal(attempts.length, 1); const attempt = attempts[0]; assert.equal(attempt.connectionId, contract.connectionId); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1); assert.equal(attempt.requestSha256, sha(JSON.stringify(request))); assert.equal(attempt.responseSha256, sha(response)); assert.equal(attempt.responseRedacted, false); assert.equal(request.model, contract.requestedModel); assert.equal(request.model, 'claude-opus-5-5'); assert.equal(request.max_tokens ?? request.max_completion_tokens, 256); assert.equal(request.stream, true); const data = response.toString('utf8').split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trim()); assert.equal(data.at(-1), '[DONE]'); const packets = data.filter(line => line !== '[DONE]').map(line => JSON.parse(line)); const models = [...new Set(packets.filter(p => Object.hasOwn(p, 'model')).map(p => p.model))]; assert.deepEqual(models, ['gpt-6.1-sol']); assert.notEqual(request.model, models[0]); const rawUsage = packets.findLast(p => p.usage)?.usage; const usage = Object.fromEntries(['prompt_tokens', 'completion_tokens', 'total_tokens'].map(key => [key, rawUsage[key]])); assert.deepEqual(rawUsage.prompt_tokens_details, { cache_write_tokens: 4304 }); assert.deepEqual(usage, attempt.usage); assert.equal(usage.prompt_tokens + usage.completion_tokens, usage.total_tokens); assert.equal(usage.total_tokens, 4334); assert.ok(usage.total_tokens <= contract.maxTaskTokens); assert.equal(packets.flatMap(p => p.choices ?? []).flatMap(c => c.delta?.tool_calls ?? []).length, 0); const answer = packets.flatMap(p => p.choices ?? []).map(c => c.delta?.content ?? '').join(''); assert.equal(answer, 'OK ' + contract.nonce); assert.equal(state.status, 'paused'); assert.equal(state.execution.run.stopReason, 'model_identity'); assert.equal(state.execution.progress.totalTools, 0); assert.equal(state.execution.accountedTaskTokens, 4334); assert.deepEqual(state.execution.run.budget, { maxSteps: 1, maxDurationMs: 320000, maxTokens: 16000, maxTaskTokens: 16000 }); const consumption = state.execution.run.consumption; assert.equal(consumption.length, 1); assert.equal(consumption[0].attemptId, attempt.id); assert.deepEqual(consumption[0].usage, usage); assert.deepEqual(consumption[0].modelIdentity.reportedModels, models); assert.equal(consumption[0].modelIdentity.policy, 'require-match'); assert.equal(consumption[0].modelIdentity.complete, true); assert.equal(consumption[0].modelIdentity.status, 'mismatch'); assert.equal(consumption[0].modelIdentity.responseSource.sha256, sha(response)); assert.ok(consumption[0].localEstimate.inputTokens + consumption[0].localEstimate.outputTokens <= contract.maxTaskTokens); assert.equal(attempt.status, 'failed'); assert.equal(attempt.diagnostic.kind, 'model_identity'); assert.equal(attempt.diagnostic.retryable, false); assert.equal(attempt.diagnostic.failoverAllowed, false); assert.equal(result.identityPassed, false); assert.equal(result.engineeringTestAllowed, false); assert.equal(result.realEngineeringDeliveryTested, false); assert.equal(result.purgeBenefitTested, false); assert.equal(result.modelWeightsAuthenticated, false); assert.deepEqual(result.usage, usage); assert.equal(exported.session.id, state.id); assert.deepEqual(exported.execution, state.execution); for (const capture of result.captures) assert.equal(capture.sha256, sha(fs.readFileSync(path.join(root, capture.file)))); const restoration = read('restoration.json'), audit = read('state-audit.json'), baseline = read('baseline-summary.json'); assert.equal(restoration.passed, true); assert.equal(audit.passed, true); assert.equal(restoration.pid, 36552); assert.equal(restoration.addedAttempts, 1); assert.equal(restoration.originalExports, 70); assert.equal(restoration.originalArtifactFiles, 3566); assert.equal(restoration.originalCredentialRevision, restoration.restoredCredentialRevision); assert.equal(restoration.originalSelectionRestored, true); assert.equal(audit.baselineSha256, baseline.beforeSha256); assert.equal(audit.afterSha256, baseline.afterSha256); for (const table of audit.checks) assert.equal(table.exact + table.normalized, table.original); assert.equal(audit.additions.length, 60); assert.deepEqual(audit.diskConfigRevisions, { original: 48, actual: 52 }); console.log(JSON.stringify({ passed: true, files: manifest.count, bytes: manifest.bytes, requests: 1, requestedModel: request.model, reportedModels: models, identityPassed: false, reportedTokens: usage.total_tokens, engineeringTested: false, purgeBenefitTested: false, scope: 'Offline original-artifact and receipt consistency. No model request; no authentication of supplier model weights or reconstruction of excluded private original user data.' }));