95 lines
7.0 KiB
JavaScript
95 lines
7.0 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
// Offline verification only. This script never opens a model connection or reads credentials.
|
|
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
const root = path.join(repo, 'docs/live-model-identity-j-evidence');
|
|
const sha = value => createHash('sha256').update(value).digest('hex');
|
|
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
|
const manifest = read('manifest.json');
|
|
const listed = new Set();
|
|
for (const file of manifest.entries) {
|
|
assert.equal(path.posix.isAbsolute(file.path), false);
|
|
assert.equal(file.path.includes('\\'), false);
|
|
assert.equal(file.path.split('/').includes('..'), false);
|
|
assert.equal(listed.has(file.path), false); listed.add(file.path);
|
|
const bytes = fs.readFileSync(path.join(root, file.path));
|
|
assert.equal(bytes.length, file.bytes, file.path + ' length');
|
|
assert.equal(sha(bytes), file.sha256, file.path + ' original bytes');
|
|
}
|
|
const actual = [];
|
|
function walk(relative = '') {
|
|
for (const entry of fs.readdirSync(path.join(root, relative), { withFileTypes: true })) {
|
|
const name = path.posix.join(relative, entry.name);
|
|
if (entry.isDirectory()) walk(name);
|
|
else { assert.ok(entry.isFile(), 'Only regular evidence files'); if (name !== 'manifest.json') actual.push(name); }
|
|
}
|
|
}
|
|
walk(); assert.deepEqual(actual.sort(), [...listed].sort());
|
|
assert.equal(manifest.count, listed.size);
|
|
assert.equal(manifest.bytes, manifest.entries.reduce((n, f) => n + f.bytes, 0));
|
|
const contract = read('contract.json'), result = read('result.json'), attempts = read('attempts.json');
|
|
const state = read('session.json'), exported = read('session-export.json');
|
|
const requestBytes = fs.readFileSync(path.join(root, 'request.json'));
|
|
const request = JSON.parse(requestBytes);
|
|
const response = fs.readFileSync(path.join(root, 'response.sse'));
|
|
assert.equal(contract.maxRequests, 1); assert.equal(contract.maxAttempts, 1);
|
|
assert.equal(contract.maxOutputTokens, 256); assert.equal(contract.maxTaskTokens, 16000);
|
|
assert.equal(contract.modelIdentityPolicy, 'require-match'); assert.deepEqual(contract.aliases, []);
|
|
assert.equal(contract.poolSwitching, false); assert.equal(contract.automaticReplay, false);
|
|
assert.equal(attempts.length, 1);
|
|
const attempt = attempts[0];
|
|
assert.equal(attempt.connectionId, contract.connectionId); assert.equal(attempt.poolId, null);
|
|
assert.equal(attempt.ordinal, 1); assert.equal(attempt.requestSha256, sha(JSON.stringify(request)));
|
|
assert.equal(attempt.responseSha256, sha(response)); assert.equal(attempt.responseRedacted, false);
|
|
assert.equal(request.model, contract.requestedModel); assert.equal(request.model, 'claude-opus-5-5');
|
|
assert.equal(request.max_tokens ?? request.max_completion_tokens, 256); assert.equal(request.stream, true);
|
|
const data = response.toString('utf8').split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trim());
|
|
assert.equal(data.at(-1), '[DONE]');
|
|
const packets = data.filter(line => line !== '[DONE]').map(line => JSON.parse(line));
|
|
const models = [...new Set(packets.filter(p => Object.hasOwn(p, 'model')).map(p => p.model))];
|
|
assert.deepEqual(models, ['gpt-6.1-sol']); assert.notEqual(request.model, models[0]);
|
|
const rawUsage = packets.findLast(p => p.usage)?.usage;
|
|
const usage = Object.fromEntries(['prompt_tokens', 'completion_tokens', 'total_tokens'].map(key => [key, rawUsage[key]]));
|
|
assert.deepEqual(rawUsage.prompt_tokens_details, { cache_write_tokens: 4304 });
|
|
assert.deepEqual(usage, attempt.usage);
|
|
assert.equal(usage.prompt_tokens + usage.completion_tokens, usage.total_tokens);
|
|
assert.equal(usage.total_tokens, 4334); assert.ok(usage.total_tokens <= contract.maxTaskTokens);
|
|
assert.equal(packets.flatMap(p => p.choices ?? []).flatMap(c => c.delta?.tool_calls ?? []).length, 0);
|
|
const answer = packets.flatMap(p => p.choices ?? []).map(c => c.delta?.content ?? '').join('');
|
|
assert.equal(answer, 'OK ' + contract.nonce);
|
|
assert.equal(state.status, 'paused'); assert.equal(state.execution.run.stopReason, 'model_identity');
|
|
assert.equal(state.execution.progress.totalTools, 0); assert.equal(state.execution.accountedTaskTokens, 4334);
|
|
assert.deepEqual(state.execution.run.budget, { maxSteps: 1, maxDurationMs: 320000, maxTokens: 16000, maxTaskTokens: 16000 });
|
|
const consumption = state.execution.run.consumption;
|
|
assert.equal(consumption.length, 1); assert.equal(consumption[0].attemptId, attempt.id);
|
|
assert.deepEqual(consumption[0].usage, usage); assert.deepEqual(consumption[0].modelIdentity.reportedModels, models);
|
|
assert.equal(consumption[0].modelIdentity.policy, 'require-match'); assert.equal(consumption[0].modelIdentity.complete, true);
|
|
assert.equal(consumption[0].modelIdentity.status, 'mismatch');
|
|
assert.equal(consumption[0].modelIdentity.responseSource.sha256, sha(response));
|
|
assert.ok(consumption[0].localEstimate.inputTokens + consumption[0].localEstimate.outputTokens <= contract.maxTaskTokens);
|
|
assert.equal(attempt.status, 'failed'); assert.equal(attempt.diagnostic.kind, 'model_identity');
|
|
assert.equal(attempt.diagnostic.retryable, false); assert.equal(attempt.diagnostic.failoverAllowed, false);
|
|
assert.equal(result.identityPassed, false); assert.equal(result.engineeringTestAllowed, false);
|
|
assert.equal(result.realEngineeringDeliveryTested, false); assert.equal(result.purgeBenefitTested, false);
|
|
assert.equal(result.modelWeightsAuthenticated, false); assert.deepEqual(result.usage, usage);
|
|
assert.equal(exported.session.id, state.id); assert.deepEqual(exported.execution, state.execution);
|
|
for (const capture of result.captures) assert.equal(capture.sha256, sha(fs.readFileSync(path.join(root, capture.file))));
|
|
const restoration = read('restoration.json'), audit = read('state-audit.json'), baseline = read('baseline-summary.json');
|
|
assert.equal(restoration.passed, true); assert.equal(audit.passed, true);
|
|
assert.equal(restoration.pid, 36552); assert.equal(restoration.addedAttempts, 1);
|
|
assert.equal(restoration.originalExports, 70); assert.equal(restoration.originalArtifactFiles, 3566);
|
|
assert.equal(restoration.originalCredentialRevision, restoration.restoredCredentialRevision);
|
|
assert.equal(restoration.originalSelectionRestored, true);
|
|
assert.equal(audit.baselineSha256, baseline.beforeSha256); assert.equal(audit.afterSha256, baseline.afterSha256);
|
|
for (const table of audit.checks) assert.equal(table.exact + table.normalized, table.original);
|
|
assert.equal(audit.additions.length, 60);
|
|
assert.deepEqual(audit.diskConfigRevisions, { original: 48, actual: 52 });
|
|
console.log(JSON.stringify({ passed: true, files: manifest.count, bytes: manifest.bytes, requests: 1,
|
|
requestedModel: request.model, reportedModels: models, identityPassed: false, reportedTokens: usage.total_tokens,
|
|
engineeringTested: false, purgeBenefitTested: false,
|
|
scope: 'Offline original-artifact and receipt consistency. No model request; no authentication of supplier model weights or reconstruction of excluded private original user data.' }));
|