Files
Agent-Workbench/scripts/verify-live-model-identity-j.mjs
2026-11-03 13:32:06 +08:00

95 lines
7.0 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { fileURLToPath } from 'node:url';
// Offline verification only. This script never opens a model connection or reads credentials.
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const root = path.join(repo, 'docs/live-model-identity-j-evidence');
const sha = value => createHash('sha256').update(value).digest('hex');
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
const manifest = read('manifest.json');
const listed = new Set();
for (const file of manifest.entries) {
assert.equal(path.posix.isAbsolute(file.path), false);
assert.equal(file.path.includes('\\'), false);
assert.equal(file.path.split('/').includes('..'), false);
assert.equal(listed.has(file.path), false); listed.add(file.path);
const bytes = fs.readFileSync(path.join(root, file.path));
assert.equal(bytes.length, file.bytes, file.path + ' length');
assert.equal(sha(bytes), file.sha256, file.path + ' original bytes');
}
const actual = [];
function walk(relative = '') {
for (const entry of fs.readdirSync(path.join(root, relative), { withFileTypes: true })) {
const name = path.posix.join(relative, entry.name);
if (entry.isDirectory()) walk(name);
else { assert.ok(entry.isFile(), 'Only regular evidence files'); if (name !== 'manifest.json') actual.push(name); }
}
}
walk(); assert.deepEqual(actual.sort(), [...listed].sort());
assert.equal(manifest.count, listed.size);
assert.equal(manifest.bytes, manifest.entries.reduce((n, f) => n + f.bytes, 0));
const contract = read('contract.json'), result = read('result.json'), attempts = read('attempts.json');
const state = read('session.json'), exported = read('session-export.json');
const requestBytes = fs.readFileSync(path.join(root, 'request.json'));
const request = JSON.parse(requestBytes);
const response = fs.readFileSync(path.join(root, 'response.sse'));
assert.equal(contract.maxRequests, 1); assert.equal(contract.maxAttempts, 1);
assert.equal(contract.maxOutputTokens, 256); assert.equal(contract.maxTaskTokens, 16000);
assert.equal(contract.modelIdentityPolicy, 'require-match'); assert.deepEqual(contract.aliases, []);
assert.equal(contract.poolSwitching, false); assert.equal(contract.automaticReplay, false);
assert.equal(attempts.length, 1);
const attempt = attempts[0];
assert.equal(attempt.connectionId, contract.connectionId); assert.equal(attempt.poolId, null);
assert.equal(attempt.ordinal, 1); assert.equal(attempt.requestSha256, sha(JSON.stringify(request)));
assert.equal(attempt.responseSha256, sha(response)); assert.equal(attempt.responseRedacted, false);
assert.equal(request.model, contract.requestedModel); assert.equal(request.model, 'claude-opus-5-5');
assert.equal(request.max_tokens ?? request.max_completion_tokens, 256); assert.equal(request.stream, true);
const data = response.toString('utf8').split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trim());
assert.equal(data.at(-1), '[DONE]');
const packets = data.filter(line => line !== '[DONE]').map(line => JSON.parse(line));
const models = [...new Set(packets.filter(p => Object.hasOwn(p, 'model')).map(p => p.model))];
assert.deepEqual(models, ['gpt-6.1-sol']); assert.notEqual(request.model, models[0]);
const rawUsage = packets.findLast(p => p.usage)?.usage;
const usage = Object.fromEntries(['prompt_tokens', 'completion_tokens', 'total_tokens'].map(key => [key, rawUsage[key]]));
assert.deepEqual(rawUsage.prompt_tokens_details, { cache_write_tokens: 4304 });
assert.deepEqual(usage, attempt.usage);
assert.equal(usage.prompt_tokens + usage.completion_tokens, usage.total_tokens);
assert.equal(usage.total_tokens, 4334); assert.ok(usage.total_tokens <= contract.maxTaskTokens);
assert.equal(packets.flatMap(p => p.choices ?? []).flatMap(c => c.delta?.tool_calls ?? []).length, 0);
const answer = packets.flatMap(p => p.choices ?? []).map(c => c.delta?.content ?? '').join('');
assert.equal(answer, 'OK ' + contract.nonce);
assert.equal(state.status, 'paused'); assert.equal(state.execution.run.stopReason, 'model_identity');
assert.equal(state.execution.progress.totalTools, 0); assert.equal(state.execution.accountedTaskTokens, 4334);
assert.deepEqual(state.execution.run.budget, { maxSteps: 1, maxDurationMs: 320000, maxTokens: 16000, maxTaskTokens: 16000 });
const consumption = state.execution.run.consumption;
assert.equal(consumption.length, 1); assert.equal(consumption[0].attemptId, attempt.id);
assert.deepEqual(consumption[0].usage, usage); assert.deepEqual(consumption[0].modelIdentity.reportedModels, models);
assert.equal(consumption[0].modelIdentity.policy, 'require-match'); assert.equal(consumption[0].modelIdentity.complete, true);
assert.equal(consumption[0].modelIdentity.status, 'mismatch');
assert.equal(consumption[0].modelIdentity.responseSource.sha256, sha(response));
assert.ok(consumption[0].localEstimate.inputTokens + consumption[0].localEstimate.outputTokens <= contract.maxTaskTokens);
assert.equal(attempt.status, 'failed'); assert.equal(attempt.diagnostic.kind, 'model_identity');
assert.equal(attempt.diagnostic.retryable, false); assert.equal(attempt.diagnostic.failoverAllowed, false);
assert.equal(result.identityPassed, false); assert.equal(result.engineeringTestAllowed, false);
assert.equal(result.realEngineeringDeliveryTested, false); assert.equal(result.purgeBenefitTested, false);
assert.equal(result.modelWeightsAuthenticated, false); assert.deepEqual(result.usage, usage);
assert.equal(exported.session.id, state.id); assert.deepEqual(exported.execution, state.execution);
for (const capture of result.captures) assert.equal(capture.sha256, sha(fs.readFileSync(path.join(root, capture.file))));
const restoration = read('restoration.json'), audit = read('state-audit.json'), baseline = read('baseline-summary.json');
assert.equal(restoration.passed, true); assert.equal(audit.passed, true);
assert.equal(restoration.pid, 36552); assert.equal(restoration.addedAttempts, 1);
assert.equal(restoration.originalExports, 70); assert.equal(restoration.originalArtifactFiles, 3566);
assert.equal(restoration.originalCredentialRevision, restoration.restoredCredentialRevision);
assert.equal(restoration.originalSelectionRestored, true);
assert.equal(audit.baselineSha256, baseline.beforeSha256); assert.equal(audit.afterSha256, baseline.afterSha256);
for (const table of audit.checks) assert.equal(table.exact + table.normalized, table.original);
assert.equal(audit.additions.length, 60);
assert.deepEqual(audit.diskConfigRevisions, { original: 48, actual: 52 });
console.log(JSON.stringify({ passed: true, files: manifest.count, bytes: manifest.bytes, requests: 1,
requestedModel: request.model, reportedModels: models, identityPassed: false, reportedTokens: usage.total_tokens,
engineeringTested: false, purgeBenefitTested: false,
scope: 'Offline original-artifact and receipt consistency. No model request; no authentication of supplier model weights or reconstruction of excluded private original user data.' }));