Files
Agent-Workbench/scripts/official-contract-check.mjs

105 lines
8.2 KiB
JavaScript

import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { modelRegistry, validateModelRegistry } from '../src/shared/contracts/official-model-registry.mjs';
import { REQUEST_CONTRACT_SCHEMA, REQUEST_CONTRACT_DEFAULTS_VERSION, REQUEST_CONTRACT_SOURCES, requestContractSettings } from '../src/shared/contracts/model-request-contract.mjs';
export const sha256 = value => createHash('sha256').update(value).digest('hex');
export const canonical = value => Array.isArray(value) ? value.map(canonical)
: value && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
const same = (a, b) => JSON.stringify(canonical(a)) === JSON.stringify(canonical(b));
const recordKey = value => JSON.stringify([value.provider, value.protocol, value.apiVersion, value.model]);
export const SOURCE_MANIFESTS = [
'tests/fixtures/official-model-registry/SOURCE.json',
'tests/fixtures/model-request-contract/SOURCE.json',
'tests/fixtures/native-cache-state/SOURCE.json',
'tests/fixtures/ci-sources/SOURCE.json',
];
export function sourceInventory(root) {
return SOURCE_MANIFESTS.map(relative => {
const file = path.join(root, relative), bytes = fs.readFileSync(file), manifest = JSON.parse(bytes);
assert.ok(Array.isArray(manifest.sources), `${relative}: missing source list`);
const names = new Set();
const sources = manifest.sources.map(source => {
assert.equal(names.has(source.id), false, `duplicate source ${source.id}`); names.add(source.id);
const sourceFile = path.resolve(path.dirname(file), source.file);
assert.ok(sourceFile.startsWith(path.join(root,'tests/fixtures') + path.sep), `source escapes pinned fixtures ${source.id}`);
assert.equal(fs.lstatSync(sourceFile).isSymbolicLink(), false, `source link ${source.id}`);
const raw = fs.readFileSync(sourceFile);
assert.equal(sha256(raw), source.sha256, `original source SHA changed: ${source.id}`);
return { id: source.id, file: source.file, url: source.url, sha256: source.sha256, bytes: raw.length, accepted: source.accepted !== false };
}).sort((a,b) => a.id.localeCompare(b.id));
return { file: relative, sha256: sha256(bytes), sources };
});
}
export function officialSnapshot(root) {
validateModelRegistry(modelRegistry);
const defaults = {};
for (const [provider, baseUrl, protocol] of [
['anthropic', 'https://api.anthropic.com', 'anthropic-messages'],
['deepseek', 'https://api.deepseek.com', 'anthropic-messages'],
['openai', 'https://api.openai.com/v1', 'responses'],
]) for (const existing of [false, true]) defaults[provider + (existing ? ':legacy' : ':new')] = requestContractSettings(undefined, { baseUrl, protocol }, { existing });
return canonical({ schemaVersion: 1,
registry: { ...structuredClone(modelRegistry), sources: [...modelRegistry.sources].sort((a,b)=>a.id.localeCompare(b.id)), records: [...modelRegistry.records].sort((a,b)=>recordKey(a).localeCompare(recordKey(b))) },
requestContract: { schemaVersion: REQUEST_CONTRACT_SCHEMA, defaultsVersion: REQUEST_CONTRACT_DEFAULTS_VERSION, sources: REQUEST_CONTRACT_SOURCES, defaults },
sourceInventory: sourceInventory(root),
});
}
export function contractDifference(before, after) {
assert.equal(before?.schemaVersion, 1, 'old contract snapshot schema'); assert.equal(after?.schemaVersion, 1, 'new contract snapshot schema');
const changes = [];
const push = (category, key, oldValue, newValue) => changes.push({ category, key, before: oldValue ?? null, after: newValue ?? null });
const beforeRecords = new Map(before.registry.records.map(item=>[recordKey(item),item]));
const afterRecords = new Map(after.registry.records.map(item=>[recordKey(item),item]));
for (const key of [...new Set([...beforeRecords.keys(), ...afterRecords.keys()])].sort()) {
const previous = beforeRecords.get(key), current = afterRecords.get(key);
if (!previous || !current) { push('model-protocol-api-version', key, previous, current); continue; }
for (const field of [...new Set([...Object.keys(previous.fields), ...Object.keys(current.fields)])].sort()) {
if (same(previous.fields[field], current.fields[field])) continue;
const category = /lifecycle|retired|alias/i.test(field) ? 'deprecation' : /cache|state/i.test(field) ? 'cache-or-state' : 'model-field';
push(category, key + '/' + field, previous.fields[field], current.fields[field]);
}
}
if (!same(before.registry.sources, after.registry.sources)) push('official-source', 'registry.sources', before.registry.sources, after.registry.sources);
if (!same(before.sourceInventory, after.sourceInventory)) push('original-source-bytes', 'sourceInventory', before.sourceInventory, after.sourceInventory);
if (!same(before.requestContract, after.requestContract)) push('request-version-auth-defaults', 'requestContract', before.requestContract, after.requestContract);
if (before.registry.version !== after.registry.version) push('registry-version', 'registry.version', before.registry.version, after.registry.version);
if (before.registry.schemaVersion !== after.registry.schemaVersion) push('registry-schema', 'registry.schemaVersion', before.registry.schemaVersion, after.registry.schemaVersion);
const registryChanged = changes.some(item=>['model-protocol-api-version','deprecation','cache-or-state','model-field','official-source','registry-schema'].includes(item.category));
return { changes, registryVersionChanged: before.registry.version !== after.registry.version,
missingRegistryVersionBump: registryChanged && before.registry.version === after.registry.version,
missingRequestVersionBump: !same(before.requestContract, after.requestContract) && before.requestContract.defaultsVersion === after.requestContract.defaultsVersion,
equal: same(before, after), reviewRequired: !same(before, after), scope: 'pinned specifications and regression inputs; no live model compatibility claim' };
}
export function verifyOfficialContracts(root) {
const indexFile = path.join(root, 'tests/fixtures/official-contracts/INDEX.json'), index = JSON.parse(fs.readFileSync(indexFile));
assert.equal(index.schemaVersion, 1); assert.ok(Array.isArray(index.history) && index.history.at(-1) === index.current);
const read = name => { assert.match(name, /^[a-zA-Z0-9_.-]+\.json$/); return JSON.parse(fs.readFileSync(path.join(path.dirname(indexFile), name))); };
const baseline = read(index.current), current = officialSnapshot(root), difference = contractDifference(baseline, current);
assert.equal(difference.equal, true, 'Official contract changed. Preserve the old snapshot, review source/version/deprecation differences and update the registry and regression baseline together.');
const suites = index.regressions;
assert.ok(Array.isArray(suites) && suites.length > 0);
const testFiles = new Set();
for (const suite of suites) {
assert.ok(typeof suite.purpose === 'string' && suite.purpose.length && Array.isArray(suite.files) && suite.files.length);
for (const file of suite.files) { assert.match(file, /^tests\/[a-z0-9-]+\.test\.mjs$/); assert.ok(fs.statSync(path.join(root, file)).isFile(), file); testFiles.add(file); }
}
const histories = index.history.map(read);
const transitions = histories.slice(1).map((value, i) => contractDifference(histories[i], value));
for (const difference of transitions) {
assert.equal(difference.missingRegistryVersionBump, false, 'Historical registry fields changed without a new version');
assert.equal(difference.missingRequestVersionBump, false, 'Historical request defaults changed without a new version');
}
return { schemaVersion: 1, passed: true, registryVersion: current.registry.version, requestDefaultsVersion: current.requestContract.defaultsVersion,
snapshotFile: index.current, snapshotSha256: sha256(fs.readFileSync(path.join(path.dirname(indexFile), index.current))),
sourceInventory: current.sourceInventory, recordsCount: current.registry.records.length, testFiles: [...testFiles].sort(), transitions,
scope: 'Fixed official bytes, model/protocol/API version facts, request defaults and named deterministic regressions. Not a live-service acceptance.' };
}