import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { Agent, fetch } from 'undici'; import { messagesOutputSetting } from '../../src/shared/contracts/model-capacity.mjs'; export let root = path.resolve('.qa/m4-live-baseline-2026-11-11'); export let origin = 'http://127.0.0.1:4318'; export function configureFixture(options) { const target = path.resolve(options.root), url = new URL(options.origin); assert.equal(path.dirname(target), path.resolve('.qa')); assert.match(path.basename(target), /^m4-driver-[a-f0-9-]{36}$/); assert.equal(url.hostname, '127.0.0.1'); assert.ok(url.port && url.port !== '4318'); root = target; origin = url.origin; } export const sha = bytes => createHash('sha256').update(bytes).digest('hex'); export const read = relative => JSON.parse(fs.readFileSync(path.join(root, relative), 'utf8')); export function save(relative, value) { const file = path.resolve(root, relative); assert.ok(file.startsWith(root + path.sep), 'Evidence must stay inside its new root.'); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' }); } export function connectionDefinition(value) { const { health, status, activeRequests, ...definition } = value; return definition; } export function compareConfig(before, after) { assert.ok(Number.isInteger(after.configRevision) && after.configRevision >= before.configRevision); assert.deepEqual({ ...after, configRevision: before.configRevision }, before); return { originalRevision: before.configRevision, currentRevision: after.configRevision, otherFields: 'exact' }; } export function compareConnectionDefinitions(before, after, beforeConfig, afterConfig) { assert.deepEqual(after.connections.map(value => value.id), before.connections.map(value => value.id)); const changes = []; for (let index = 0; index < before.connections.length; index++) { const original = connectionDefinition(before.connections[index]), current = connectionDefinition(after.connections[index]); if (original.id === 'legacy-default') { // The legacy public projection uses configRevision, rather than a saved // connection revision. Both deliberate select/restore saves advance it. const expected = original.revision + afterConfig.configRevision - beforeConfig.configRevision; assert.equal(current.revision, expected, 'Legacy projection tracks the exact declared config revision increment.'); changes.push({ id: original.id, originalRevision: original.revision, currentRevision: current.revision, reason: 'public-legacy-projection-of-configRevision' }); current.revision = original.revision; } assert.deepEqual(current, original, 'All original connection fields: ' + original.id); } return { originalFields: 'exact', declaredPublicProjectionChanges: changes }; } export function expectedOutputFields(connection) { const fields = Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'] .filter(key => Object.hasOwn(connection.nativeParameters ?? {}, key)).map(key => [key, connection.nativeParameters[key]])); if (connection.protocol !== 'anthropic-messages') return { fields, source: Object.keys(fields).length ? 'explicit_native' : 'passthrough_no_local_output_cap' }; const setting = messagesOutputSetting(connection); if (setting.value !== undefined) fields.max_tokens = setting.value; return { fields, source: setting.source, declaration: setting }; } export async function client() { const dispatcher = new Agent({ connect: { autoSelectFamily: true, timeout: 5000 } }); let token; const api = async (route, input, timeout = 30000) => { const response = await fetch(origin + '/api' + route, { dispatcher, method: input === undefined ? 'GET' : 'POST', redirect: 'error', signal: AbortSignal.timeout(timeout), headers: { ...(token ? { 'X-Workbench-Token': token } : {}), 'Content-Type': 'application/json', Connection: 'close' }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) }); const value = await response.json(); if (!response.ok) throw Object.assign(new Error(route + ': ' + (value.error ?? response.status)), { status: response.status, response: value }); return value; }; const boot = await api('/bootstrap'); token = boot.token; api.download = async (route, timeout = 30000) => { const response = await fetch(origin + '/api' + route, { dispatcher, method: 'GET', redirect: 'error', signal: AbortSignal.timeout(timeout), headers: { 'X-Workbench-Token': token, Connection: 'close' } }); if (!response.ok) throw Object.assign(new Error(route + ': artifact download HTTP ' + response.status), { status: response.status }); return Buffer.from(await response.arrayBuffer()); }; // The bootstrap authentication token is only kept in this client's closure. delete boot.token; return { api, boot, close: () => dispatcher.close() }; } export async function sessionAttempts(api, sessionId) { let cursor, total, snapshot; const rows = [], references = []; do { const page = await api('/model-center/attempts?sessionId=' + sessionId + '&limit=50' + (cursor ? '&cursor=' + encodeURIComponent(cursor) : '')); if (total === undefined) { total = page.total; snapshot = page.snapshotSequence; } assert.equal(page.total, total); assert.equal(page.snapshotSequence, snapshot); rows.push(...page.items); references.push(...page.references); cursor = page.nextCursor; assert.equal(page.hasMore, !!cursor); } while (cursor); assert.equal(rows.length, total); assert.equal(new Set(rows.map(value => value.id)).size, total); // Public pages are newest first. Store experiment receipts in dispatch order. return { rows: rows.reverse(), references: references.reverse(), total, snapshot }; } export async function artifactBytes(api, sessionId, artifact) { assert.equal(typeof api.download, 'function', 'Original artifacts need the byte-preserving download path.'); const bytes = await api.download('/artifacts/' + artifact.id + '?session=' + sessionId + '&download=1'); assert.equal(sha(bytes), artifact.sha256); if (artifact.bytes !== undefined) assert.equal(bytes.length, artifact.bytes); return bytes; } export async function captureSession(api, sessionId, prefix) { const state = await api('/sessions/' + sessionId), exported = await api('/sessions/' + sessionId + '/export'); save(prefix + '/session.json', state); save(prefix + '/export.json', exported); const ledger = await sessionAttempts(api, sessionId); save(prefix + '/attempts.json', ledger.rows); save(prefix + '/ledger-references.json', ledger); const captures = [], capturedIds = new Set(); for (const row of ledger.rows) for (const kind of ['request', 'response']) { const id = row[kind + 'ArtifactId']; if (!id) continue; const artifact = state.execution.artifacts.find(value => value.id === id) ?? { id, sha256: row[kind + 'Sha256'] }; const bytes = await artifactBytes(api, sessionId, artifact); assert.equal(sha(bytes), row[kind + 'Sha256']); const relative = prefix + '/captures/' + row.id + '-' + kind + (kind === 'request' ? '.json' : row.streaming ? '.sse' : '.txt'); save(relative, bytes); captures.push({ attemptId: row.id, artifactId: id, kind, path: relative, sha256: sha(bytes), bytes: bytes.length }); capturedIds.add(id); } for (const artifact of state.execution.artifacts) { if (capturedIds.has(artifact.id) || !/^(text\/|application\/(json|x-ndjson))/.test(artifact.mediaType ?? '')) continue; const bytes = await artifactBytes(api, sessionId, artifact), relative = prefix + '/artifacts/' + artifact.id + (artifact.mediaType === 'application/json' ? '.json' : '.txt'); save(relative, bytes); captures.push({ artifactId: artifact.id, kind: artifact.kind, path: relative, sha256: sha(bytes), bytes: bytes.length }); } save(prefix + '/artifact-sources.json', captures); return { state, exported, ...ledger, captures }; }