Files
Agent-Workbench/scripts/m4-live/common.mjs
T
2026-11-11 22:58:21 +08:00

120 lines
8.0 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { Agent, fetch } from 'undici';
import { messagesOutputSetting } from '../../src/shared/contracts/model-capacity.mjs';
export let root = path.resolve('.qa/m4-live-baseline-2026-11-11');
export let origin = 'http://127.0.0.1:4318';
export function configureFixture(options) {
const target = path.resolve(options.root), url = new URL(options.origin);
assert.equal(path.dirname(target), path.resolve('.qa'));
assert.match(path.basename(target), /^m4-driver-[a-f0-9-]{36}$/);
assert.equal(url.hostname, '127.0.0.1'); assert.ok(url.port && url.port !== '4318');
root = target; origin = url.origin;
}
export const sha = bytes => createHash('sha256').update(bytes).digest('hex');
export const read = relative => JSON.parse(fs.readFileSync(path.join(root, relative), 'utf8'));
export function save(relative, value) {
const file = path.resolve(root, relative);
assert.ok(file.startsWith(root + path.sep), 'Evidence must stay inside its new root.');
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
}
export function connectionDefinition(value) {
const { health, status, activeRequests, ...definition } = value;
return definition;
}
export function compareConfig(before, after) {
assert.ok(Number.isInteger(after.configRevision) && after.configRevision >= before.configRevision);
assert.deepEqual({ ...after, configRevision: before.configRevision }, before);
return { originalRevision: before.configRevision, currentRevision: after.configRevision, otherFields: 'exact' };
}
export function compareConnectionDefinitions(before, after, beforeConfig, afterConfig) {
assert.deepEqual(after.connections.map(value => value.id), before.connections.map(value => value.id));
const changes = [];
for (let index = 0; index < before.connections.length; index++) {
const original = connectionDefinition(before.connections[index]), current = connectionDefinition(after.connections[index]);
if (original.id === 'legacy-default') {
// The legacy public projection uses configRevision, rather than a saved
// connection revision. Both deliberate select/restore saves advance it.
const expected = original.revision + afterConfig.configRevision - beforeConfig.configRevision;
assert.equal(current.revision, expected, 'Legacy projection tracks the exact declared config revision increment.');
changes.push({ id: original.id, originalRevision: original.revision, currentRevision: current.revision, reason: 'public-legacy-projection-of-configRevision' });
current.revision = original.revision;
}
assert.deepEqual(current, original, 'All original connection fields: ' + original.id);
}
return { originalFields: 'exact', declaredPublicProjectionChanges: changes };
}
export function expectedOutputFields(connection) {
const fields = Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens']
.filter(key => Object.hasOwn(connection.nativeParameters ?? {}, key)).map(key => [key, connection.nativeParameters[key]]));
if (connection.protocol !== 'anthropic-messages') return { fields, source: Object.keys(fields).length ? 'explicit_native' : 'passthrough_no_local_output_cap' };
const setting = messagesOutputSetting(connection);
if (setting.value !== undefined) fields.max_tokens = setting.value;
return { fields, source: setting.source, declaration: setting };
}
export async function client() {
const dispatcher = new Agent({ connect: { autoSelectFamily: true, timeout: 5000 } }); let token;
const api = async (route, input, timeout = 30000) => {
const response = await fetch(origin + '/api' + route, { dispatcher, method: input === undefined ? 'GET' : 'POST', redirect: 'error', signal: AbortSignal.timeout(timeout),
headers: { ...(token ? { 'X-Workbench-Token': token } : {}), 'Content-Type': 'application/json', Connection: 'close' },
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const value = await response.json();
if (!response.ok) throw Object.assign(new Error(route + ': ' + (value.error ?? response.status)), { status: response.status, response: value });
return value;
};
const boot = await api('/bootstrap'); token = boot.token;
api.download = async (route, timeout = 30000) => {
const response = await fetch(origin + '/api' + route, { dispatcher, method: 'GET', redirect: 'error', signal: AbortSignal.timeout(timeout),
headers: { 'X-Workbench-Token': token, Connection: 'close' } });
if (!response.ok) throw Object.assign(new Error(route + ': artifact download HTTP ' + response.status), { status: response.status });
return Buffer.from(await response.arrayBuffer());
};
// The bootstrap authentication token is only kept in this client's closure.
delete boot.token;
return { api, boot, close: () => dispatcher.close() };
}
export async function sessionAttempts(api, sessionId) {
let cursor, total, snapshot; const rows = [], references = [];
do {
const page = await api('/model-center/attempts?sessionId=' + sessionId + '&limit=50' + (cursor ? '&cursor=' + encodeURIComponent(cursor) : ''));
if (total === undefined) { total = page.total; snapshot = page.snapshotSequence; }
assert.equal(page.total, total); assert.equal(page.snapshotSequence, snapshot);
rows.push(...page.items); references.push(...page.references); cursor = page.nextCursor;
assert.equal(page.hasMore, !!cursor);
} while (cursor);
assert.equal(rows.length, total); assert.equal(new Set(rows.map(value => value.id)).size, total);
// Public pages are newest first. Store experiment receipts in dispatch order.
return { rows: rows.reverse(), references: references.reverse(), total, snapshot };
}
export async function artifactBytes(api, sessionId, artifact) {
assert.equal(typeof api.download, 'function', 'Original artifacts need the byte-preserving download path.');
const bytes = await api.download('/artifacts/' + artifact.id + '?session=' + sessionId + '&download=1');
assert.equal(sha(bytes), artifact.sha256); if (artifact.bytes !== undefined) assert.equal(bytes.length, artifact.bytes);
return bytes;
}
export async function captureSession(api, sessionId, prefix) {
const state = await api('/sessions/' + sessionId), exported = await api('/sessions/' + sessionId + '/export');
save(prefix + '/session.json', state); save(prefix + '/export.json', exported);
const ledger = await sessionAttempts(api, sessionId); save(prefix + '/attempts.json', ledger.rows); save(prefix + '/ledger-references.json', ledger);
const captures = [], capturedIds = new Set();
for (const row of ledger.rows) for (const kind of ['request', 'response']) {
const id = row[kind + 'ArtifactId']; if (!id) continue;
const artifact = state.execution.artifacts.find(value => value.id === id) ?? { id, sha256: row[kind + 'Sha256'] };
const bytes = await artifactBytes(api, sessionId, artifact);
assert.equal(sha(bytes), row[kind + 'Sha256']);
const relative = prefix + '/captures/' + row.id + '-' + kind + (kind === 'request' ? '.json' : row.streaming ? '.sse' : '.txt');
save(relative, bytes); captures.push({ attemptId: row.id, artifactId: id, kind, path: relative, sha256: sha(bytes), bytes: bytes.length }); capturedIds.add(id);
}
for (const artifact of state.execution.artifacts) {
if (capturedIds.has(artifact.id) || !/^(text\/|application\/(json|x-ndjson))/.test(artifact.mediaType ?? '')) continue;
const bytes = await artifactBytes(api, sessionId, artifact), relative = prefix + '/artifacts/' + artifact.id + (artifact.mediaType === 'application/json' ? '.json' : '.txt');
save(relative, bytes); captures.push({ artifactId: artifact.id, kind: artifact.kind, path: relative, sha256: sha(bytes), bytes: bytes.length });
}
save(prefix + '/artifact-sources.json', captures);
return { state, exported, ...ledger, captures };
}