Files
Agent-Workbench/scripts/live-ad/compatibility.mjs
T

62 lines
6.0 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { root, save, read, client, captureSession, connectionDefinition } from './common.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
const original = read('current.json'); assert.equal(fs.existsSync(path.join(root, 'compatibility')), false, 'Never replay this batch.');
save('compatibility/started.json', { at: new Date().toISOString(), noRetry: true, noConfigurationSave: true });
const { api, boot, close } = await client(), results = [];
try {
assert.equal(boot.version, '1.25.8'); await inspectorClosed();
for (const connection of original.connections) {
const prefix = 'compatibility/' + connection.id, current = (await api('/model-center')).connections.find(value => value.id === connection.id);
assert.deepEqual(connectionDefinition(current), connectionDefinition(connection), 'Frozen connection changed before its experiment.');
const result = { connectionId: connection.id, name: connection.name, model: connection.model, protocol: connection.protocol, preflights: [] };
console.log(JSON.stringify({ phase: 'compatibility', connection: connection.name, protocol: connection.protocol }));
let started;
try {
started = await api('/model-center/network-diagnostics', { id: connection.id, revision: connection.revision }); save(prefix + '/network-start.json', started);
const deadline = Date.now() + 30000; let network;
do { const status = await api('/model-center/network-diagnostics'); network = status.jobs.find(value => value.id === started.job.id); if (network && network.finishedAt) break;
if (Date.now() > deadline) { const stopped = await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {}); network = stopped.job; result.networkObservationEnded = true; break; } await delay(250); } while (true);
save(prefix + '/network.json', network); result.network = network.status;
assert.equal(network.httpRequests, 0); assert.equal(network.modelRequests, 0); assert.equal(network.credentialAccess, false);
} catch (error) {
result.network = 'error'; save(prefix + '/network-error.json', { error: error.stack, status: error.status, response: error.response, noRetry: true });
if (started?.job?.id) save(prefix + '/network-owned-stop.json', await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {}));
}
const requestId = randomUUID();
try {
const catalog = await api('/model-center/catalog', { id: connection.id, revision: connection.revision, requestId }, 90000);
save(prefix + '/catalog.json', catalog); result.catalog = { complete: catalog.complete, models: catalog.entries.length, selectedListed: catalog.entries.some(value => value.id === connection.model), rawHttpArtifactAvailable: false };
} catch (error) { result.catalog = { complete: false, error: error.message, status: error.status, response: error.response, noRetry: true }; save(prefix + '/catalog-error.json', result.catalog); }
finally { save(prefix + '/catalog-cancel.json', await api('/model-center/catalog/cancel', { id: requestId })); }
const projectRoot = path.join(root, prefix, 'project'); fs.mkdirSync(projectRoot, { recursive: true }); fs.writeFileSync(path.join(projectRoot, 'README.md'), 'AD isolated connection preflight only. No engineering implementation.\n', { flag: 'wx' });
const project = await api('/projects', { root: projectRoot }); result.projectId = project.id;
for (const scope of ['text', 'tools']) {
const session = await api('/sessions', { projectId: project.id }), nonce = randomUUID();
const input = { pid: original.host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, projectId: project.id, sessionId: session.id, connection, scope, nonce };
save(prefix + '/' + scope + '/input.json', input);
let invoked;
try { invoked = await invokeProbe(input); save(prefix + '/' + scope + '/invocation.json', invoked); }
catch (error) { save(prefix + '/' + scope + '/invocation-error.json', { error: error.stack, noRetry: true }); }
const captured = await captureSession(api, session.id, prefix + '/' + scope), attempt = captured.rows[0];
const passed = !!invoked?.result.contentPassed && invoked.result.actualToolsExecuted === 0 && captured.rows.length === 1
&& attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched' && attempt.modelIdentity.complete === true;
const entry = { scope, passed, sessionId: session.id, identity: attempt?.modelIdentity ?? null, usage: attempt?.usage ?? null,
usageStatus: attempt?.usageStatus ?? 'not_dispatched', status: attempt?.status ?? 'not_dispatched', diagnostic: invoked?.result.diagnostic ?? null, requests: captured.rows.length, captures: captured.captures };
save(prefix + '/' + scope + '/result.json', entry); result.preflights.push(entry); console.log(JSON.stringify({ connection: connection.name, scope, passed, identity: entry.identity?.status, usage: entry.usage }));
assert.ok(captured.rows.length <= 1); await inspectorClosed();
}
result.engineeringAllowed = result.catalog.complete && result.catalog.selectedListed && result.preflights.every(value => value.passed);
save(prefix + '/result.json', result); results.push(result);
}
const center = await api('/model-center');
assert.deepEqual(center.selected, original.center.selected); assert.deepEqual(center.connections.map(connectionDefinition), original.center.connections.map(connectionDefinition)); assert.deepEqual(center.pools, original.center.pools);
save('compatibility/result.json', { completed: true, results, requests: results.reduce((sum, row) => sum + row.preflights.reduce((n, value) => n + value.requests, 0), 0),
originalConnectionDefinitionsPoolsSelection: 'exact', healthObservationChangesRetained: true, keysReadOrResaved: false, historicalSamplesReplayed: false });
} finally { await inspectorClosed(); await close(); }