import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; import { root, save, read, client, captureSession, connectionDefinition } from './common.mjs'; import { invokeProbe, inspectorClosed } from './inspector.mjs'; const original = read('current.json'); assert.equal(fs.existsSync(path.join(root, 'compatibility')), false, 'Never replay this batch.'); save('compatibility/started.json', { at: new Date().toISOString(), noRetry: true, noConfigurationSave: true }); const { api, boot, close } = await client(), results = []; try { assert.equal(boot.version, '1.25.8'); await inspectorClosed(); for (const connection of original.connections) { const prefix = 'compatibility/' + connection.id, current = (await api('/model-center')).connections.find(value => value.id === connection.id); assert.deepEqual(connectionDefinition(current), connectionDefinition(connection), 'Frozen connection changed before its experiment.'); const result = { connectionId: connection.id, name: connection.name, model: connection.model, protocol: connection.protocol, preflights: [] }; console.log(JSON.stringify({ phase: 'compatibility', connection: connection.name, protocol: connection.protocol })); let started; try { started = await api('/model-center/network-diagnostics', { id: connection.id, revision: connection.revision }); save(prefix + '/network-start.json', started); const deadline = Date.now() + 30000; let network; do { const status = await api('/model-center/network-diagnostics'); network = status.jobs.find(value => value.id === started.job.id); if (network && network.finishedAt) break; if (Date.now() > deadline) { const stopped = await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {}); network = stopped.job; result.networkObservationEnded = true; break; } await delay(250); } while (true); save(prefix + '/network.json', network); result.network = network.status; assert.equal(network.httpRequests, 0); assert.equal(network.modelRequests, 0); assert.equal(network.credentialAccess, false); } catch (error) { result.network = 'error'; save(prefix + '/network-error.json', { error: error.stack, status: error.status, response: error.response, noRetry: true }); if (started?.job?.id) save(prefix + '/network-owned-stop.json', await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {})); } const requestId = randomUUID(); try { const catalog = await api('/model-center/catalog', { id: connection.id, revision: connection.revision, requestId }, 90000); save(prefix + '/catalog.json', catalog); result.catalog = { complete: catalog.complete, models: catalog.entries.length, selectedListed: catalog.entries.some(value => value.id === connection.model), rawHttpArtifactAvailable: false }; } catch (error) { result.catalog = { complete: false, error: error.message, status: error.status, response: error.response, noRetry: true }; save(prefix + '/catalog-error.json', result.catalog); } finally { save(prefix + '/catalog-cancel.json', await api('/model-center/catalog/cancel', { id: requestId })); } const projectRoot = path.join(root, prefix, 'project'); fs.mkdirSync(projectRoot, { recursive: true }); fs.writeFileSync(path.join(projectRoot, 'README.md'), 'AD isolated connection preflight only. No engineering implementation.\n', { flag: 'wx' }); const project = await api('/projects', { root: projectRoot }); result.projectId = project.id; for (const scope of ['text', 'tools']) { const session = await api('/sessions', { projectId: project.id }), nonce = randomUUID(); const input = { pid: original.host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, projectId: project.id, sessionId: session.id, connection, scope, nonce }; save(prefix + '/' + scope + '/input.json', input); let invoked; try { invoked = await invokeProbe(input); save(prefix + '/' + scope + '/invocation.json', invoked); } catch (error) { save(prefix + '/' + scope + '/invocation-error.json', { error: error.stack, noRetry: true }); } const captured = await captureSession(api, session.id, prefix + '/' + scope), attempt = captured.rows[0]; const passed = !!invoked?.result.contentPassed && invoked.result.actualToolsExecuted === 0 && captured.rows.length === 1 && attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched' && attempt.modelIdentity.complete === true; const entry = { scope, passed, sessionId: session.id, identity: attempt?.modelIdentity ?? null, usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched', status: attempt?.status ?? 'not_dispatched', diagnostic: invoked?.result.diagnostic ?? null, requests: captured.rows.length, captures: captured.captures }; save(prefix + '/' + scope + '/result.json', entry); result.preflights.push(entry); console.log(JSON.stringify({ connection: connection.name, scope, passed, identity: entry.identity?.status, usage: entry.usage })); assert.ok(captured.rows.length <= 1); await inspectorClosed(); } result.engineeringAllowed = result.catalog.complete && result.catalog.selectedListed && result.preflights.every(value => value.passed); save(prefix + '/result.json', result); results.push(result); } const center = await api('/model-center'); assert.deepEqual(center.selected, original.center.selected); assert.deepEqual(center.connections.map(connectionDefinition), original.center.connections.map(connectionDefinition)); assert.deepEqual(center.pools, original.center.pools); save('compatibility/result.json', { completed: true, results, requests: results.reduce((sum, row) => sum + row.preflights.reduce((n, value) => n + value.requests, 0), 0), originalConnectionDefinitionsPoolsSelection: 'exact', healthObservationChangesRetained: true, keysReadOrResaved: false, historicalSamplesReplayed: false }); } finally { await inspectorClosed(); await close(); }