import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; import { root, save, read, client, captureSession, connectionDefinition, expectedOutputFields } from './common.mjs'; import { invokeProbe, inspectorClosed } from './inspector.mjs'; const frozen = read('frozen.json'); assert.equal(fs.existsSync(path.join(root, 'compatibility')), false, 'Never replay a compatibility batch.'); save('compatibility/claim.json', { id: randomUUID(), at: new Date().toISOString(), scopes: ['text', 'tools'], streaming: 'Separately assessed from each configured streaming request; no redundant real request.', noRetry: true }); const { api, boot, close } = await client(), results = []; try { assert.equal(boot.version, '1.25.13'); await inspectorClosed(); assert.equal((await api('/host/status')).identity.pid, frozen.host.identity.pid); for (const connection of frozen.connections) { const prefix = 'compatibility/' + connection.id; assert.deepEqual(connectionDefinition((await api('/model-center')).connections.find(value => value.id === connection.id)), connectionDefinition(connection), 'Frozen connection changed.'); const result = { connectionId: connection.id, name: connection.name, model: connection.model, protocol: connection.protocol, endpointKind: new URL(connection.baseUrl).hostname === 'api.deepseek.com' ? 'official-deepseek-endpoint' : 'compatible-proxy-endpoint', preflights: [], media: 'not_exercised_in_this_slice' }; console.log(JSON.stringify({ phase: 'compatibility', connection: connection.name, protocol: connection.protocol })); let started; try { started = await api('/model-center/network-diagnostics', { id: connection.id, revision: connection.revision }); save(prefix + '/network-start.json', started); const deadline = Date.now() + 30000; let network; while (true) { network = (await api('/model-center/network-diagnostics')).jobs.find(value => value.id === started.job.id); if (network?.finishedAt) break; if (Date.now() >= deadline) { network = (await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {})).job; result.networkObservationEnded = true; break; } await delay(250); } save(prefix + '/network.json', network); result.network = network.status; assert.equal(network.httpRequests, 0); assert.equal(network.modelRequests, 0); assert.equal(network.credentialAccess, false); } catch (error) { result.network = 'error'; save(prefix + '/network-error.json', { error: error.stack, response: error.response, noRetry: true }); if (started?.job?.id) save(prefix + '/network-owned-stop.json', await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {})); } const requestId = randomUUID(); try { const catalog = await api('/model-center/catalog', { id: connection.id, revision: connection.revision, requestId }, 90000); save(prefix + '/catalog.json', catalog); result.catalog = { complete: catalog.complete, models: catalog.entries.length, selectedListed: catalog.entries.some(value => value.id === connection.model), evidenceScope: 'Product catalog preview including its supplied source metadata; no invented raw HTTP artifact.' }; } catch (error) { result.catalog = { complete: false, error: error.message, response: error.response, noRetry: true }; save(prefix + '/catalog-error.json', result.catalog); } finally { save(prefix + '/catalog-cancel.json', await api('/model-center/catalog/cancel', { id: requestId })); } const projectRoot = path.join(root, prefix, 'project'); fs.mkdirSync(projectRoot, { recursive: true }); fs.writeFileSync(path.join(projectRoot, 'README.md'), 'M4 independent compatibility observations only. No engineering implementation.\n', { flag: 'wx' }); const project = await api('/projects', { root: projectRoot }); result.projectId = project.id; for (const scope of ['text', 'tools']) { const session = await api('/sessions', { projectId: project.id }); const input = { pid: frozen.host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, projectId: project.id, sessionId: session.id, connection, scope, nonce: randomUUID(), expectedOutput: expectedOutputFields(connection), observationMs: 180000 }; save(prefix + '/' + scope + '/input.json', input); let invoked; try { invoked = await invokeProbe(input); save(prefix + '/' + scope + '/invocation.json', invoked); } catch (error) { save(prefix + '/' + scope + '/invocation-error.json', { error: error.stack, noRetry: true }); } const captured = await captureSession(api, session.id, prefix + '/' + scope), attempt = captured.rows[0]; const request = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'request'); const response = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'response'); const body = request ? JSON.parse(fs.readFileSync(path.join(root, request.path), 'utf8')) : null; const raw = response ? fs.readFileSync(path.join(root, response.path), 'utf8') : ''; const outputFields = body ? Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])) : null; const streaming = { requested: body?.stream === true, sseObserved: /^(?:event:|data:)/m.test(raw), complete: attempt?.status === 'completed' && attempt?.modelIdentity?.complete === true }; const wirePassed = !!body && JSON.stringify(outputFields) === JSON.stringify(input.expectedOutput.fields) && (!body.stream || streaming.sseObserved); const passed = !!invoked?.result.contentPassed && invoked.result.actualToolsExecuted === 0 && captured.total === 1 && wirePassed && attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched' && attempt.modelIdentity.complete === true; const entry = { scope, passed, sessionId: session.id, taskId: invoked?.result.taskId, runId: invoked?.result.runId, identity: attempt?.modelIdentity ?? null, usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched', status: attempt?.status ?? 'not_dispatched', diagnostic: invoked?.result.diagnostic ?? null, actualToolsExecuted: invoked?.result.actualToolsExecuted ?? null, requests: captured.total, streaming, wirePassed, outputFields, expectedOutput: input.expectedOutput, captures: captured.captures }; save(prefix + '/' + scope + '/result.json', entry); result.preflights.push(entry); console.log(JSON.stringify({ connection: connection.name, scope, passed, identity: entry.identity?.status, usageStatus: entry.usageStatus, streaming, outputFields, diagnostic: entry.diagnostic?.kind })); assert.ok(captured.total <= 1); await inspectorClosed(); } result.engineeringAllowed = result.catalog.complete && result.catalog.selectedListed && result.preflights.every(value => value.passed && value.usageStatus === 'reported'); save(prefix + '/result.json', result); results.push(result); } const center = await api('/model-center'); assert.deepEqual(center.selected, frozen.center.selected); assert.deepEqual(center.connections.map(connectionDefinition), frozen.center.connections.map(connectionDefinition)); assert.deepEqual(center.pools, frozen.center.pools); save('compatibility/result.json', { completed: true, results, requests: results.reduce((sum, row) => sum + row.preflights.reduce((n, value) => n + value.requests, 0), 0), connectionDefinitionsPoolsSelection: 'exact', healthChangesRetained: true, keysReadOrResaved: false, historicalSamplesReplayed: false }); } finally { await inspectorClosed(); await close(); }