Files
Agent-Workbench/scripts/m4-live/compatibility.mjs
T
2026-11-11 22:58:21 +08:00

80 lines
7.7 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { root, save, read, client, captureSession, connectionDefinition, expectedOutputFields } from './common.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
const frozen = read('frozen.json'); assert.equal(fs.existsSync(path.join(root, 'compatibility')), false, 'Never replay a compatibility batch.');
save('compatibility/claim.json', { id: randomUUID(), at: new Date().toISOString(), scopes: ['text', 'tools'], streaming: 'Separately assessed from each configured streaming request; no redundant real request.', noRetry: true });
const { api, boot, close } = await client(), results = [];
try {
assert.equal(boot.version, '1.25.13'); await inspectorClosed();
assert.equal((await api('/host/status')).identity.pid, frozen.host.identity.pid);
for (const connection of frozen.connections) {
const prefix = 'compatibility/' + connection.id;
assert.deepEqual(connectionDefinition((await api('/model-center')).connections.find(value => value.id === connection.id)), connectionDefinition(connection), 'Frozen connection changed.');
const result = { connectionId: connection.id, name: connection.name, model: connection.model, protocol: connection.protocol,
endpointKind: new URL(connection.baseUrl).hostname === 'api.deepseek.com' ? 'official-deepseek-endpoint' : 'compatible-proxy-endpoint', preflights: [], media: 'not_exercised_in_this_slice' };
console.log(JSON.stringify({ phase: 'compatibility', connection: connection.name, protocol: connection.protocol }));
let started;
try {
started = await api('/model-center/network-diagnostics', { id: connection.id, revision: connection.revision }); save(prefix + '/network-start.json', started);
const deadline = Date.now() + 30000; let network;
while (true) {
network = (await api('/model-center/network-diagnostics')).jobs.find(value => value.id === started.job.id);
if (network?.finishedAt) break;
if (Date.now() >= deadline) { network = (await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {})).job; result.networkObservationEnded = true; break; }
await delay(250);
}
save(prefix + '/network.json', network); result.network = network.status;
assert.equal(network.httpRequests, 0); assert.equal(network.modelRequests, 0); assert.equal(network.credentialAccess, false);
} catch (error) {
result.network = 'error'; save(prefix + '/network-error.json', { error: error.stack, response: error.response, noRetry: true });
if (started?.job?.id) save(prefix + '/network-owned-stop.json', await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {}));
}
const requestId = randomUUID();
try {
const catalog = await api('/model-center/catalog', { id: connection.id, revision: connection.revision, requestId }, 90000);
save(prefix + '/catalog.json', catalog); result.catalog = { complete: catalog.complete, models: catalog.entries.length, selectedListed: catalog.entries.some(value => value.id === connection.model),
evidenceScope: 'Product catalog preview including its supplied source metadata; no invented raw HTTP artifact.' };
} catch (error) { result.catalog = { complete: false, error: error.message, response: error.response, noRetry: true }; save(prefix + '/catalog-error.json', result.catalog); }
finally { save(prefix + '/catalog-cancel.json', await api('/model-center/catalog/cancel', { id: requestId })); }
const projectRoot = path.join(root, prefix, 'project'); fs.mkdirSync(projectRoot, { recursive: true });
fs.writeFileSync(path.join(projectRoot, 'README.md'), 'M4 independent compatibility observations only. No engineering implementation.\n', { flag: 'wx' });
const project = await api('/projects', { root: projectRoot }); result.projectId = project.id;
for (const scope of ['text', 'tools']) {
const session = await api('/sessions', { projectId: project.id });
const input = { pid: frozen.host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, projectId: project.id, sessionId: session.id, connection,
scope, nonce: randomUUID(), expectedOutput: expectedOutputFields(connection), observationMs: 180000 };
save(prefix + '/' + scope + '/input.json', input); let invoked;
try { invoked = await invokeProbe(input); save(prefix + '/' + scope + '/invocation.json', invoked); }
catch (error) { save(prefix + '/' + scope + '/invocation-error.json', { error: error.stack, noRetry: true }); }
const captured = await captureSession(api, session.id, prefix + '/' + scope), attempt = captured.rows[0];
const request = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'request');
const response = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'response');
const body = request ? JSON.parse(fs.readFileSync(path.join(root, request.path), 'utf8')) : null;
const raw = response ? fs.readFileSync(path.join(root, response.path), 'utf8') : '';
const outputFields = body ? Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])) : null;
const streaming = { requested: body?.stream === true, sseObserved: /^(?:event:|data:)/m.test(raw), complete: attempt?.status === 'completed' && attempt?.modelIdentity?.complete === true };
const wirePassed = !!body && JSON.stringify(outputFields) === JSON.stringify(input.expectedOutput.fields) && (!body.stream || streaming.sseObserved);
const passed = !!invoked?.result.contentPassed && invoked.result.actualToolsExecuted === 0 && captured.total === 1 && wirePassed
&& attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched' && attempt.modelIdentity.complete === true;
const entry = { scope, passed, sessionId: session.id, taskId: invoked?.result.taskId, runId: invoked?.result.runId, identity: attempt?.modelIdentity ?? null,
usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched', status: attempt?.status ?? 'not_dispatched',
diagnostic: invoked?.result.diagnostic ?? null, actualToolsExecuted: invoked?.result.actualToolsExecuted ?? null, requests: captured.total,
streaming, wirePassed, outputFields, expectedOutput: input.expectedOutput, captures: captured.captures };
save(prefix + '/' + scope + '/result.json', entry); result.preflights.push(entry);
console.log(JSON.stringify({ connection: connection.name, scope, passed, identity: entry.identity?.status, usageStatus: entry.usageStatus, streaming, outputFields, diagnostic: entry.diagnostic?.kind }));
assert.ok(captured.total <= 1); await inspectorClosed();
}
result.engineeringAllowed = result.catalog.complete && result.catalog.selectedListed && result.preflights.every(value => value.passed && value.usageStatus === 'reported');
save(prefix + '/result.json', result); results.push(result);
}
const center = await api('/model-center');
assert.deepEqual(center.selected, frozen.center.selected); assert.deepEqual(center.connections.map(connectionDefinition), frozen.center.connections.map(connectionDefinition)); assert.deepEqual(center.pools, frozen.center.pools);
save('compatibility/result.json', { completed: true, results, requests: results.reduce((sum, row) => sum + row.preflights.reduce((n, value) => n + value.requests, 0), 0),
connectionDefinitionsPoolsSelection: 'exact', healthChangesRetained: true, keysReadOrResaved: false, historicalSamplesReplayed: false });
} finally { await inspectorClosed(); await close(); }