Files
Agent-Workbench/docs/messages-m-evidence/drivers/probe-fixture-first.mjs
T

62 lines
5.7 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { launchWorkbench } from '../tests/helpers.mjs';
import { invokeProbe, inspectorClosed } from './messages-m-inspector.mjs';
const root = path.resolve('.qa/messages-m/probe-fixture-' + randomUUID()); fs.mkdirSync(root);
const results = [];
for (const mode of ['match', 'mismatch', '429']) {
const dir = path.join(root, mode); fs.mkdirSync(dir); const disposers = [], requests = [];
const save = (name, value) => fs.writeFileSync(path.join(dir, name), typeof value === 'string' ? value : JSON.stringify(value, null, 2), { flag: 'wx' });
const upstream = http.createServer(async (req, res) => {
const chunks = []; for await (const bytes of req) chunks.push(bytes);
const body = JSON.parse(Buffer.concat(chunks));
requests.push({ route: req.url, body, authCorrect: req.headers.authorization === 'Bearer owned-m-probe', version: req.headers['anthropic-version'] });
if (mode === '429') { res.writeHead(429, { 'Content-Type': 'application/json', 'request-id': 'owned_m_429', 'Retry-After': '1' }); res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'Owned preflight rate limit' } })); return; }
const model = mode === 'match' ? 'claude-owned-m' : 'gpt-owned-other';
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'request-id': 'owned_m_' + mode });
const send = (type, fields) => res.write('event: ' + type + '\ndata: ' + JSON.stringify({ type, ...fields }) + '\n\n');
send('message_start', { message: { id: 'msg_owned_m', type: 'message', role: 'assistant', model, content: [], stop_reason: null, usage: { input_tokens: 40, output_tokens: 0 } } });
send('content_block_start', { index: 0, content_block: { type: 'text', text: '' } });
send('content_block_delta', { index: 0, delta: { type: 'text_delta', text: 'OK owned-m' } });
send('content_block_stop', { index: 0 });
send('message_delta', { delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } });
send('message_stop', {}); res.end();
});
await new Promise(resolve => upstream.listen(0, '127.0.0.1', resolve));
let host;
try {
host = await launchWorkbench({ after: fn => disposers.push(fn) }, path.join(dir, 'data'));
const projectRoot = path.join(dir, 'project'); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), 'Owned no-tool probe.');
const project = await host.api('/projects', { root: projectRoot }), session = await host.api('/sessions', { projectId: project.id });
const baseUrl = 'http://127.0.0.1:' + upstream.address().port;
const connection = await host.api('/model-center/connections', { name: 'owned M ' + mode, model: 'claude-owned-m', protocol: 'anthropic-messages', messagesAuth: 'bearer',
baseUrl, apiKey: 'owned-m-probe', streaming: true, capabilityDeclarations: { text: 'supported', tools: 'supported', streaming: 'supported', vision: 'unsupported' },
nativeParameters: {}, contextLimit: 16000, maxOutputTokens: 256, concurrency: 1, enabled: true, requiresKey: true, modelIdentityPolicy: 'require-match', modelIdentityAliases: [], modelIdentityAliasSource: '' });
const boot = await (await fetch(host.url + '/api/bootstrap')).json();
const input = { pid: host.child.pid, port: Number(new URL(host.url).port), sessionId: session.id, projectId: project.id, runtimeRoot: boot.runtimeRoot,
connectionId: connection.id, credentialRevision: connection.credentialRevision, connectionRevision: connection.revision,
baseUrl, model: connection.model, title: 'Owned M probe ' + mode, prompt: 'Reply OK owned-m. Do not use tools.' };
save('input.json', input);
const called = await invokeProbe(input); save('expression.js', called.expression); save('result.json', called.result);
assert.equal(requests.length, 1); assert.equal(requests[0].route, '/v1/messages'); assert.equal(requests[0].authCorrect, true);
assert.equal(requests[0].version, '2023-06-01'); assert.equal(requests[0].body.tools?.length ?? 0, 0); assert.equal(requests[0].body.max_tokens, 256);
assert.equal(called.result.actualTools, 0); assert.equal(called.result.attempts, 1); assert.equal(called.result.receipts, 1);
assert.equal(called.result.status, mode === 'match' ? 'completed' : mode === 'mismatch' ? 'paused' : 'failed');
if (mode === 'mismatch') assert.equal(called.result.stopReason, 'model_identity');
const center = await host.api('/model-center'), attempt = center.attempts.find(a => a.sessionId === session.id);
assert.equal(attempt.requestUrl, baseUrl + '/v1/messages'); assert.equal(attempt.poolId, null);
const exported = await host.api('/sessions/' + session.id + '/export'); save('export.json', exported); save('attempt.json', attempt); save('wire.json', requests);
const taskUsage = exported.execution.task.usage;
assert.equal(taskUsage.requests, 1); assert.equal(taskUsage.totalTokens, mode === '429' ? 0 : 45);
assert.equal(taskUsage.unknownRequests, mode === '429' ? 1 : 0);
assert.equal((await host.api('/host/status')).identity.pid, host.child.pid); await inspectorClosed();
const result = { mode, passed: true, requests: requests.length, actualTools: 0, status: called.result.status, inspectorClosed: true };
results.push(result); console.log(JSON.stringify(result));
} finally { for (const dispose of disposers.reverse()) await dispose(); upstream.closeAllConnections(); await new Promise(resolve => upstream.close(resolve)); }
}
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify({ passed: true, realModelRequests: 0, results }, null, 2), { flag: 'wx' });
console.log(JSON.stringify({ passed: true, root }));