62 lines
5.7 KiB
JavaScript
62 lines
5.7 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import http from 'node:http';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { launchWorkbench } from '../tests/helpers.mjs';
|
|
import { invokeProbe, inspectorClosed } from './messages-m-inspector.mjs';
|
|
const root = path.resolve('.qa/messages-m/probe-fixture-' + randomUUID()); fs.mkdirSync(root);
|
|
const results = [];
|
|
for (const mode of ['match', 'mismatch', '429']) {
|
|
const dir = path.join(root, mode); fs.mkdirSync(dir); const disposers = [], requests = [];
|
|
const save = (name, value) => fs.writeFileSync(path.join(dir, name), typeof value === 'string' ? value : JSON.stringify(value, null, 2), { flag: 'wx' });
|
|
const upstream = http.createServer(async (req, res) => {
|
|
const chunks = []; for await (const bytes of req) chunks.push(bytes);
|
|
const body = JSON.parse(Buffer.concat(chunks));
|
|
requests.push({ route: req.url, body, authCorrect: req.headers.authorization === 'Bearer owned-m-probe', version: req.headers['anthropic-version'] });
|
|
if (mode === '429') { res.writeHead(429, { 'Content-Type': 'application/json', 'request-id': 'owned_m_429', 'Retry-After': '1' }); res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'Owned preflight rate limit' } })); return; }
|
|
const model = mode === 'match' ? 'claude-owned-m' : 'gpt-owned-other';
|
|
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'request-id': 'owned_m_' + mode });
|
|
const send = (type, fields) => res.write('event: ' + type + '\ndata: ' + JSON.stringify({ type, ...fields }) + '\n\n');
|
|
send('message_start', { message: { id: 'msg_owned_m', type: 'message', role: 'assistant', model, content: [], stop_reason: null, usage: { input_tokens: 40, output_tokens: 0 } } });
|
|
send('content_block_start', { index: 0, content_block: { type: 'text', text: '' } });
|
|
send('content_block_delta', { index: 0, delta: { type: 'text_delta', text: 'OK owned-m' } });
|
|
send('content_block_stop', { index: 0 });
|
|
send('message_delta', { delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } });
|
|
send('message_stop', {}); res.end();
|
|
});
|
|
await new Promise(resolve => upstream.listen(0, '127.0.0.1', resolve));
|
|
let host;
|
|
try {
|
|
host = await launchWorkbench({ after: fn => disposers.push(fn) }, path.join(dir, 'data'));
|
|
const projectRoot = path.join(dir, 'project'); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), 'Owned no-tool probe.');
|
|
const project = await host.api('/projects', { root: projectRoot }), session = await host.api('/sessions', { projectId: project.id });
|
|
const baseUrl = 'http://127.0.0.1:' + upstream.address().port;
|
|
const connection = await host.api('/model-center/connections', { name: 'owned M ' + mode, model: 'claude-owned-m', protocol: 'anthropic-messages', messagesAuth: 'bearer',
|
|
baseUrl, apiKey: 'owned-m-probe', streaming: true, capabilityDeclarations: { text: 'supported', tools: 'supported', streaming: 'supported', vision: 'unsupported' },
|
|
nativeParameters: {}, contextLimit: 16000, maxOutputTokens: 256, concurrency: 1, enabled: true, requiresKey: true, modelIdentityPolicy: 'require-match', modelIdentityAliases: [], modelIdentityAliasSource: '' });
|
|
const boot = await (await fetch(host.url + '/api/bootstrap')).json();
|
|
const input = { pid: host.child.pid, port: Number(new URL(host.url).port), sessionId: session.id, projectId: project.id, runtimeRoot: boot.runtimeRoot,
|
|
connectionId: connection.id, credentialRevision: connection.credentialRevision, connectionRevision: connection.revision,
|
|
baseUrl, model: connection.model, title: 'Owned M probe ' + mode, prompt: 'Reply OK owned-m. Do not use tools.' };
|
|
save('input.json', input);
|
|
const called = await invokeProbe(input); save('expression.js', called.expression); save('result.json', called.result);
|
|
assert.equal(requests.length, 1); assert.equal(requests[0].route, '/v1/messages'); assert.equal(requests[0].authCorrect, true);
|
|
assert.equal(requests[0].version, '2023-06-01'); assert.equal(requests[0].body.tools?.length ?? 0, 0); assert.equal(requests[0].body.max_tokens, 256);
|
|
assert.equal(called.result.actualTools, 0); assert.equal(called.result.attempts, 1); assert.equal(called.result.receipts, 1);
|
|
assert.equal(called.result.status, mode === 'match' ? 'completed' : mode === 'mismatch' ? 'paused' : 'failed');
|
|
if (mode === 'mismatch') assert.equal(called.result.stopReason, 'model_identity');
|
|
const center = await host.api('/model-center'), attempt = center.attempts.find(a => a.sessionId === session.id);
|
|
assert.equal(attempt.requestUrl, baseUrl + '/v1/messages'); assert.equal(attempt.poolId, null);
|
|
const exported = await host.api('/sessions/' + session.id + '/export'); save('export.json', exported); save('attempt.json', attempt); save('wire.json', requests);
|
|
const taskUsage = exported.execution.task.usage;
|
|
assert.equal(taskUsage.requests, 1); assert.equal(taskUsage.totalTokens, mode === '429' ? 0 : 45);
|
|
assert.equal(taskUsage.unknownRequests, mode === '429' ? 1 : 0);
|
|
assert.equal((await host.api('/host/status')).identity.pid, host.child.pid); await inspectorClosed();
|
|
const result = { mode, passed: true, requests: requests.length, actualTools: 0, status: called.result.status, inspectorClosed: true };
|
|
results.push(result); console.log(JSON.stringify(result));
|
|
} finally { for (const dispose of disposers.reverse()) await dispose(); upstream.closeAllConnections(); await new Promise(resolve => upstream.close(resolve)); }
|
|
}
|
|
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify({ passed: true, realModelRequests: 0, results }, null, 2), { flag: 'wx' });
|
|
console.log(JSON.stringify({ passed: true, root }));
|