Files
Agent-Workbench/scripts/live-ad/verify-driver.mjs
T

44 lines
4.3 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { fixture, compatibleFixture, launchWorkbench, completion, call, assertToolPairs } from '../../tests/helpers.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
import { connectionDefinition } from './common.mjs';
const output = path.resolve('.qa/ad-driver-' + randomUUID()); fs.mkdirSync(output); const disposers = [], t = { after: fn => disposers.push(fn) };
const f = fixture(t, { modernDefaults: true }); f.store.close(); let host, result; const results = [];
try {
await inspectorClosed();
const model = await compatibleFixture(t, (_url, body) => {
const prompt = JSON.stringify(body.messages ?? body.input), nonce = prompt.match(/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/)?.[0];
assert.ok(nonce); const tool = body.tools?.length > 0;
assert.ok(['max_tokens','max_output_tokens','max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
if (body.model === 'owned-chat') return { ...completion(tool ? [call('call_' + nonce, 'workbench_echo_nonce', { nonce })] : [], tool ? '' : 'OK ' + nonce), model: body.model };
if (body.model === 'owned-responses') return { body: { id: 'resp_owned', model: body.model, status: 'completed', output: tool
? [{ type: 'function_call', id: 'fc_owned', call_id: 'call_' + nonce, name: 'workbench_echo_nonce', arguments: JSON.stringify({ nonce }) }]
: [{ type: 'message', id: 'msg_owned', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'OK ' + nonce }] }], usage: { input_tokens: 10, output_tokens: 4, total_tokens: 14 } } };
return { id: 'msg_owned', type: 'message', role: 'assistant', model: body.model, content: tool
? [{ type: 'tool_use', id: 'call_' + nonce, name: 'workbench_echo_nonce', input: { nonce } }] : [{ type: 'text', text: 'OK ' + nonce }],
stop_reason: tool ? 'tool_use' : 'end_turn', usage: { input_tokens: 10, output_tokens: 4 } };
});
host = await launchWorkbench(t, f.data); const originalCenter = await host.api('/model-center'), health = await host.api('/host/status');
for (const [protocol, name] of [['chat-completions', 'owned-chat'], ['responses', 'owned-responses'], ['anthropic-messages', 'owned-messages']]) {
const connection = await host.api('/model-center/connections', { name, model: name, baseUrl: model.baseUrl, protocol, apiKey: 'owned-ad-fixture-secret', streaming: false });
for (const scope of ['text', 'tools']) {
const session = await host.api('/sessions', { projectId: f.project.id }), input = { pid: health.identity.pid, port: new URL(host.url).port * 1,
runtimeRoot: host.bootstrap.runtimeRoot, projectId: f.project.id, sessionId: session.id, connection, nonce: randomUUID(), scope };
const invoked = await invokeProbe(input); fs.writeFileSync(path.join(output, protocol + '-' + scope + '.json'), JSON.stringify(invoked, null, 2));
assert.equal(invoked.result.contentPassed, true, JSON.stringify(invoked.result)); assert.equal(invoked.result.actualToolsExecuted, 0); assert.equal(invoked.result.attempts, 1);
const state = await host.api('/sessions/' + session.id), center = await host.api('/model-center'), attempt = center.attempts.find(value => value.sessionId === session.id);
assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true); assert.equal(state.status, 'completed');
const exported = await host.api('/sessions/' + session.id + '/export'); assertToolPairs(exported.session.providerMessages);
assert.deepEqual(center.selected, originalCenter.selected); assert.deepEqual(connectionDefinition(center.connections.find(value => value.id === connection.id)), connectionDefinition(connection));
results.push({ protocol, scope, passed: true });
}
}
assert.equal(model.requests.length, 6); result = { success: true, results, fixtureRequests: 6, realRequests: 0, keysReadOrResaved: false };
} catch (error) { result = { success: false, error: error.stack, results }; process.exitCode = 1; }
finally { await inspectorClosed(); for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2)); }
console.log(JSON.stringify({ output, ...result }));