Files
Agent-Workbench/scripts/live-aa/common.mjs
T
2026-11-05 03:15:01 +08:00

72 lines
4.9 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
import { Agent, fetch } from 'undici';
export const origin = 'http://127.0.0.1:4318';
export const batch = path.resolve('.qa/live-validation-aa');
export const routes = [
{ key: 'deepseek', id: '1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd', off: { thinking: { type: 'disabled' } }, on: { thinking: { type: 'enabled' }, reasoning_effort: 'low' } },
{ key: 'claude', id: '9ae348a5-d0a6-42ed-8664-0da5f9effe34', off: { thinking: { type: 'disabled' } }, on: { thinking: { type: 'enabled', budget_tokens: 1024 }, output_config: { effort: 'low' } } },
{ key: 'gpt', id: '93d0cf6d-4f32-4918-9da8-6f125aa8a6e5', off: { reasoning_effort: 'none' }, on: { reasoning_effort: 'low' } },
];
export const target = process.argv[2];
export const experiment = routes.find(row => row.key === target);
export const root = experiment ? path.join(batch, target) : batch;
export const connectionId = experiment?.id;
if (experiment && fs.existsSync(path.join(root, 'current.json'))) {
const frozen = JSON.parse(fs.readFileSync(path.join(root, 'current.json'))).experiment;
experiment.model = frozen.model; experiment.protocol = frozen.protocol;
const connection = JSON.parse(fs.readFileSync(path.join(root, 'current.json'))).center.connections.find(c => c.id === connectionId);
experiment.credentialRevision = connection.credentialRevision;
}
export const sha = value => createHash('sha256').update(value).digest('hex');
export const saveAt = (directory, relative, value) => {
const file = path.join(directory, relative); fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
};
export const save = (relative, value) => saveAt(root, relative, value);
export const readRoot = () => JSON.parse(fs.readFileSync(path.join(root, 'current.json')));
export const dataRoot = () => JSON.parse(fs.readFileSync(path.join(batch, 'current.json'))).host.identity.dataRoot;
export function meta(name) {
assert.ok(['config', 'modelRegistry', 'modelAttempts', 'modelNetworkDiagnostics'].includes(name));
const db = new DatabaseSync(path.join(dataRoot(), 'workbench.sqlite'), { readOnly: true });
try { const row = db.prepare('SELECT value FROM metadata WHERE key=?').get(name); return row ? JSON.parse(row.value) : null; }
finally { db.close(); }
}
export async function client() {
const dispatcher = new Agent({ connect: { autoSelectFamily: true, timeout: 5000 } }); let token;
const api = async (route, input, timeout = 30000) => {
const response = await fetch(origin + '/api' + route, { dispatcher,
method: input === undefined ? 'GET' : 'POST', redirect: 'error', signal: AbortSignal.timeout(timeout),
headers: { ...(token ? { 'X-Workbench-Token': token } : {}), 'Content-Type': 'application/json', Connection: 'close' },
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const value = await response.json();
if (!response.ok) throw Object.assign(new Error(route + ': ' + (value.error ?? response.status)), { status: response.status, response: value });
return value;
};
const boot = await api('/bootstrap'); token = boot.token;
return { api, boot, close: () => dispatcher.close() };
}
export async function captureAttempts(api, attempts, relative) {
const captures = [];
for (const attempt of attempts) for (const kind of ['request', 'response']) {
const id = attempt[kind + 'ArtifactId']; if (!id) continue;
let content = '', offset = 0, page;
do { page = await api('/artifacts/' + id + '?session=' + attempt.sessionId + '&offset=' + offset + '&limit=16000');
assert.ok(page.content.length || !page.truncated); content += page.content; offset += page.content.length;
} while (page.truncated);
assert.equal(sha(content), page.sha256); assert.equal(page.sha256, attempt[kind + 'Sha256']);
const name = `${relative}/${attempt.id}-${kind}.${kind === 'request' ? 'json' : attempt.streaming ? 'sse' : 'txt'}`;
save(name, content); captures.push({ attemptId: attempt.id, kind, artifactId: id, path: name, sha256: sha(content), bytes: Buffer.byteLength(content) });
}
return captures;
}
export function batchUsage() {
const before = new Set(JSON.parse(fs.readFileSync(path.join(batch, 'original-attempts.json'))).map(row => row.id));
const attempts = (meta('modelAttempts') ?? []).filter(row => !before.has(row.id));
return { requests: attempts.length, tokens: attempts.reduce((sum, row) => sum + (row.usage?.total_tokens ?? 0), 0), unknown: attempts.filter(row => row.usageStatus !== 'reported').length };
}
export function assertBatchBudget() { const usage = batchUsage(); assert.ok(usage.requests < 100 && usage.tokens < 700000, 'AA batch budget exhausted'); return usage; }