Files
Agent-Workbench/scripts/live-af/verify-driver.mjs
T
2026-11-06 13:40:29 +08:00

63 lines
5.2 KiB
JavaScript

// Synthetic fixed model output exercises the real driver and native processes.
// These implementation bytes are never sent to any real provider.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
import { fixture, compatibleFixture, launchWorkbench, completion, call } from '../../tests/helpers.mjs';
import { configurePurgeFixture, save, publicMetadata } from '../live-ad/common.mjs';
import { publicSnapshot } from '../retained-host-state.mjs';
import { resolveControlledInstallation } from '../../src/core/engines/controlled-installation.mjs';
import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs';
import { allowed } from './contract.mjs';
const output = path.resolve('.qa/af-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
fs.mkdirSync(output); const f = fixture(t, { modernDefaults: true });
f.store.saveConfig({ approvalMode: 'full', mode: 'compatible' });
// Model the existing production metadata shape before freezing this fresh fixture.
for (const key of ['modelAttempts', 'modelNetworkDiagnostics']) f.store.storage.database.prepare('INSERT OR IGNORE INTO metadata(key,value) VALUES (?,?)').run(key, '[]');
f.store.close(); let result;
try {
const solution = path.resolve('.qa/live-validation-ad/node/expense-project');
assert.equal(JSON.parse(fs.readFileSync('.qa/live-validation-ad/node/result.json')).success, true);
const files = Object.fromEntries(allowed.map(name => [name, fs.readFileSync(path.join(solution, name), 'utf8')]));
let requests = 0;
const model = await compatibleFixture(t, (_url, body) => {
requests++; const step = (requests - 1) % 4 + 1;
assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
let calls;
if (step === 1) calls = Object.entries(files).map(([name, content], index) => call('owned-file-' + index, 'propose_file', { path: name, content, reason: 'Local fixed HTTP fixture only.' }));
else if (step === 2) calls = [call('owned-plan', 'plan_checks', { names: ['test', 'build'] })];
else if (step === 3) {
const db = new DatabaseSync(path.join(f.data, 'workbench.sqlite'), { readOnly: true });
let plans; try { plans = db.prepare("SELECT payload FROM records WHERE kind='check_plan' ORDER BY rowid").all().map(row => JSON.parse(row.payload)); } finally { db.close(); }
calls = [call('owned-check', 'run_checks', { plan_id: plans.at(-1).id })];
} else calls = [];
assert.ok(requests <= 16); console.log(JSON.stringify({ fixtureRequest: requests, step }));
return { ...completion(calls, calls.length ? '' : 'Owned fixture done.'), model: body.model };
});
const host = await launchWorkbench(t, f.data), originalCenter = await host.api('/model-center');
const connection = await host.api('/model-center/connections', { name: 'Owned AF local fixture', brand: 'Owned fixture', model: 'owned-af', protocol: 'chat-completions',
baseUrl: model.baseUrl, apiKey: 'owned-local-fixture-only', streaming: false, nativeParameters: {} });
const boot = await host.api('/bootstrap'), center = await host.api('/model-center'), health = await host.api('/host/status');
configurePurgeFixture({ root: output, origin: host.url });
const installation = resolveControlledInstallation(process.cwd()), source = resolveOriginalPurgeSource(process.cwd(), undefined, installation);
save('current.json', { host: health, config: boot.config, center, connection, originalProjects: boot.projects,
originalPublicMetadata: publicMetadata(f.data), baseline: await publicSnapshot(host.url, f.data), installation, source, fixture: true });
await import('./run.mjs'); const batch = JSON.parse(fs.readFileSync(path.join(output, 'result.json')));
assert.equal(batch.success, true, JSON.stringify(batch.samples.map(row => ({ sample: row.sample, failure: row.failure, cleanupErrors: row.cleanupErrors }))));
assert.equal(requests, 16); assert.equal(batch.samples.length, 4); assert.ok(batch.samples.every(row => row.final.test.stats.pass === 9));
await import('./finish.mjs');
const audits = fs.readdirSync(path.join(output, 'audits')); assert.equal(audits.length, 1);
const auditPath = 'audits/' + audits[0] + '/result.json';
assert.equal(JSON.parse(fs.readFileSync(path.join(output, auditPath))).success, true);
const oldArgument = process.argv[2]; process.argv[2] = auditPath;
try { await import('./seal.mjs'); } finally { process.argv[2] = oldArgument; }
const after = await host.api('/model-center'); assert.deepEqual(after.selected, originalCenter.selected);
result = { success: true, realModelRequests: 0, fixedHttpRequests: requests, actualBatchDriverExecuted: true,
originalTestsPerSample: 9, completeSamples: 4, originalStateAuditPassed: true, implementationSuppliedToLocalFixtureOnly: true };
} catch (error) { result = { success: false, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
finally { for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(output, 'driver-result.json'), JSON.stringify(result, null, 2)); }
console.log(JSON.stringify({ output, ...result }));