Files
Agent-Workbench/scripts/live-af/verify-driver.mjs
T
2026-11-06 13:53:53 +08:00

71 lines
6.2 KiB
JavaScript

// Synthetic fixed model output exercises the real driver and native processes.
// These implementation bytes are never sent to any real provider.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
import { fixture, compatibleFixture, launchWorkbench, completion, call } from '../../tests/helpers.mjs';
import { configurePurgeFixture, save, publicMetadata } from '../live-ad/common.mjs';
import { publicSnapshot } from '../retained-host-state.mjs';
import { resolveControlledInstallation } from '../../src/core/engines/controlled-installation.mjs';
import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs';
import { allowed } from './contract.mjs';
const batch = process.argv.includes('--batch=ag') ? 'ag' : 'af';
const output = path.resolve('.qa/' + batch + '-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
fs.mkdirSync(output); const f = fixture(t, { modernDefaults: true });
f.store.saveConfig({ approvalMode: 'full', mode: 'compatible' });
// Model the existing production metadata shape before freezing this fresh fixture.
for (const key of ['modelAttempts', 'modelNetworkDiagnostics']) f.store.storage.database.prepare('INSERT OR IGNORE INTO metadata(key,value) VALUES (?,?)').run(key, '[]');
f.store.close(); let result;
try {
const solution = path.resolve('.qa/live-validation-ad/node/expense-project');
assert.equal(JSON.parse(fs.readFileSync('.qa/live-validation-ad/node/result.json')).success, true);
const files = Object.fromEntries(allowed.map(name => [name, fs.readFileSync(path.join(solution, name), 'utf8')]));
let requests = 0;
const model = await compatibleFixture(t, (_url, body) => {
requests++; const step = (requests - 1) % 4 + 1;
assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
let calls;
if (step === 1) calls = Object.entries(files).map(([name, content], index) => call('owned-file-' + index, 'propose_file', { path: name, content, reason: 'Local fixed HTTP fixture only.' }));
else if (step === 2) calls = [call('owned-plan', 'plan_checks', { names: ['test', 'build'] })];
else if (step === 3) {
const db = new DatabaseSync(path.join(f.data, 'workbench.sqlite'), { readOnly: true });
let plans; try { plans = db.prepare("SELECT payload FROM records WHERE kind='check_plan' ORDER BY rowid").all().map(row => JSON.parse(row.payload)); } finally { db.close(); }
calls = [call('owned-check', 'run_checks', { plan_id: plans.at(-1).id })];
} else calls = [];
assert.ok(requests <= 16); console.log(JSON.stringify({ fixtureRequest: requests, step }));
return { ...completion(calls, calls.length ? '' : 'Owned fixture done.'), model: body.model };
});
const hostNode = batch === 'ag' ? JSON.parse(fs.readFileSync('.qa/live-purge-af/host-availability.json')).installation.node.path : process.execPath;
const host = await launchWorkbench(t, f.data, {}, { nodePath: hostNode }), originalCenter = await host.api('/model-center');
const connection = await host.api('/model-center/connections', { name: 'Owned AF local fixture', brand: 'Owned fixture', model: 'owned-af', protocol: 'chat-completions',
baseUrl: model.baseUrl, apiKey: 'owned-local-fixture-only', streaming: false, nativeParameters: {} });
const boot = await host.api('/bootstrap'), center = await host.api('/model-center'), health = await host.api('/host/status');
configurePurgeFixture({ root: output, origin: host.url });
const localInstallation = resolveControlledInstallation(process.cwd());
const installation = batch === 'ag' ? await host.api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(localInstallation.root)) : localInstallation;
const source = batch === 'ag' ? await host.api('/engines/purge/source?kind=purge-original-restricted-v1&harnessRoot=' + encodeURIComponent(installation.root))
: resolveOriginalPurgeSource(process.cwd(), undefined, installation);
if (batch === 'ag') { assert.notEqual(installation.node.path, localInstallation.node.path); assert.equal(installation.node.sha256, localInstallation.node.sha256); }
save('current.json', { host: health, config: boot.config, center, connection, originalProjects: boot.projects,
originalPublicMetadata: publicMetadata(f.data), baseline: await publicSnapshot(host.url, f.data), installation, source, fixture: true,
batch, ...(batch === 'ag' ? { availabilityAuthority: 'host-api', localInstallation } : {}) });
await import('./run.mjs'); const batchResult = JSON.parse(fs.readFileSync(path.join(output, 'result.json')));
assert.equal(batchResult.success, true, JSON.stringify(batchResult.samples.map(row => ({ sample: row.sample, failure: row.failure, cleanupErrors: row.cleanupErrors }))));
assert.equal(requests, 16); assert.equal(batchResult.samples.length, 4); assert.ok(batchResult.samples.every(row => row.final.test.stats.pass === 9));
await import('./finish.mjs');
const audits = fs.readdirSync(path.join(output, 'audits')); assert.equal(audits.length, 1);
const auditPath = 'audits/' + audits[0] + '/result.json';
assert.equal(JSON.parse(fs.readFileSync(path.join(output, auditPath))).success, true);
const oldArgument = process.argv[2]; process.argv[2] = auditPath;
try { await import('./seal.mjs'); } finally { process.argv[2] = oldArgument; }
const after = await host.api('/model-center'); assert.deepEqual(after.selected, originalCenter.selected);
result = { success: true, realModelRequests: 0, fixedHttpRequests: requests, actualBatchDriverExecuted: true,
originalTestsPerSample: 9, completeSamples: 4, originalStateAuditPassed: true, implementationSuppliedToLocalFixtureOnly: true,
distinctHostExecutablePathVerified: batch === 'ag', availabilityAuthority: batch === 'ag' ? 'host-api' : 'source' };
} catch (error) { result = { success: false, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
finally { for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(output, 'driver-result.json'), JSON.stringify(result, null, 2)); }
console.log(JSON.stringify({ output, ...result }));