// Synthetic fixed model output exercises the real driver and native processes. // These implementation bytes are never sent to any real provider. import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; import { DatabaseSync } from 'node:sqlite'; import { fixture, compatibleFixture, launchWorkbench, completion, call } from '../../tests/helpers.mjs'; import { configurePurgeFixture, save, publicMetadata } from '../live-ad/common.mjs'; import { publicSnapshot } from '../retained-host-state.mjs'; import { resolveControlledInstallation } from '../../src/core/engines/controlled-installation.mjs'; import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs'; import { allowed } from './contract.mjs'; const batch = process.argv.includes('--batch=ag') ? 'ag' : 'af'; const output = path.resolve('.qa/' + batch + '-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) }; fs.mkdirSync(output); const f = fixture(t, { modernDefaults: true }); f.store.saveConfig({ approvalMode: 'full', mode: 'compatible' }); // Model the existing production metadata shape before freezing this fresh fixture. for (const key of ['modelAttempts', 'modelNetworkDiagnostics']) f.store.storage.database.prepare('INSERT OR IGNORE INTO metadata(key,value) VALUES (?,?)').run(key, '[]'); f.store.close(); let result; try { const solution = path.resolve('.qa/live-validation-ad/node/expense-project'); assert.equal(JSON.parse(fs.readFileSync('.qa/live-validation-ad/node/result.json')).success, true); const files = Object.fromEntries(allowed.map(name => [name, fs.readFileSync(path.join(solution, name), 'utf8')])); let requests = 0; const model = await compatibleFixture(t, (_url, body) => { requests++; const step = (requests - 1) % 4 + 1; assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key))); let calls; if (step === 1) calls = Object.entries(files).map(([name, content], index) => call('owned-file-' + index, 'propose_file', { path: name, content, reason: 'Local fixed HTTP fixture only.' })); else if (step === 2) calls = [call('owned-plan', 'plan_checks', { names: ['test', 'build'] })]; else if (step === 3) { const db = new DatabaseSync(path.join(f.data, 'workbench.sqlite'), { readOnly: true }); let plans; try { plans = db.prepare("SELECT payload FROM records WHERE kind='check_plan' ORDER BY rowid").all().map(row => JSON.parse(row.payload)); } finally { db.close(); } calls = [call('owned-check', 'run_checks', { plan_id: plans.at(-1).id })]; } else calls = []; assert.ok(requests <= 16); console.log(JSON.stringify({ fixtureRequest: requests, step })); return { ...completion(calls, calls.length ? '' : 'Owned fixture done.'), model: body.model }; }); const hostNode = batch === 'ag' ? JSON.parse(fs.readFileSync('.qa/live-purge-af/host-availability.json')).installation.node.path : process.execPath; const host = await launchWorkbench(t, f.data, {}, { nodePath: hostNode }), originalCenter = await host.api('/model-center'); const connection = await host.api('/model-center/connections', { name: 'Owned AF local fixture', brand: 'Owned fixture', model: 'owned-af', protocol: 'chat-completions', baseUrl: model.baseUrl, apiKey: 'owned-local-fixture-only', streaming: false, nativeParameters: {} }); const boot = await host.api('/bootstrap'), center = await host.api('/model-center'), health = await host.api('/host/status'); configurePurgeFixture({ root: output, origin: host.url }); const localInstallation = resolveControlledInstallation(process.cwd()); const installation = batch === 'ag' ? await host.api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(localInstallation.root)) : localInstallation; const source = batch === 'ag' ? await host.api('/engines/purge/source?kind=purge-original-restricted-v1&harnessRoot=' + encodeURIComponent(installation.root)) : resolveOriginalPurgeSource(process.cwd(), undefined, installation); if (batch === 'ag') { assert.notEqual(installation.node.path, localInstallation.node.path); assert.equal(installation.node.sha256, localInstallation.node.sha256); } save('current.json', { host: health, config: boot.config, center, connection, originalProjects: boot.projects, originalPublicMetadata: publicMetadata(f.data), baseline: await publicSnapshot(host.url, f.data), installation, source, fixture: true, batch, ...(batch === 'ag' ? { availabilityAuthority: 'host-api', localInstallation } : {}) }); await import('./run.mjs'); const batchResult = JSON.parse(fs.readFileSync(path.join(output, 'result.json'))); assert.equal(batchResult.success, true, JSON.stringify(batchResult.samples.map(row => ({ sample: row.sample, failure: row.failure, cleanupErrors: row.cleanupErrors })))); assert.equal(requests, 16); assert.equal(batchResult.samples.length, 4); assert.ok(batchResult.samples.every(row => row.final.test.stats.pass === 9)); await import('./finish.mjs'); const audits = fs.readdirSync(path.join(output, 'audits')); assert.equal(audits.length, 1); const auditPath = 'audits/' + audits[0] + '/result.json'; assert.equal(JSON.parse(fs.readFileSync(path.join(output, auditPath))).success, true); const oldArgument = process.argv[2]; process.argv[2] = auditPath; try { await import('./seal.mjs'); } finally { process.argv[2] = oldArgument; } const after = await host.api('/model-center'); assert.deepEqual(after.selected, originalCenter.selected); result = { success: true, realModelRequests: 0, fixedHttpRequests: requests, actualBatchDriverExecuted: true, originalTestsPerSample: 9, completeSamples: 4, originalStateAuditPassed: true, implementationSuppliedToLocalFixtureOnly: true, distinctHostExecutablePathVerified: batch === 'ag', availabilityAuthority: batch === 'ag' ? 'host-api' : 'source' }; } catch (error) { result = { success: false, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; } finally { for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(output, 'driver-result.json'), JSON.stringify(result, null, 2)); } console.log(JSON.stringify({ output, ...result }));