test: freeze host-authoritative original Purge validation
This commit is contained in:
@@ -8,10 +8,10 @@ import { Agent, fetch } from 'undici';
|
||||
export let root = path.resolve('.qa/live-validation-ad'), origin = 'http://127.0.0.1:4318';
|
||||
export function engineeringEvidence() { root = path.resolve('.qa/live-engineering-ae'); }
|
||||
let purgeFixture;
|
||||
export function purgeEvidence() { root = purgeFixture?.root ?? path.resolve('.qa/live-purge-af'); if (purgeFixture) origin = purgeFixture.origin; }
|
||||
export function purgeEvidence() { root = purgeFixture?.root ?? path.resolve(process.argv.includes('--batch=ag') ? '.qa/live-purge-ag' : '.qa/live-purge-af'); if (purgeFixture) origin = purgeFixture.origin; }
|
||||
export function configurePurgeFixture(options) {
|
||||
const target = path.resolve(options.root), url = new URL(options.origin);
|
||||
assert.equal(path.dirname(target), path.resolve('.qa')); assert.match(path.basename(target), /^af-driver-[a-f0-9-]{36}$/);
|
||||
assert.equal(path.dirname(target), path.resolve('.qa')); assert.match(path.basename(target), /^a[fg]-driver-[a-f0-9-]{36}$/);
|
||||
assert.equal(url.hostname, '127.0.0.1'); assert.notEqual(url.port, '4318'); assert.ok(url.port);
|
||||
purgeFixture = { root: target, origin: url.origin }; purgeEvidence();
|
||||
}
|
||||
|
||||
@@ -80,8 +80,12 @@ try {
|
||||
const store = (file, value) => save(prefix + '/' + file, value), startedAt = Date.now();
|
||||
let own, session, worker, group, workerState, failure, result, route;
|
||||
try {
|
||||
const installation = resolveControlledInstallation(process.cwd(), frozen.installation.root);
|
||||
const source = resolveOriginalPurgeSource(process.cwd(), frozen.source.root, installation);
|
||||
const installation = frozen.availabilityAuthority === 'host-api'
|
||||
? await api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(frozen.installation.root))
|
||||
: resolveControlledInstallation(process.cwd(), frozen.installation.root);
|
||||
const source = frozen.availabilityAuthority === 'host-api'
|
||||
? await api('/engines/purge/source?kind=' + ORIGINAL_PURGE_KIND + '&root=' + encodeURIComponent(frozen.source.root) + '&harnessRoot=' + encodeURIComponent(installation.root))
|
||||
: resolveOriginalPurgeSource(process.cwd(), frozen.source.root, installation);
|
||||
assert.equal(installation.controlledSha256, frozen.installation.controlledSha256);
|
||||
assert.equal(source.sourceSha256, frozen.source.sourceSha256); assert.equal(source.runtimeSha256, frozen.source.runtimeSha256);
|
||||
const current = await api('/model-center');
|
||||
|
||||
@@ -8,6 +8,8 @@ const auditPath = process.argv[2]; assert.ok(auditPath, 'Explicit completed audi
|
||||
const audit = read(auditPath); assert.equal(audit.success, true);
|
||||
const frozen = read('current.json'), result = read('result.json'), samples = [], allAttempts = [], tools = new Set();
|
||||
for (const sample of result.samples) {
|
||||
if (!sample.workerSessionId) { samples.push({ ...sample, requests: 0, usage: { requests: 0, inputTokens: 0, outputTokens: 0, totalTokens: 0,
|
||||
unknownRequests: 0, estimatedUnknownTokens: 0 }, dispatchStage: 'before-worker-and-real-model', originalPurge: [] }); continue; }
|
||||
const prefix = sample.sample, state = read(prefix + '/worker/session.json'), attempts = read(prefix + '/worker/attempts.json');
|
||||
const jobs = read(prefix + '/worker/engine-jobs.json'), proof = read(prefix + '/worker/engine-proof.json');
|
||||
assert.equal(attempts.length, state.execution.task.usage.requests);
|
||||
@@ -68,9 +70,9 @@ function walk(directory) {
|
||||
}
|
||||
}
|
||||
walk(root); files.sort((a, b) => a.path.localeCompare(b.path)); const bytes = files.reduce((sum, row) => sum + row.bytes, 0);
|
||||
save('FINAL-MANIFEST.json', { schema: 1, files, bytes, scope: 'All AF raw requests/responses, original engine inputs/reports, source/Git, checks and audits. Historical evidence is not repackaged.' });
|
||||
save('FINAL-MANIFEST.json', { schema: 1, files, bytes, scope: 'All ' + (frozen.batch ?? 'af').toUpperCase() + ' raw requests/responses, original engine inputs/reports, source/Git, checks and audits. Historical evidence is not repackaged.' });
|
||||
for (const row of files) { const value = fs.readFileSync(path.join(root, row.path)); assert.equal(value.length, row.bytes); assert.equal(sha(value), row.sha256); }
|
||||
if (!frozen.fixture) fs.writeFileSync('docs/live-purge-af-evidence.json', JSON.stringify({ summary, evidence: { root, files: files.length, bytes,
|
||||
if (frozen.fixture !== true) fs.writeFileSync('docs/live-purge-' + (frozen.batch ?? 'af') + '-evidence.json', JSON.stringify({ summary, evidence: { root, files: files.length, bytes,
|
||||
manifestSha256: sha(fs.readFileSync(path.join(root, 'FINAL-MANIFEST.json'))) } }, null, 2) + '\n', { flag: 'wx' });
|
||||
console.log(JSON.stringify({ success: true, realRequests: summary.requests, reportedTokens: summary.reportedTokens, unknown: summary.unknownRequests,
|
||||
allFourSuccessful: fourSuccessful, sameTools, comparison, files: files.length, bytes }));
|
||||
|
||||
@@ -12,7 +12,8 @@ import { resolveControlledInstallation } from '../../src/core/engines/controlled
|
||||
import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs';
|
||||
import { allowed } from './contract.mjs';
|
||||
|
||||
const output = path.resolve('.qa/af-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
|
||||
const batch = process.argv.includes('--batch=ag') ? 'ag' : 'af';
|
||||
const output = path.resolve('.qa/' + batch + '-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
|
||||
fs.mkdirSync(output); const f = fixture(t, { modernDefaults: true });
|
||||
f.store.saveConfig({ approvalMode: 'full', mode: 'compatible' });
|
||||
// Model the existing production metadata shape before freezing this fresh fixture.
|
||||
@@ -37,17 +38,23 @@ try {
|
||||
assert.ok(requests <= 16); console.log(JSON.stringify({ fixtureRequest: requests, step }));
|
||||
return { ...completion(calls, calls.length ? '' : 'Owned fixture done.'), model: body.model };
|
||||
});
|
||||
const host = await launchWorkbench(t, f.data), originalCenter = await host.api('/model-center');
|
||||
const hostNode = batch === 'ag' ? JSON.parse(fs.readFileSync('.qa/live-purge-af/host-availability.json')).installation.node.path : process.execPath;
|
||||
const host = await launchWorkbench(t, f.data, {}, { nodePath: hostNode }), originalCenter = await host.api('/model-center');
|
||||
const connection = await host.api('/model-center/connections', { name: 'Owned AF local fixture', brand: 'Owned fixture', model: 'owned-af', protocol: 'chat-completions',
|
||||
baseUrl: model.baseUrl, apiKey: 'owned-local-fixture-only', streaming: false, nativeParameters: {} });
|
||||
const boot = await host.api('/bootstrap'), center = await host.api('/model-center'), health = await host.api('/host/status');
|
||||
configurePurgeFixture({ root: output, origin: host.url });
|
||||
const installation = resolveControlledInstallation(process.cwd()), source = resolveOriginalPurgeSource(process.cwd(), undefined, installation);
|
||||
const localInstallation = resolveControlledInstallation(process.cwd());
|
||||
const installation = batch === 'ag' ? await host.api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(localInstallation.root)) : localInstallation;
|
||||
const source = batch === 'ag' ? await host.api('/engines/purge/source?kind=purge-original-restricted-v1&harnessRoot=' + encodeURIComponent(installation.root))
|
||||
: resolveOriginalPurgeSource(process.cwd(), undefined, installation);
|
||||
if (batch === 'ag') { assert.notEqual(installation.node.path, localInstallation.node.path); assert.equal(installation.node.sha256, localInstallation.node.sha256); }
|
||||
save('current.json', { host: health, config: boot.config, center, connection, originalProjects: boot.projects,
|
||||
originalPublicMetadata: publicMetadata(f.data), baseline: await publicSnapshot(host.url, f.data), installation, source, fixture: true });
|
||||
await import('./run.mjs'); const batch = JSON.parse(fs.readFileSync(path.join(output, 'result.json')));
|
||||
assert.equal(batch.success, true, JSON.stringify(batch.samples.map(row => ({ sample: row.sample, failure: row.failure, cleanupErrors: row.cleanupErrors }))));
|
||||
assert.equal(requests, 16); assert.equal(batch.samples.length, 4); assert.ok(batch.samples.every(row => row.final.test.stats.pass === 9));
|
||||
originalPublicMetadata: publicMetadata(f.data), baseline: await publicSnapshot(host.url, f.data), installation, source, fixture: true,
|
||||
batch, ...(batch === 'ag' ? { availabilityAuthority: 'host-api', localInstallation } : {}) });
|
||||
await import('./run.mjs'); const batchResult = JSON.parse(fs.readFileSync(path.join(output, 'result.json')));
|
||||
assert.equal(batchResult.success, true, JSON.stringify(batchResult.samples.map(row => ({ sample: row.sample, failure: row.failure, cleanupErrors: row.cleanupErrors }))));
|
||||
assert.equal(requests, 16); assert.equal(batchResult.samples.length, 4); assert.ok(batchResult.samples.every(row => row.final.test.stats.pass === 9));
|
||||
await import('./finish.mjs');
|
||||
const audits = fs.readdirSync(path.join(output, 'audits')); assert.equal(audits.length, 1);
|
||||
const auditPath = 'audits/' + audits[0] + '/result.json';
|
||||
@@ -56,7 +63,8 @@ try {
|
||||
try { await import('./seal.mjs'); } finally { process.argv[2] = oldArgument; }
|
||||
const after = await host.api('/model-center'); assert.deepEqual(after.selected, originalCenter.selected);
|
||||
result = { success: true, realModelRequests: 0, fixedHttpRequests: requests, actualBatchDriverExecuted: true,
|
||||
originalTestsPerSample: 9, completeSamples: 4, originalStateAuditPassed: true, implementationSuppliedToLocalFixtureOnly: true };
|
||||
originalTestsPerSample: 9, completeSamples: 4, originalStateAuditPassed: true, implementationSuppliedToLocalFixtureOnly: true,
|
||||
distinctHostExecutablePathVerified: batch === 'ag', availabilityAuthority: batch === 'ag' ? 'host-api' : 'source' };
|
||||
} catch (error) { result = { success: false, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
|
||||
finally { for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(output, 'driver-result.json'), JSON.stringify(result, null, 2)); }
|
||||
console.log(JSON.stringify({ output, ...result }));
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import fs from 'node:fs';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { purgeEvidence, root, save, client, publicMetadata } from '../live-ad/common.mjs';
|
||||
import { publicSnapshot } from '../retained-host-state.mjs';
|
||||
import { inspectorClosed } from '../live-ad/inspector.mjs';
|
||||
import { order, allowed, goal, purgePrompt, observations } from '../live-af/contract.mjs';
|
||||
|
||||
assert.ok(process.argv.includes('--batch=ag'));
|
||||
const af = JSON.parse(fs.readFileSync('.qa/live-purge-af/summary.json'));
|
||||
assert.equal(af.requests, 0); assert.equal(af.originalStateAuditResult.success, true);
|
||||
const ae = JSON.parse(fs.readFileSync('.qa/live-engineering-ae/summary.json'));
|
||||
assert.ok(ae.samples.length === 2 && ae.samples.every(row => row.success));
|
||||
const fixturePath = process.argv[2], fixture = JSON.parse(fs.readFileSync(fixturePath));
|
||||
for (const key of ['success', 'originalStateAuditPassed', 'actualBatchDriverExecuted', 'distinctHostExecutablePathVerified']) assert.equal(fixture[key], true);
|
||||
assert.equal(fixture.realModelRequests, 0); assert.equal(fixture.completeSamples, 4); assert.equal(fixture.availabilityAuthority, 'host-api');
|
||||
purgeEvidence(); assert.equal(fs.existsSync(root), false, 'Never overwrite or continue an observed AG batch.');
|
||||
const git = args => execFileSync('git', args, { encoding: 'utf8', windowsHide: true });
|
||||
const workingChanges = { tracked: git(['diff', '--name-only', 'HEAD', '-z']).split('\0').filter(Boolean), untracked: git(['ls-files', '--others', '--exclude-standard', '-z']).split('\0').filter(Boolean) };
|
||||
const parallelBrandingPaths = ['DESIGN.md', 'README.md', 'index.html', 'mobile.html', 'package.json', 'src/desktop/main.mjs',
|
||||
'scripts/generate-orven-icons.mjs', 'scripts/verify-orven-ui.mjs', 'scripts/verify-model-center-ui.mjs'];
|
||||
for (const name of [...workingChanges.tracked, ...workingChanges.untracked]) assert.ok(parallelBrandingPaths.includes(name)
|
||||
|| ['src/client/', 'public/', 'docs/orven/', 'output/'].some(prefix => name.startsWith(prefix)), 'Effective uncommitted source: ' + name);
|
||||
const { api, boot, close } = await client();
|
||||
try {
|
||||
await inspectorClosed(); const host = await api('/host/status'), center = await api('/model-center');
|
||||
assert.equal(boot.version, '1.25.8'); assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0);
|
||||
for (const [key, value] of Object.entries({ executionMode: 'unrestricted', approvalMode: 'full', capacityMode: 'passthrough', historyPolicy: 'preserve',
|
||||
maxSteps: 0, maxRunTokens: 0, maxTaskTokens: 0, maxRunDurationMs: 0 })) assert.equal(boot.config[key], value);
|
||||
const connection = center.connections.find(row => row.id === '1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd');
|
||||
assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.model, 'deepseek-flash');
|
||||
assert.equal(connection.credentialRevision, 1); assert.deepEqual(connection.nativeParameters, {});
|
||||
const installation = await api('/engines/deepseek/controlled-availability'), source = await api('/engines/purge/source?kind=purge-original-restricted-v1&harnessRoot=' + encodeURIComponent(installation.root));
|
||||
assert.equal(installation.controlledStatus, 'available'); assert.equal(source.status, 'available');
|
||||
const prior = JSON.parse(fs.readFileSync('.qa/live-purge-af/host-availability.json'));
|
||||
assert.equal(installation.controlledSha256, prior.installation.controlledSha256); assert.equal(source.sourceSha256, prior.source.sourceSha256); assert.equal(source.runtimeSha256, prior.source.runtimeSha256);
|
||||
const baseline = await publicSnapshot('http://127.0.0.1:4318', host.identity.dataRoot);
|
||||
save('current.json', { batch: 'ag', availabilityAuthority: 'host-api', host, config: boot.config, center, connection, originalProjects: boot.projects,
|
||||
originalPublicMetadata: publicMetadata(host.identity.dataRoot), baseline, installation, source, order, allowed, goal, purgePrompt, observations,
|
||||
fixturePath, driverFixture: fixture, workingChanges, previousBatch: { id: 'AF', requests: 0, outcome: 'Host rejected source-runtime identity before model dispatch; sealed unchanged.' }, driverCommit: git(['rev-parse', 'HEAD']).trim() });
|
||||
console.log(JSON.stringify({ frozen: true, batch: 'AG', requests: 0, version: boot.version, pid: host.identity.pid, projects: baseline.projects,
|
||||
sessions: baseline.database.sessionIds.length, connectionRevision: connection.revision, controlledSha256: installation.controlledSha256, runtimeSha256: source.runtimeSha256 }));
|
||||
} finally { await close(); }
|
||||
Reference in New Issue
Block a user