test: freeze host-authoritative original Purge validation

This commit is contained in:
Agent Workbench Dev
2026-11-06 13:53:53 +08:00
parent 01e64f3fcf
commit a6830bbf50
16 changed files with 256 additions and 15 deletions
+6 -2
View File
@@ -80,8 +80,12 @@ try {
const store = (file, value) => save(prefix + '/' + file, value), startedAt = Date.now();
let own, session, worker, group, workerState, failure, result, route;
try {
const installation = resolveControlledInstallation(process.cwd(), frozen.installation.root);
const source = resolveOriginalPurgeSource(process.cwd(), frozen.source.root, installation);
const installation = frozen.availabilityAuthority === 'host-api'
? await api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(frozen.installation.root))
: resolveControlledInstallation(process.cwd(), frozen.installation.root);
const source = frozen.availabilityAuthority === 'host-api'
? await api('/engines/purge/source?kind=' + ORIGINAL_PURGE_KIND + '&root=' + encodeURIComponent(frozen.source.root) + '&harnessRoot=' + encodeURIComponent(installation.root))
: resolveOriginalPurgeSource(process.cwd(), frozen.source.root, installation);
assert.equal(installation.controlledSha256, frozen.installation.controlledSha256);
assert.equal(source.sourceSha256, frozen.source.sourceSha256); assert.equal(source.runtimeSha256, frozen.source.runtimeSha256);
const current = await api('/model-center');
+4 -2
View File
@@ -8,6 +8,8 @@ const auditPath = process.argv[2]; assert.ok(auditPath, 'Explicit completed audi
const audit = read(auditPath); assert.equal(audit.success, true);
const frozen = read('current.json'), result = read('result.json'), samples = [], allAttempts = [], tools = new Set();
for (const sample of result.samples) {
if (!sample.workerSessionId) { samples.push({ ...sample, requests: 0, usage: { requests: 0, inputTokens: 0, outputTokens: 0, totalTokens: 0,
unknownRequests: 0, estimatedUnknownTokens: 0 }, dispatchStage: 'before-worker-and-real-model', originalPurge: [] }); continue; }
const prefix = sample.sample, state = read(prefix + '/worker/session.json'), attempts = read(prefix + '/worker/attempts.json');
const jobs = read(prefix + '/worker/engine-jobs.json'), proof = read(prefix + '/worker/engine-proof.json');
assert.equal(attempts.length, state.execution.task.usage.requests);
@@ -68,9 +70,9 @@ function walk(directory) {
}
}
walk(root); files.sort((a, b) => a.path.localeCompare(b.path)); const bytes = files.reduce((sum, row) => sum + row.bytes, 0);
save('FINAL-MANIFEST.json', { schema: 1, files, bytes, scope: 'All AF raw requests/responses, original engine inputs/reports, source/Git, checks and audits. Historical evidence is not repackaged.' });
save('FINAL-MANIFEST.json', { schema: 1, files, bytes, scope: 'All ' + (frozen.batch ?? 'af').toUpperCase() + ' raw requests/responses, original engine inputs/reports, source/Git, checks and audits. Historical evidence is not repackaged.' });
for (const row of files) { const value = fs.readFileSync(path.join(root, row.path)); assert.equal(value.length, row.bytes); assert.equal(sha(value), row.sha256); }
if (!frozen.fixture) fs.writeFileSync('docs/live-purge-af-evidence.json', JSON.stringify({ summary, evidence: { root, files: files.length, bytes,
if (frozen.fixture !== true) fs.writeFileSync('docs/live-purge-' + (frozen.batch ?? 'af') + '-evidence.json', JSON.stringify({ summary, evidence: { root, files: files.length, bytes,
manifestSha256: sha(fs.readFileSync(path.join(root, 'FINAL-MANIFEST.json'))) } }, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify({ success: true, realRequests: summary.requests, reportedTokens: summary.reportedTokens, unknown: summary.unknownRequests,
allFourSuccessful: fourSuccessful, sameTools, comparison, files: files.length, bytes }));
+16 -8
View File
@@ -12,7 +12,8 @@ import { resolveControlledInstallation } from '../../src/core/engines/controlled
import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs';
import { allowed } from './contract.mjs';
const output = path.resolve('.qa/af-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
const batch = process.argv.includes('--batch=ag') ? 'ag' : 'af';
const output = path.resolve('.qa/' + batch + '-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
fs.mkdirSync(output); const f = fixture(t, { modernDefaults: true });
f.store.saveConfig({ approvalMode: 'full', mode: 'compatible' });
// Model the existing production metadata shape before freezing this fresh fixture.
@@ -37,17 +38,23 @@ try {
assert.ok(requests <= 16); console.log(JSON.stringify({ fixtureRequest: requests, step }));
return { ...completion(calls, calls.length ? '' : 'Owned fixture done.'), model: body.model };
});
const host = await launchWorkbench(t, f.data), originalCenter = await host.api('/model-center');
const hostNode = batch === 'ag' ? JSON.parse(fs.readFileSync('.qa/live-purge-af/host-availability.json')).installation.node.path : process.execPath;
const host = await launchWorkbench(t, f.data, {}, { nodePath: hostNode }), originalCenter = await host.api('/model-center');
const connection = await host.api('/model-center/connections', { name: 'Owned AF local fixture', brand: 'Owned fixture', model: 'owned-af', protocol: 'chat-completions',
baseUrl: model.baseUrl, apiKey: 'owned-local-fixture-only', streaming: false, nativeParameters: {} });
const boot = await host.api('/bootstrap'), center = await host.api('/model-center'), health = await host.api('/host/status');
configurePurgeFixture({ root: output, origin: host.url });
const installation = resolveControlledInstallation(process.cwd()), source = resolveOriginalPurgeSource(process.cwd(), undefined, installation);
const localInstallation = resolveControlledInstallation(process.cwd());
const installation = batch === 'ag' ? await host.api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(localInstallation.root)) : localInstallation;
const source = batch === 'ag' ? await host.api('/engines/purge/source?kind=purge-original-restricted-v1&harnessRoot=' + encodeURIComponent(installation.root))
: resolveOriginalPurgeSource(process.cwd(), undefined, installation);
if (batch === 'ag') { assert.notEqual(installation.node.path, localInstallation.node.path); assert.equal(installation.node.sha256, localInstallation.node.sha256); }
save('current.json', { host: health, config: boot.config, center, connection, originalProjects: boot.projects,
originalPublicMetadata: publicMetadata(f.data), baseline: await publicSnapshot(host.url, f.data), installation, source, fixture: true });
await import('./run.mjs'); const batch = JSON.parse(fs.readFileSync(path.join(output, 'result.json')));
assert.equal(batch.success, true, JSON.stringify(batch.samples.map(row => ({ sample: row.sample, failure: row.failure, cleanupErrors: row.cleanupErrors }))));
assert.equal(requests, 16); assert.equal(batch.samples.length, 4); assert.ok(batch.samples.every(row => row.final.test.stats.pass === 9));
originalPublicMetadata: publicMetadata(f.data), baseline: await publicSnapshot(host.url, f.data), installation, source, fixture: true,
batch, ...(batch === 'ag' ? { availabilityAuthority: 'host-api', localInstallation } : {}) });
await import('./run.mjs'); const batchResult = JSON.parse(fs.readFileSync(path.join(output, 'result.json')));
assert.equal(batchResult.success, true, JSON.stringify(batchResult.samples.map(row => ({ sample: row.sample, failure: row.failure, cleanupErrors: row.cleanupErrors }))));
assert.equal(requests, 16); assert.equal(batchResult.samples.length, 4); assert.ok(batchResult.samples.every(row => row.final.test.stats.pass === 9));
await import('./finish.mjs');
const audits = fs.readdirSync(path.join(output, 'audits')); assert.equal(audits.length, 1);
const auditPath = 'audits/' + audits[0] + '/result.json';
@@ -56,7 +63,8 @@ try {
try { await import('./seal.mjs'); } finally { process.argv[2] = oldArgument; }
const after = await host.api('/model-center'); assert.deepEqual(after.selected, originalCenter.selected);
result = { success: true, realModelRequests: 0, fixedHttpRequests: requests, actualBatchDriverExecuted: true,
originalTestsPerSample: 9, completeSamples: 4, originalStateAuditPassed: true, implementationSuppliedToLocalFixtureOnly: true };
originalTestsPerSample: 9, completeSamples: 4, originalStateAuditPassed: true, implementationSuppliedToLocalFixtureOnly: true,
distinctHostExecutablePathVerified: batch === 'ag', availabilityAuthority: batch === 'ag' ? 'host-api' : 'source' };
} catch (error) { result = { success: false, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
finally { for (const dispose of disposers.reverse()) await dispose(); fs.writeFileSync(path.join(output, 'driver-result.json'), JSON.stringify(result, null, 2)); }
console.log(JSON.stringify({ output, ...result }));