test: freeze unrestricted original Purge comparison driver
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { fixture, compatibleFixture, call } from '../tests/helpers.mjs';
|
||||
import { createDeliveryProject, deliveryGit } from '../tests/fixtures/delivery-project.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
import { Runtime } from '../src/server/runtime.mjs';
|
||||
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
||||
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
||||
import { resolveOriginalPurgeSource, ORIGINAL_PURGE_KIND, assertPurgeModelInput } from '../src/core/engines/purge-strategy.mjs';
|
||||
import { originalEngineRun, verifyEngineModelInput } from '../src/shared/contracts/engine-execution.mjs';
|
||||
import { purgePrompt as prompt } from './live-af/contract.mjs';
|
||||
|
||||
const directory = path.resolve('.qa/open-original-engine-' + randomUUID());
|
||||
fs.mkdirSync(directory);
|
||||
const installation = resolveControlledInstallation(process.cwd());
|
||||
assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
const source = resolveOriginalPurgeSource(process.cwd(), undefined, installation);
|
||||
assert.equal(source.status, 'available', source.message);
|
||||
const results = [];
|
||||
for (const enabled of [false, true]) {
|
||||
const name = enabled ? 'on' : 'off', output = path.join(directory, name), disposers = [];
|
||||
fs.mkdirSync(output);
|
||||
const t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true });
|
||||
let runtime, group, timeout, result;
|
||||
try {
|
||||
createDeliveryProject(f);
|
||||
const owner = ownerBytes(f);
|
||||
const ownerIndex = deliveryGit(f, ['ls-files', '--stage', '--', 'owner-staged.txt', 'owner-unstaged.txt']);
|
||||
let requests = 0;
|
||||
const model = await compatibleFixture(t, async (_route, body) => {
|
||||
requests++;
|
||||
console.log(JSON.stringify({ name, ownedHttpRequest: requests }));
|
||||
assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
|
||||
const worker = group.children[0], plan = [...f.store.core.check_plan.values()].find(row => row.taskId === worker.taskId);
|
||||
const tools = requests === 1 ? [call('owned-read', 'read_file', { path: 'src/add.mjs' }),
|
||||
call('owned-fix', 'propose_file', { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b;\n', reason: 'Owned HTTP fixture addition repair.' })]
|
||||
: requests === 2 ? [call('owned-plan', 'plan_checks', { names: ['test', 'build'] })]
|
||||
: requests === 3 ? [call('owned-check', 'run_checks', { plan_id: plan.id })] : [];
|
||||
assert.ok(requests <= 4, 'No hidden fixture retry or continuation.');
|
||||
return { model: 'owned-open-engine', choices: [{ message: { role: 'assistant', content: tools.length ? '' : 'Owned fixture completed.',
|
||||
...(tools.length ? { tool_calls: tools } : {}) }, finish_reason: tools.length ? 'tool_calls' : 'stop' }],
|
||||
usage: { prompt_tokens: 3000000, completion_tokens: 17, total_tokens: 3000017 } };
|
||||
});
|
||||
f.store.saveConfig({ mode: 'compatible', model: 'owned-open-engine', baseUrl: model.baseUrl, approvalMode: 'full',
|
||||
executionMode: 'unrestricted', capacityMode: 'passthrough', historyPolicy: 'preserve',
|
||||
maxRunTokens: 0, maxTaskTokens: 0, maxSteps: 0, maxRunDurationMs: 0 });
|
||||
const provider = new Provider(); provider.setKey('owned-local-http-fixture-only');
|
||||
runtime = new Runtime(f.store, provider);
|
||||
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine') });
|
||||
const manager = runtime.workerManager();
|
||||
group = await manager.plan(f.session, { goal: 'Repair the original addition fixture with full automatic access; preserve owner state.',
|
||||
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256,
|
||||
...(enabled ? { strategy: { kind: ORIGINAL_PURGE_KIND, root: source.root, sourceSha256: source.sourceSha256,
|
||||
runtimeSha256: source.runtimeSha256, prompt } } : {}) },
|
||||
children: [{ label: 'candidate', goal: 'Repair only src/add.mjs, run the original test/build using plan_checks/run_checks; do not stage or commit the child workspace. Return evidence for parent delivery.', paths: ['src/add.mjs'] }] });
|
||||
assert.deepEqual(group.budget, { maxTokens: 0, maxSteps: 0, maxToolCalls: 0, maxDurationMs: 0 });
|
||||
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise;
|
||||
assert.equal(group.status, 'ready', group.summary);
|
||||
const worker = group.children[0], session = f.store.session(worker.sessionId);
|
||||
timeout = setTimeout(() => { runtime.stopAll().catch(() => {}); }, 120000);
|
||||
await manager.start(f.session, group.id, group.fingerprint).promise;
|
||||
assert.equal(group.status, 'ready_to_collect', group.summary);
|
||||
const job = [...f.store.core.engine_run.values()].find(row => row.taskId === worker.taskId);
|
||||
originalEngineRun(f.store, session, job);
|
||||
assert.equal(job.status, 'completed', job.error);
|
||||
assert.equal(job.requests.length, 4); assert.equal(requests, 4);
|
||||
assert.equal(job.native.cleanup.rootExited, true); assert.equal(job.native.cleanup.treeConfirmed, true);
|
||||
assert.equal(f.store.core.task.get(worker.taskId).usage.totalTokens, 12000068);
|
||||
for (const request of job.requests) {
|
||||
const input = verifyEngineModelInput(f.store, session, job, request);
|
||||
assertPurgeModelInput(input.native, job.installation.strategy);
|
||||
assert.equal(input.view.mode, 'full'); assert.equal(input.view.inputBudget, null);
|
||||
}
|
||||
if (enabled) {
|
||||
assert.equal(job.native.originalPurge.readiness.originalEntryLoaded, true);
|
||||
assert.equal(job.native.originalPurge.readiness.originalApplyCalled, true);
|
||||
assert.equal(job.native.originalPurge.nativeLoads.length, 2);
|
||||
} else assert.equal(job.native.originalPurge, undefined);
|
||||
await manager.collect(f.session, group.id, group.fingerprint).promise;
|
||||
await manager.prepareMerge(f.session, group.id, group.fingerprint).promise;
|
||||
assert.equal(group.status, 'merge_prepared', group.summary);
|
||||
assert.equal(f.store.core.change_set.get(group.merge.changeSetId).status, 'completed');
|
||||
const checks = runtime.checkRunner(), plan = await checks.plan(f.session, ['test', 'build']);
|
||||
const checked = await (await checks.start(f.session, plan.id)).promise;
|
||||
assert.equal(checked.status, 'passed');
|
||||
const git = runtime.gitDelivery(), deliveryPlan = await git.plan(f.session, { message: 'Owned open engine fixture delivery' });
|
||||
const delivery = await (await git.start(f.session, deliveryPlan.id)).promise;
|
||||
assert.equal(delivery.status, 'completed');
|
||||
await manager.finalize(f.session, group.id, group.fingerprint);
|
||||
assert.equal(group.status, 'completed'); assert.deepEqual(ownerBytes(f).files, owner.files);
|
||||
assert.equal(deliveryGit(f, ['ls-files', '--stage', '--', 'owner-staged.txt', 'owner-unstaged.txt']), ownerIndex);
|
||||
assert.notEqual(delivery.commit, owner.head);
|
||||
assert.ok(session.approvals.every(row => row.status === 'approved' && row.decisionSource === 'full-access-policy'));
|
||||
result = { success: true, enabled, realModelRequests: 0, ownedHttpRequests: requests,
|
||||
fixtureReportedTokens: 12000068, zeroBudgetsMeanUnlimited: true, fullHistory: true, outputCapAbsent: true,
|
||||
entryAndApplyVerified: enabled, allProcessesExited: true, ownerBytesAndIndexPreserved: true,
|
||||
humanApprovals: 0, group, job, check: checked, delivery, modelRequests: model.requests.map(({ headers, ...value }) => value) };
|
||||
console.log(JSON.stringify({ name, success: true, ownedHttpRequests: requests, fixtureReportedTokens: 12000068, commit: delivery.commit }));
|
||||
} catch (error) {
|
||||
result = { success: false, enabled, error: error.stack, group, realModelRequests: 0 };
|
||||
process.exitCode = 1;
|
||||
console.log(JSON.stringify({ name, success: false, error: error.message.split('\n')[0] }));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
await runtime?.stopAll();
|
||||
f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2));
|
||||
fs.cpSync(f.root, path.join(output, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
}
|
||||
results.push({ name, success: result.success, error: result.error ?? null });
|
||||
if (!result.success) break;
|
||||
}
|
||||
const result = { success: results.length === 2 && results.every(row => row.success), directory, results,
|
||||
realModelRequests: 0, scope: 'Actual pinned native engines with local synthetic HTTP only; no autonomous reasoning or Purge benefit claim.' };
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
||||
console.log(JSON.stringify(result));
|
||||
Reference in New Issue
Block a user