Files
Agent-Workbench/scripts/verify-open-original-engine.mjs
T
2026-11-06 13:40:29 +08:00

121 lines
8.8 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { fixture, compatibleFixture, call } from '../tests/helpers.mjs';
import { createDeliveryProject, deliveryGit } from '../tests/fixtures/delivery-project.mjs';
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
import { resolveOriginalPurgeSource, ORIGINAL_PURGE_KIND, assertPurgeModelInput } from '../src/core/engines/purge-strategy.mjs';
import { originalEngineRun, verifyEngineModelInput } from '../src/shared/contracts/engine-execution.mjs';
import { purgePrompt as prompt } from './live-af/contract.mjs';
const directory = path.resolve('.qa/open-original-engine-' + randomUUID());
fs.mkdirSync(directory);
const installation = resolveControlledInstallation(process.cwd());
assert.equal(installation.controlledStatus, 'available', installation.message);
const source = resolveOriginalPurgeSource(process.cwd(), undefined, installation);
assert.equal(source.status, 'available', source.message);
const results = [];
for (const enabled of [false, true]) {
const name = enabled ? 'on' : 'off', output = path.join(directory, name), disposers = [];
fs.mkdirSync(output);
const t = { after: fn => disposers.push(fn) }, f = fixture(t, { modernDefaults: true });
let runtime, group, timeout, result;
try {
createDeliveryProject(f);
const owner = ownerBytes(f);
const ownerIndex = deliveryGit(f, ['ls-files', '--stage', '--', 'owner-staged.txt', 'owner-unstaged.txt']);
let requests = 0;
const model = await compatibleFixture(t, async (_route, body) => {
requests++;
console.log(JSON.stringify({ name, ownedHttpRequest: requests }));
assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
const worker = group.children[0], plan = [...f.store.core.check_plan.values()].find(row => row.taskId === worker.taskId);
const tools = requests === 1 ? [call('owned-read', 'read_file', { path: 'src/add.mjs' }),
call('owned-fix', 'propose_file', { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b;\n', reason: 'Owned HTTP fixture addition repair.' })]
: requests === 2 ? [call('owned-plan', 'plan_checks', { names: ['test', 'build'] })]
: requests === 3 ? [call('owned-check', 'run_checks', { plan_id: plan.id })] : [];
assert.ok(requests <= 4, 'No hidden fixture retry or continuation.');
return { model: 'owned-open-engine', choices: [{ message: { role: 'assistant', content: tools.length ? '' : 'Owned fixture completed.',
...(tools.length ? { tool_calls: tools } : {}) }, finish_reason: tools.length ? 'tool_calls' : 'stop' }],
usage: { prompt_tokens: 3000000, completion_tokens: 17, total_tokens: 3000017 } };
});
f.store.saveConfig({ mode: 'compatible', model: 'owned-open-engine', baseUrl: model.baseUrl, approvalMode: 'full',
executionMode: 'unrestricted', capacityMode: 'passthrough', historyPolicy: 'preserve',
maxRunTokens: 0, maxTaskTokens: 0, maxSteps: 0, maxRunDurationMs: 0 });
const provider = new Provider(); provider.setKey('owned-local-http-fixture-only');
runtime = new Runtime(f.store, provider);
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine') });
const manager = runtime.workerManager();
group = await manager.plan(f.session, { goal: 'Repair the original addition fixture with full automatic access; preserve owner state.',
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256,
...(enabled ? { strategy: { kind: ORIGINAL_PURGE_KIND, root: source.root, sourceSha256: source.sourceSha256,
runtimeSha256: source.runtimeSha256, prompt } } : {}) },
children: [{ label: 'candidate', goal: 'Repair only src/add.mjs, run the original test/build using plan_checks/run_checks; do not stage or commit the child workspace. Return evidence for parent delivery.', paths: ['src/add.mjs'] }] });
assert.deepEqual(group.budget, { maxTokens: 0, maxSteps: 0, maxToolCalls: 0, maxDurationMs: 0 });
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise;
assert.equal(group.status, 'ready', group.summary);
const worker = group.children[0], session = f.store.session(worker.sessionId);
timeout = setTimeout(() => { runtime.stopAll().catch(() => {}); }, 120000);
await manager.start(f.session, group.id, group.fingerprint).promise;
assert.equal(group.status, 'ready_to_collect', group.summary);
const job = [...f.store.core.engine_run.values()].find(row => row.taskId === worker.taskId);
originalEngineRun(f.store, session, job);
assert.equal(job.status, 'completed', job.error);
assert.equal(job.requests.length, 4); assert.equal(requests, 4);
assert.equal(job.native.cleanup.rootExited, true); assert.equal(job.native.cleanup.treeConfirmed, true);
assert.equal(f.store.core.task.get(worker.taskId).usage.totalTokens, 12000068);
for (const request of job.requests) {
const input = verifyEngineModelInput(f.store, session, job, request);
assertPurgeModelInput(input.native, job.installation.strategy);
assert.equal(input.view.mode, 'full'); assert.equal(input.view.inputBudget, null);
}
if (enabled) {
assert.equal(job.native.originalPurge.readiness.originalEntryLoaded, true);
assert.equal(job.native.originalPurge.readiness.originalApplyCalled, true);
assert.equal(job.native.originalPurge.nativeLoads.length, 2);
} else assert.equal(job.native.originalPurge, undefined);
await manager.collect(f.session, group.id, group.fingerprint).promise;
await manager.prepareMerge(f.session, group.id, group.fingerprint).promise;
assert.equal(group.status, 'merge_prepared', group.summary);
assert.equal(f.store.core.change_set.get(group.merge.changeSetId).status, 'completed');
const checks = runtime.checkRunner(), plan = await checks.plan(f.session, ['test', 'build']);
const checked = await (await checks.start(f.session, plan.id)).promise;
assert.equal(checked.status, 'passed');
const git = runtime.gitDelivery(), deliveryPlan = await git.plan(f.session, { message: 'Owned open engine fixture delivery' });
const delivery = await (await git.start(f.session, deliveryPlan.id)).promise;
assert.equal(delivery.status, 'completed');
await manager.finalize(f.session, group.id, group.fingerprint);
assert.equal(group.status, 'completed'); assert.deepEqual(ownerBytes(f).files, owner.files);
assert.equal(deliveryGit(f, ['ls-files', '--stage', '--', 'owner-staged.txt', 'owner-unstaged.txt']), ownerIndex);
assert.notEqual(delivery.commit, owner.head);
assert.ok(session.approvals.every(row => row.status === 'approved' && row.decisionSource === 'full-access-policy'));
result = { success: true, enabled, realModelRequests: 0, ownedHttpRequests: requests,
fixtureReportedTokens: 12000068, zeroBudgetsMeanUnlimited: true, fullHistory: true, outputCapAbsent: true,
entryAndApplyVerified: enabled, allProcessesExited: true, ownerBytesAndIndexPreserved: true,
humanApprovals: 0, group, job, check: checked, delivery, modelRequests: model.requests.map(({ headers, ...value }) => value) };
console.log(JSON.stringify({ name, success: true, ownedHttpRequests: requests, fixtureReportedTokens: 12000068, commit: delivery.commit }));
} catch (error) {
result = { success: false, enabled, error: error.stack, group, realModelRequests: 0 };
process.exitCode = 1;
console.log(JSON.stringify({ name, success: false, error: error.message.split('\n')[0] }));
} finally {
clearTimeout(timeout);
await runtime?.stopAll();
f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2));
fs.cpSync(f.root, path.join(output, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
for (const dispose of disposers.reverse()) await dispose();
}
results.push({ name, success: result.success, error: result.error ?? null });
if (!result.success) break;
}
const result = { success: results.length === 2 && results.every(row => row.success), directory, results,
realModelRequests: 0, scope: 'Actual pinned native engines with local synthetic HTTP only; no autonomous reasoning or Purge benefit claim.' };
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
console.log(JSON.stringify(result));