Files
Agent-Workbench/scripts/live-af/run.mjs
T
2026-11-06 13:53:53 +08:00

222 lines
20 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { DatabaseSync } from 'node:sqlite';
import { setTimeout as delay } from 'node:timers/promises';
import { purgeEvidence, root, read, save, client, sha, captureSession, connectionDefinition, connectionInput, compareConfig } from '../live-ad/common.mjs';
import { prepareNode } from './project.mjs';
import { order, allowed, goal, purgePrompt, observations as limits } from './contract.mjs';
import { ORIGINAL_PURGE_KIND, assertPurgeModelInput } from '../../src/core/engines/purge-strategy.mjs';
import { resolveControlledInstallation } from '../../src/core/engines/controlled-installation.mjs';
import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs';
import { requestMetrics, requestVariants } from '../../src/shared/contracts/model-request.mjs';
purgeEvidence(); const frozen = read('current.json');
assert.equal(fs.existsSync(path.join(root, 'batch-start.json')), false, 'No replay or partial batch continuation.');
const { api, boot, close } = await client(); let original, configured, selected = false, currentRoute, results = [], finalFailure;
const owned = [];
const settle = async () => { if (currentRoute) await api(currentRoute + '/stop', {}, 120000); };
async function waitParent(route, active, deadline = Date.now() + 120000) {
for (;;) {
const state = await api(route), group = state.execution.collaborations.at(-1);
if (!active.includes(group.status)) return { state, group };
assert.ok(Date.now() < deadline, 'Owned parent operation observation ended.'); await delay(750);
}
}
async function artifact(sessionId, id) {
let content = '', page;
do { page = await api('/artifacts/' + id + '?session=' + sessionId + '&offset=' + content.length + '&limit=16000'); content += page.content; } while (page.truncated);
assert.equal(sha(content), page.sha256); return content;
}
function engineJobs(sessionId) {
const db = new DatabaseSync(path.join(frozen.host.identity.dataRoot, 'workbench.sqlite'), { readOnly: true });
try { return db.prepare("SELECT payload FROM records WHERE kind='engine_run' AND session_id=?").all(sessionId).map(row => JSON.parse(row.payload)); }
finally { db.close(); }
}
async function captureEngine(sessionId, prefix) {
const jobs = engineJobs(sessionId); save(prefix + '/engine-jobs.json', jobs);
const proof = [];
for (const job of jobs) {
const base = prefix + '/engines/' + job.id;
const input = await artifact(sessionId, job.inputArtifactId); assert.equal(sha(input), job.inputSha256); save(base + '/input.json', input);
if (job.reportArtifactId) {
const report = await api('/sessions/' + sessionId + '/engine-runs/' + job.id + '/download');
assert.equal(sha(JSON.stringify(report)), job.reportSha256); save(base + '/report.json', JSON.stringify(report));
}
if (job.seedContext) save(base + '/seed.json', await artifact(sessionId, job.seedContext.artifactId));
for (const request of job.requests) {
const text = await artifact(sessionId, request.inputArtifactId); assert.equal(sha(text), request.inputSha256);
const input = JSON.parse(text); assertPurgeModelInput(input.native, job.installation.strategy);
const metrics = requestMetrics(input.body); assert.equal(metrics.sha256, request.requestSha256); assert.equal(metrics.bytes, request.requestBytes);
assert.equal(input.view.mode, 'full'); assert.equal(input.view.inputBudget, null);
const variants = requestVariants(input.body);
assert.equal(variants.length, 1);
assert.ok(variants.every(body => ['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key))));
save(base + '/model-' + request.id + '.json', text);
if (input.view.sourceArtifactId) save(base + '/source-' + request.id + '.json', await artifact(sessionId, input.view.sourceArtifactId));
proof.push({ jobId: job.id, requestId: request.id, requestSha256: metrics.sha256,
variants: variants.map(body => ({ bodySha256: sha(JSON.stringify(body)), toolSha256: sha(JSON.stringify(body.tools)), toolCount: body.tools.length })),
nativeSha256: request.sourceSha256, fullHistory: true, outputTokenCap: false });
}
for (const call of job.calls) for (const type of ['input', 'output']) {
if (call[type + 'ArtifactId']) save(base + '/tool-' + sha(call.sourceCallId) + '-' + type + '.json', await artifact(sessionId, call[type + 'ArtifactId']));
}
}
save(prefix + '/engine-proof.json', proof); return { jobs, proof };
}
try {
const host = await api('/host/status'), center = await api('/model-center');
assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0); assert.equal(boot.version, '1.25.8');
compareConfig(frozen.config, boot.config); assert.deepEqual(center.selected, frozen.center.selected); assert.deepEqual(center.pools, frozen.center.pools);
original = center.connections.find(row => row.id === frozen.connection.id); assert.deepEqual(connectionDefinition(original), connectionDefinition(frozen.connection));
save('batch-start.json', { order, goal, purgePrompt, limits, config: boot.config, connection: original, localCapsChanged: false, startedAt: new Date().toISOString() });
configured = await api('/model-center/connections', { ...connectionInput(original), modelIdentityPolicy: 'require-match' });
assert.equal(configured.hasKey, original.hasKey); assert.equal(configured.credentialRevision, original.credentialRevision);
await api('/model-center/select', { type: 'connection', id: original.id }); selected = true;
save('active-model.json', { connection: configured, config: (await api('/bootstrap')).config });
let baseHead;
for (const sample of order) {
const prefix = sample.id; assert.equal(fs.existsSync(path.join(root, prefix)), false);
const store = (file, value) => save(prefix + '/' + file, value), startedAt = Date.now();
let own, session, worker, group, workerState, failure, result, route;
try {
const installation = frozen.availabilityAuthority === 'host-api'
? await api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(frozen.installation.root))
: resolveControlledInstallation(process.cwd(), frozen.installation.root);
const source = frozen.availabilityAuthority === 'host-api'
? await api('/engines/purge/source?kind=' + ORIGINAL_PURGE_KIND + '&root=' + encodeURIComponent(frozen.source.root) + '&harnessRoot=' + encodeURIComponent(installation.root))
: resolveOriginalPurgeSource(process.cwd(), frozen.source.root, installation);
assert.equal(installation.controlledSha256, frozen.installation.controlledSha256);
assert.equal(source.sourceSha256, frozen.source.sourceSha256); assert.equal(source.runtimeSha256, frozen.source.runtimeSha256);
const current = await api('/model-center');
const currentHost = await api('/host/status'), currentBoot = await api('/bootstrap');
assert.equal(currentHost.identity.pid, frozen.host.identity.pid); assert.equal(currentHost.identity.instanceId, frozen.host.identity.instanceId);
assert.equal(currentHost.identity.runtimeRoot, frozen.host.identity.runtimeRoot); assert.equal(currentBoot.version, '1.25.8');
compareConfig(read('active-model.json').config, currentBoot.config);
assert.deepEqual(connectionDefinition(current.connections.find(row => row.id === original.id)), connectionDefinition(configured));
assert.deepEqual(current.selected, { type: 'connection', id: original.id });
own = prepareNode(path.join(root, prefix), store);
baseHead ??= own.initialHead; assert.equal(own.initialHead, baseHead, 'Every comparison starts with identical original Git content.');
const before = own.checks('baseline'); assert.equal(before.test.stats.tests, 9); assert.equal(before.test.stats.fail, 9); assert.equal(before.build.status, 0);
const project = await api('/projects', { root: own.sourceRoot }); session = await api('/sessions', { projectId: project.id });
route = '/sessions/' + session.id; currentRoute = route; owned.push({ projectId: project.id, sessionId: session.id });
store('owned-parent.json', { project, session });
const planned = await api(route + '/collaborations', { goal, engine: { id: 'deepseek-harness', root: installation.root,
controlledSha256: installation.controlledSha256, ...(sample.enabled ? { strategy: { kind: ORIGINAL_PURGE_KIND,
root: source.root, sourceSha256: source.sourceSha256, runtimeSha256: source.runtimeSha256, prompt: purgePrompt } } : {}) },
children: [{ label: 'expense-candidate', goal, paths: allowed }] });
group = planned.group; store('group-planned.json', group);
assert.deepEqual(group.budget, { maxTokens: 0, maxSteps: 0, maxToolCalls: 0, maxDurationMs: 0 });
const groupRoute = route + '/collaborations/' + group.id, fingerprint = group.fingerprint;
await api(groupRoute + '/prepare', { fingerprint }, 120000);
group = (await waitParent(route, ['preparing'])).group; assert.equal(group.status, 'ready', group.summary);
worker = group.children[0]; owned.push({ projectId: worker.projectId, sessionId: worker.sessionId }); store('owned-worker.json', worker);
workerState = await api('/sessions/' + worker.sessionId); store('worker-prepared.json', workerState);
const workerProject = (await api('/bootstrap')).projects.find(row => row.id === worker.projectId); assert.ok(workerProject);
store('worker-project.json', workerProject);
await api(groupRoute + '/start', { fingerprint }, 120000);
const runningAt = Date.now(); let novelAt = runningAt, signature = '', previous = '', latestMessageAt = 0;
const observed = [];
for (;;) {
const parent = await api(route); group = parent.execution.collaborations.at(-1);
workerState = await api('/sessions/' + worker.sessionId); own.protect();
const changes = allowed.map(name => sha(fs.readFileSync(path.join(workerProject.root, name))));
const completedTools = workerState.events.filter(row => row.kind === 'tool' && row.phase === 'completed');
const facts = completedTools.filter(row => ['read_file', 'read_artifact', 'list_files', 'search_files', 'query_code'].includes(row.tool))
.map(row => row.tool + JSON.stringify(row.input) + sha(row.output ?? ''));
const next = JSON.stringify({ changes, facts: [...new Set(facts)].sort(), checks: workerState.execution.checks.map(row => ({ status: row.status, source: row.sourceFingerprint })) });
if (signature !== next) { signature = next; novelAt = Date.now(); }
const observation = { sample: prefix, group: group.status, worker: workerState.status,
requests: workerState.execution.task?.usage.requests, tokens: workerState.execution.task?.usage.totalTokens,
changes: workerState.changes.length, checks: workerState.execution.checks.map(row => row.status),
engine: workerState.execution.engineRuns?.at(-1)?.status, lastTool: completedTools.at(-1)?.tool, elapsedMs: Date.now() - runningAt };
const key = JSON.stringify({ ...observation, elapsedMs: 0 });
if (key !== previous || Date.now() - latestMessageAt > 45000) {
observed.push(observation); console.log(JSON.stringify(observation)); previous = key; latestMessageAt = Date.now();
}
if (!['starting', 'running', 'stopping'].includes(group.status)) break;
if (Date.now() - runningAt > limits.maximumMs || Date.now() - novelAt > limits.noNewFactMs) {
failure = { reason: 'experiment_observation_ended', elapsedMs: Date.now() - runningAt, noNewFactMs: Date.now() - novelAt, productCapsChanged: false };
await settle(); break;
}
await delay(1000);
}
store('observations.json', observed); group = (await api(route)).execution.collaborations.at(-1);
if (!failure && group.status === 'ready_to_collect') {
await api(groupRoute + '/collect', { fingerprint }, 120000); group = (await waitParent(route, ['collecting'])).group;
assert.equal(group.status, 'collected', group.summary);
await api(groupRoute + '/merge', { fingerprint }, 120000); group = (await waitParent(route, ['merge_preparing'])).group;
assert.equal(group.status, 'merge_prepared', group.summary);
const checkState = await api(route + '/check-plans', { names: ['test', 'build'] }); const plan = checkState.execution.plans.at(-1);
await api(route + '/check-jobs', { planId: plan.id });
let state; do { state = await api(route); if (['queued', 'running'].includes(state.execution.checks.at(-1).status)) await delay(500); else break; } while (Date.now() - startedAt < 900000);
assert.equal(state.execution.checks.at(-1).status, 'passed');
const deliveryState = await api(route + '/delivery-plans', { message: 'Complete frozen AF expense engineering' });
await api(route + '/delivery-jobs', { planId: deliveryState.execution.deliveryPlans.at(-1).id });
do { state = await api(route); if (['queued', 'preparing', 'prepared', 'publishing', 'committed', 'reporting'].includes(state.execution.deliveries.at(-1).status)) await delay(500); else break; } while (Date.now() - startedAt < 900000);
assert.equal(state.execution.deliveries.at(-1).status, 'completed');
await api(groupRoute + '/finalize', { fingerprint }, 120000); group = (await api(route)).execution.collaborations.at(-1);
assert.equal(group.status, 'completed');
} else failure ??= { reason: 'model_or_engine_stopped', groupStatus: group.status, summary: group.summary };
const final = own.checks('independent-final'), commit = own.git(['rev-parse', 'HEAD']).trim();
const committedPaths = own.git(['diff', '--name-only', own.initialHead, commit]).trim().split(/\r?\n/).filter(Boolean).sort();
const committedCurrent = allowed.every(name => sha(Buffer.from(own.git(['show', commit + ':' + name]))) === sha(fs.readFileSync(path.join(own.sourceRoot, name))));
result = { sample: prefix, enabled: sample.enabled, success: !failure && group.status === 'completed' && final.test.stats.pass === 9
&& final.test.stats.fail === 0 && final.build.status === 0 && commit !== own.initialHead && committedCurrent
&& JSON.stringify(committedPaths) === JSON.stringify(allowed.slice().sort()),
failure: failure ?? null, final, commit, initialHead: own.initialHead, committedPaths, committedCurrent,
parentSessionId: session.id, workerSessionId: worker.sessionId, ownerBytesAndIndexPreserved: true,
supervisorApprovals: 0, supervisorContinuations: 0, automaticParentCollectionAndChecks: true };
} catch (error) {
result = { sample: prefix, enabled: sample.enabled, success: false, failure: { error: error.stack, retry: false }, parentSessionId: session?.id, workerSessionId: worker?.sessionId };
} finally {
const cleanupErrors = [];
try { await settle(); } catch (error) { cleanupErrors.push({ phase: 'settle', error: error.stack }); }
if (worker) {
try {
const captured = await captureSession(api, worker.sessionId, prefix + '/worker'), engine = await captureEngine(worker.sessionId, prefix + '/worker');
const identities = captured.rows.length > 0 && captured.rows.every(row => row.status === 'completed' && row.connectionId === original.id
&& row.modelIdentity?.complete && row.modelIdentity.status === 'matched');
const nativePassed = engine.jobs.length === 1 && engine.jobs.every(job => job.status === 'completed' && !job.requiresVerification && job.verified
&& job.native?.cleanup?.rootExited && job.native?.cleanup?.treeConfirmed && (!sample.enabled
? !job.native.originalPurge : job.native.originalPurge?.readiness.originalEntryLoaded && job.native.originalPurge?.readiness.originalApplyCalled));
result = { ...result, success: result.success && identities && nativePassed && captured.state.execution.task.usage.unknownRequests === 0,
identitiesPassed: identities, nativePassed, requests: captured.rows.length, usage: captured.state.execution.task.usage,
reportedCachedInputTokens: captured.rows.reduce((sum, row) => sum + (row.usage?.prompt_tokens_details?.cached_tokens ?? 0), 0),
elapsedMs: Date.now() - startedAt, engineElapsedMs: engine.jobs.reduce((sum, job) => sum + Date.parse(job.finishedAt) - Date.parse(job.startedAt), 0),
modelIdentityPolicy: 'require-match', captures: captured.captures, engineRequestProof: engine.proof };
} catch (error) { cleanupErrors.push({ phase: 'capture-worker', error: error.stack }); }
}
if (session) try { await captureSession(api, session.id, prefix + '/parent'); } catch (error) { cleanupErrors.push({ phase: 'capture-parent', error: error.stack }); }
if (own) { try { own.protect(); store('git-status.txt', own.git(['status', '--porcelain=v1'])); store('git-log.txt', own.git(['log', '--stat', own.initialHead + '..HEAD'])); } catch (error) { cleanupErrors.push({ phase: 'owner', error: error.stack }); } }
if (cleanupErrors.length) result = { ...result, success: false, cleanupErrors };
store('result.json', result); save('owned-' + prefix + '.json', owned); currentRoute = null; results.push(result);
console.log(JSON.stringify({ sample: prefix, success: result.success, requests: result.requests, tokens: result.usage?.totalTokens, failure: result.failure, cleanupErrors: result.cleanupErrors }));
if (cleanupErrors.length || result.failure?.error) { finalFailure = 'Driver or capture failed; later samples not dispatched.'; break; }
}
}
} catch (error) { finalFailure = error.stack; process.exitCode = 1; }
finally {
const errors = [], restoration = {};
try { await settle(); } catch (error) { errors.push({ phase: 'settle', error: error.stack }); }
try {
if (configured) {
const center = await api('/model-center'), current = center.connections.find(row => row.id === original.id);
if (current.revision === configured.revision) restoration.connection = await api('/model-center/connections', { ...connectionInput(original), revision: current.revision,
modelIdentityPolicy: original.modelIdentityPolicy ?? 'record', modelIdentityAliases: original.modelIdentityAliases ?? [], modelIdentityAliasSource: original.modelIdentityAliasSource ?? '' });
else { restoration.externalConnectionChangePreserved = true; errors.push({ phase: 'connection', error: 'External connection revision changed; preserved.' }); }
}
if (selected) {
const center = await api('/model-center');
if (center.selected.type === 'connection' && center.selected.id === original.id) { await api('/model-center/select', frozen.center.selected); restoration.selection = frozen.center.selected; }
else { restoration.externalSelectionChangePreserved = true; errors.push({ phase: 'selection', error: 'External selection changed; preserved.' }); }
}
restoration.config = (await api('/bootstrap')).config; restoration.comparison = compareConfig(frozen.config, restoration.config);
} catch (error) { errors.push({ phase: 'restore', error: error.stack }); }
restoration.keysReadOrResaved = false; restoration.errors = errors; save('restoration.json', restoration);
save('owned.json', owned); save('result.json', { success: results.length === 4 && results.every(row => row.success) && !errors.length && !finalFailure,
samples: results, finalFailure: finalFailure ?? null, restorationErrors: errors, purgeBenefit: 'not yet assessed', historicalSamplesReplayed: false });
await close();
}
if (finalFailure) console.log(JSON.stringify({ finalFailure }));