218 lines
19 KiB
JavaScript
218 lines
19 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { purgeEvidence, root, read, save, client, sha, captureSession, connectionDefinition, connectionInput, compareConfig } from '../live-ad/common.mjs';
|
|
import { prepareNode } from './project.mjs';
|
|
import { order, allowed, goal, purgePrompt, observations as limits } from './contract.mjs';
|
|
import { ORIGINAL_PURGE_KIND, assertPurgeModelInput } from '../../src/core/engines/purge-strategy.mjs';
|
|
import { resolveControlledInstallation } from '../../src/core/engines/controlled-installation.mjs';
|
|
import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs';
|
|
import { requestMetrics, requestVariants } from '../../src/shared/contracts/model-request.mjs';
|
|
|
|
purgeEvidence(); const frozen = read('current.json');
|
|
assert.equal(fs.existsSync(path.join(root, 'batch-start.json')), false, 'No replay or partial batch continuation.');
|
|
const { api, boot, close } = await client(); let original, configured, selected = false, currentRoute, results = [], finalFailure;
|
|
const owned = [];
|
|
const settle = async () => { if (currentRoute) await api(currentRoute + '/stop', {}, 120000); };
|
|
async function waitParent(route, active, deadline = Date.now() + 120000) {
|
|
for (;;) {
|
|
const state = await api(route), group = state.execution.collaborations.at(-1);
|
|
if (!active.includes(group.status)) return { state, group };
|
|
assert.ok(Date.now() < deadline, 'Owned parent operation observation ended.'); await delay(750);
|
|
}
|
|
}
|
|
async function artifact(sessionId, id) {
|
|
let content = '', page;
|
|
do { page = await api('/artifacts/' + id + '?session=' + sessionId + '&offset=' + content.length + '&limit=16000'); content += page.content; } while (page.truncated);
|
|
assert.equal(sha(content), page.sha256); return content;
|
|
}
|
|
function engineJobs(sessionId) {
|
|
const db = new DatabaseSync(path.join(frozen.host.identity.dataRoot, 'workbench.sqlite'), { readOnly: true });
|
|
try { return db.prepare("SELECT payload FROM records WHERE kind='engine_run' AND session_id=?").all(sessionId).map(row => JSON.parse(row.payload)); }
|
|
finally { db.close(); }
|
|
}
|
|
async function captureEngine(sessionId, prefix) {
|
|
const jobs = engineJobs(sessionId); save(prefix + '/engine-jobs.json', jobs);
|
|
const proof = [];
|
|
for (const job of jobs) {
|
|
const base = prefix + '/engines/' + job.id;
|
|
const input = await artifact(sessionId, job.inputArtifactId); assert.equal(sha(input), job.inputSha256); save(base + '/input.json', input);
|
|
if (job.reportArtifactId) {
|
|
const report = await api('/sessions/' + sessionId + '/engine-runs/' + job.id + '/download');
|
|
assert.equal(sha(JSON.stringify(report)), job.reportSha256); save(base + '/report.json', JSON.stringify(report));
|
|
}
|
|
if (job.seedContext) save(base + '/seed.json', await artifact(sessionId, job.seedContext.artifactId));
|
|
for (const request of job.requests) {
|
|
const text = await artifact(sessionId, request.inputArtifactId); assert.equal(sha(text), request.inputSha256);
|
|
const input = JSON.parse(text); assertPurgeModelInput(input.native, job.installation.strategy);
|
|
const metrics = requestMetrics(input.body); assert.equal(metrics.sha256, request.requestSha256); assert.equal(metrics.bytes, request.requestBytes);
|
|
assert.equal(input.view.mode, 'full'); assert.equal(input.view.inputBudget, null);
|
|
const variants = requestVariants(input.body);
|
|
assert.equal(variants.length, 1);
|
|
assert.ok(variants.every(body => ['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key))));
|
|
save(base + '/model-' + request.id + '.json', text);
|
|
if (input.view.sourceArtifactId) save(base + '/source-' + request.id + '.json', await artifact(sessionId, input.view.sourceArtifactId));
|
|
proof.push({ jobId: job.id, requestId: request.id, requestSha256: metrics.sha256,
|
|
variants: variants.map(body => ({ bodySha256: sha(JSON.stringify(body)), toolSha256: sha(JSON.stringify(body.tools)), toolCount: body.tools.length })),
|
|
nativeSha256: request.sourceSha256, fullHistory: true, outputTokenCap: false });
|
|
}
|
|
for (const call of job.calls) for (const type of ['input', 'output']) {
|
|
if (call[type + 'ArtifactId']) save(base + '/tool-' + sha(call.sourceCallId) + '-' + type + '.json', await artifact(sessionId, call[type + 'ArtifactId']));
|
|
}
|
|
}
|
|
save(prefix + '/engine-proof.json', proof); return { jobs, proof };
|
|
}
|
|
try {
|
|
const host = await api('/host/status'), center = await api('/model-center');
|
|
assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0); assert.equal(boot.version, '1.25.8');
|
|
compareConfig(frozen.config, boot.config); assert.deepEqual(center.selected, frozen.center.selected); assert.deepEqual(center.pools, frozen.center.pools);
|
|
original = center.connections.find(row => row.id === frozen.connection.id); assert.deepEqual(connectionDefinition(original), connectionDefinition(frozen.connection));
|
|
save('batch-start.json', { order, goal, purgePrompt, limits, config: boot.config, connection: original, localCapsChanged: false, startedAt: new Date().toISOString() });
|
|
configured = await api('/model-center/connections', { ...connectionInput(original), modelIdentityPolicy: 'require-match' });
|
|
assert.equal(configured.hasKey, original.hasKey); assert.equal(configured.credentialRevision, original.credentialRevision);
|
|
await api('/model-center/select', { type: 'connection', id: original.id }); selected = true;
|
|
save('active-model.json', { connection: configured, config: (await api('/bootstrap')).config });
|
|
let baseHead;
|
|
for (const sample of order) {
|
|
const prefix = sample.id; assert.equal(fs.existsSync(path.join(root, prefix)), false);
|
|
const store = (file, value) => save(prefix + '/' + file, value), startedAt = Date.now();
|
|
let own, session, worker, group, workerState, failure, result, route;
|
|
try {
|
|
const installation = resolveControlledInstallation(process.cwd(), frozen.installation.root);
|
|
const source = resolveOriginalPurgeSource(process.cwd(), frozen.source.root, installation);
|
|
assert.equal(installation.controlledSha256, frozen.installation.controlledSha256);
|
|
assert.equal(source.sourceSha256, frozen.source.sourceSha256); assert.equal(source.runtimeSha256, frozen.source.runtimeSha256);
|
|
const current = await api('/model-center');
|
|
const currentHost = await api('/host/status'), currentBoot = await api('/bootstrap');
|
|
assert.equal(currentHost.identity.pid, frozen.host.identity.pid); assert.equal(currentHost.identity.instanceId, frozen.host.identity.instanceId);
|
|
assert.equal(currentHost.identity.runtimeRoot, frozen.host.identity.runtimeRoot); assert.equal(currentBoot.version, '1.25.8');
|
|
compareConfig(read('active-model.json').config, currentBoot.config);
|
|
assert.deepEqual(connectionDefinition(current.connections.find(row => row.id === original.id)), connectionDefinition(configured));
|
|
assert.deepEqual(current.selected, { type: 'connection', id: original.id });
|
|
own = prepareNode(path.join(root, prefix), store);
|
|
baseHead ??= own.initialHead; assert.equal(own.initialHead, baseHead, 'Every comparison starts with identical original Git content.');
|
|
const before = own.checks('baseline'); assert.equal(before.test.stats.tests, 9); assert.equal(before.test.stats.fail, 9); assert.equal(before.build.status, 0);
|
|
const project = await api('/projects', { root: own.sourceRoot }); session = await api('/sessions', { projectId: project.id });
|
|
route = '/sessions/' + session.id; currentRoute = route; owned.push({ projectId: project.id, sessionId: session.id });
|
|
store('owned-parent.json', { project, session });
|
|
const planned = await api(route + '/collaborations', { goal, engine: { id: 'deepseek-harness', root: installation.root,
|
|
controlledSha256: installation.controlledSha256, ...(sample.enabled ? { strategy: { kind: ORIGINAL_PURGE_KIND,
|
|
root: source.root, sourceSha256: source.sourceSha256, runtimeSha256: source.runtimeSha256, prompt: purgePrompt } } : {}) },
|
|
children: [{ label: 'expense-candidate', goal, paths: allowed }] });
|
|
group = planned.group; store('group-planned.json', group);
|
|
assert.deepEqual(group.budget, { maxTokens: 0, maxSteps: 0, maxToolCalls: 0, maxDurationMs: 0 });
|
|
const groupRoute = route + '/collaborations/' + group.id, fingerprint = group.fingerprint;
|
|
await api(groupRoute + '/prepare', { fingerprint }, 120000);
|
|
group = (await waitParent(route, ['preparing'])).group; assert.equal(group.status, 'ready', group.summary);
|
|
worker = group.children[0]; owned.push({ projectId: worker.projectId, sessionId: worker.sessionId }); store('owned-worker.json', worker);
|
|
workerState = await api('/sessions/' + worker.sessionId); store('worker-prepared.json', workerState);
|
|
const workerProject = (await api('/bootstrap')).projects.find(row => row.id === worker.projectId); assert.ok(workerProject);
|
|
store('worker-project.json', workerProject);
|
|
await api(groupRoute + '/start', { fingerprint }, 120000);
|
|
const runningAt = Date.now(); let novelAt = runningAt, signature = '', previous = '', latestMessageAt = 0;
|
|
const observed = [];
|
|
for (;;) {
|
|
const parent = await api(route); group = parent.execution.collaborations.at(-1);
|
|
workerState = await api('/sessions/' + worker.sessionId); own.protect();
|
|
const changes = allowed.map(name => sha(fs.readFileSync(path.join(workerProject.root, name))));
|
|
const completedTools = workerState.events.filter(row => row.kind === 'tool' && row.phase === 'completed');
|
|
const facts = completedTools.filter(row => ['read_file', 'read_artifact', 'list_files', 'search_files', 'query_code'].includes(row.tool))
|
|
.map(row => row.tool + JSON.stringify(row.input) + sha(row.output ?? ''));
|
|
const next = JSON.stringify({ changes, facts: [...new Set(facts)].sort(), checks: workerState.execution.checks.map(row => ({ status: row.status, source: row.sourceFingerprint })) });
|
|
if (signature !== next) { signature = next; novelAt = Date.now(); }
|
|
const observation = { sample: prefix, group: group.status, worker: workerState.status,
|
|
requests: workerState.execution.task?.usage.requests, tokens: workerState.execution.task?.usage.totalTokens,
|
|
changes: workerState.changes.length, checks: workerState.execution.checks.map(row => row.status),
|
|
engine: workerState.execution.engineRuns?.at(-1)?.status, lastTool: completedTools.at(-1)?.tool, elapsedMs: Date.now() - runningAt };
|
|
const key = JSON.stringify({ ...observation, elapsedMs: 0 });
|
|
if (key !== previous || Date.now() - latestMessageAt > 45000) {
|
|
observed.push(observation); console.log(JSON.stringify(observation)); previous = key; latestMessageAt = Date.now();
|
|
}
|
|
if (!['starting', 'running', 'stopping'].includes(group.status)) break;
|
|
if (Date.now() - runningAt > limits.maximumMs || Date.now() - novelAt > limits.noNewFactMs) {
|
|
failure = { reason: 'experiment_observation_ended', elapsedMs: Date.now() - runningAt, noNewFactMs: Date.now() - novelAt, productCapsChanged: false };
|
|
await settle(); break;
|
|
}
|
|
await delay(1000);
|
|
}
|
|
store('observations.json', observed); group = (await api(route)).execution.collaborations.at(-1);
|
|
if (!failure && group.status === 'ready_to_collect') {
|
|
await api(groupRoute + '/collect', { fingerprint }, 120000); group = (await waitParent(route, ['collecting'])).group;
|
|
assert.equal(group.status, 'collected', group.summary);
|
|
await api(groupRoute + '/merge', { fingerprint }, 120000); group = (await waitParent(route, ['merge_preparing'])).group;
|
|
assert.equal(group.status, 'merge_prepared', group.summary);
|
|
const checkState = await api(route + '/check-plans', { names: ['test', 'build'] }); const plan = checkState.execution.plans.at(-1);
|
|
await api(route + '/check-jobs', { planId: plan.id });
|
|
let state; do { state = await api(route); if (['queued', 'running'].includes(state.execution.checks.at(-1).status)) await delay(500); else break; } while (Date.now() - startedAt < 900000);
|
|
assert.equal(state.execution.checks.at(-1).status, 'passed');
|
|
const deliveryState = await api(route + '/delivery-plans', { message: 'Complete frozen AF expense engineering' });
|
|
await api(route + '/delivery-jobs', { planId: deliveryState.execution.deliveryPlans.at(-1).id });
|
|
do { state = await api(route); if (['queued', 'preparing', 'prepared', 'publishing', 'committed', 'reporting'].includes(state.execution.deliveries.at(-1).status)) await delay(500); else break; } while (Date.now() - startedAt < 900000);
|
|
assert.equal(state.execution.deliveries.at(-1).status, 'completed');
|
|
await api(groupRoute + '/finalize', { fingerprint }, 120000); group = (await api(route)).execution.collaborations.at(-1);
|
|
assert.equal(group.status, 'completed');
|
|
} else failure ??= { reason: 'model_or_engine_stopped', groupStatus: group.status, summary: group.summary };
|
|
const final = own.checks('independent-final'), commit = own.git(['rev-parse', 'HEAD']).trim();
|
|
const committedPaths = own.git(['diff', '--name-only', own.initialHead, commit]).trim().split(/\r?\n/).filter(Boolean).sort();
|
|
const committedCurrent = allowed.every(name => sha(Buffer.from(own.git(['show', commit + ':' + name]))) === sha(fs.readFileSync(path.join(own.sourceRoot, name))));
|
|
result = { sample: prefix, enabled: sample.enabled, success: !failure && group.status === 'completed' && final.test.stats.pass === 9
|
|
&& final.test.stats.fail === 0 && final.build.status === 0 && commit !== own.initialHead && committedCurrent
|
|
&& JSON.stringify(committedPaths) === JSON.stringify(allowed.slice().sort()),
|
|
failure: failure ?? null, final, commit, initialHead: own.initialHead, committedPaths, committedCurrent,
|
|
parentSessionId: session.id, workerSessionId: worker.sessionId, ownerBytesAndIndexPreserved: true,
|
|
supervisorApprovals: 0, supervisorContinuations: 0, automaticParentCollectionAndChecks: true };
|
|
} catch (error) {
|
|
result = { sample: prefix, enabled: sample.enabled, success: false, failure: { error: error.stack, retry: false }, parentSessionId: session?.id, workerSessionId: worker?.sessionId };
|
|
} finally {
|
|
const cleanupErrors = [];
|
|
try { await settle(); } catch (error) { cleanupErrors.push({ phase: 'settle', error: error.stack }); }
|
|
if (worker) {
|
|
try {
|
|
const captured = await captureSession(api, worker.sessionId, prefix + '/worker'), engine = await captureEngine(worker.sessionId, prefix + '/worker');
|
|
const identities = captured.rows.length > 0 && captured.rows.every(row => row.status === 'completed' && row.connectionId === original.id
|
|
&& row.modelIdentity?.complete && row.modelIdentity.status === 'matched');
|
|
const nativePassed = engine.jobs.length === 1 && engine.jobs.every(job => job.status === 'completed' && !job.requiresVerification && job.verified
|
|
&& job.native?.cleanup?.rootExited && job.native?.cleanup?.treeConfirmed && (!sample.enabled
|
|
? !job.native.originalPurge : job.native.originalPurge?.readiness.originalEntryLoaded && job.native.originalPurge?.readiness.originalApplyCalled));
|
|
result = { ...result, success: result.success && identities && nativePassed && captured.state.execution.task.usage.unknownRequests === 0,
|
|
identitiesPassed: identities, nativePassed, requests: captured.rows.length, usage: captured.state.execution.task.usage,
|
|
reportedCachedInputTokens: captured.rows.reduce((sum, row) => sum + (row.usage?.prompt_tokens_details?.cached_tokens ?? 0), 0),
|
|
elapsedMs: Date.now() - startedAt, engineElapsedMs: engine.jobs.reduce((sum, job) => sum + Date.parse(job.finishedAt) - Date.parse(job.startedAt), 0),
|
|
modelIdentityPolicy: 'require-match', captures: captured.captures, engineRequestProof: engine.proof };
|
|
} catch (error) { cleanupErrors.push({ phase: 'capture-worker', error: error.stack }); }
|
|
}
|
|
if (session) try { await captureSession(api, session.id, prefix + '/parent'); } catch (error) { cleanupErrors.push({ phase: 'capture-parent', error: error.stack }); }
|
|
if (own) { try { own.protect(); store('git-status.txt', own.git(['status', '--porcelain=v1'])); store('git-log.txt', own.git(['log', '--stat', own.initialHead + '..HEAD'])); } catch (error) { cleanupErrors.push({ phase: 'owner', error: error.stack }); } }
|
|
if (cleanupErrors.length) result = { ...result, success: false, cleanupErrors };
|
|
store('result.json', result); save('owned-' + prefix + '.json', owned); currentRoute = null; results.push(result);
|
|
console.log(JSON.stringify({ sample: prefix, success: result.success, requests: result.requests, tokens: result.usage?.totalTokens, failure: result.failure, cleanupErrors: result.cleanupErrors }));
|
|
if (cleanupErrors.length || result.failure?.error) { finalFailure = 'Driver or capture failed; later samples not dispatched.'; break; }
|
|
}
|
|
}
|
|
} catch (error) { finalFailure = error.stack; process.exitCode = 1; }
|
|
finally {
|
|
const errors = [], restoration = {};
|
|
try { await settle(); } catch (error) { errors.push({ phase: 'settle', error: error.stack }); }
|
|
try {
|
|
if (configured) {
|
|
const center = await api('/model-center'), current = center.connections.find(row => row.id === original.id);
|
|
if (current.revision === configured.revision) restoration.connection = await api('/model-center/connections', { ...connectionInput(original), revision: current.revision,
|
|
modelIdentityPolicy: original.modelIdentityPolicy ?? 'record', modelIdentityAliases: original.modelIdentityAliases ?? [], modelIdentityAliasSource: original.modelIdentityAliasSource ?? '' });
|
|
else { restoration.externalConnectionChangePreserved = true; errors.push({ phase: 'connection', error: 'External connection revision changed; preserved.' }); }
|
|
}
|
|
if (selected) {
|
|
const center = await api('/model-center');
|
|
if (center.selected.type === 'connection' && center.selected.id === original.id) { await api('/model-center/select', frozen.center.selected); restoration.selection = frozen.center.selected; }
|
|
else { restoration.externalSelectionChangePreserved = true; errors.push({ phase: 'selection', error: 'External selection changed; preserved.' }); }
|
|
}
|
|
restoration.config = (await api('/bootstrap')).config; restoration.comparison = compareConfig(frozen.config, restoration.config);
|
|
} catch (error) { errors.push({ phase: 'restore', error: error.stack }); }
|
|
restoration.keysReadOrResaved = false; restoration.errors = errors; save('restoration.json', restoration);
|
|
save('owned.json', owned); save('result.json', { success: results.length === 4 && results.every(row => row.success) && !errors.length && !finalFailure,
|
|
samples: results, finalFailure: finalFailure ?? null, restorationErrors: errors, purgeBenefit: 'not yet assessed', historicalSamplesReplayed: false });
|
|
await close();
|
|
}
|
|
if (finalFailure) console.log(JSON.stringify({ finalFailure }));
|