import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { DatabaseSync } from 'node:sqlite'; import { setTimeout as delay } from 'node:timers/promises'; import { purgeEvidence, root, read, save, client, sha, captureSession, connectionDefinition, connectionInput, compareConfig } from '../live-ad/common.mjs'; import { prepareNode } from './project.mjs'; import { order, allowed, goal, purgePrompt, observations as limits } from './contract.mjs'; import { ORIGINAL_PURGE_KIND, assertPurgeModelInput } from '../../src/core/engines/purge-strategy.mjs'; import { resolveControlledInstallation } from '../../src/core/engines/controlled-installation.mjs'; import { resolveOriginalPurgeSource } from '../../src/core/engines/purge-strategy.mjs'; import { requestMetrics, requestVariants } from '../../src/shared/contracts/model-request.mjs'; purgeEvidence(); const frozen = read('current.json'); assert.equal(fs.existsSync(path.join(root, 'batch-start.json')), false, 'No replay or partial batch continuation.'); const { api, boot, close } = await client(); let original, configured, selected = false, currentRoute, results = [], finalFailure; const owned = []; const settle = async () => { if (currentRoute) await api(currentRoute + '/stop', {}, 120000); }; async function waitParent(route, active, deadline = Date.now() + 120000) { for (;;) { const state = await api(route), group = state.execution.collaborations.at(-1); if (!active.includes(group.status)) return { state, group }; assert.ok(Date.now() < deadline, 'Owned parent operation observation ended.'); await delay(750); } } async function artifact(sessionId, id) { let content = '', page; do { page = await api('/artifacts/' + id + '?session=' + sessionId + '&offset=' + content.length + '&limit=16000'); content += page.content; } while (page.truncated); assert.equal(sha(content), page.sha256); return content; } function engineJobs(sessionId) { const db = new DatabaseSync(path.join(frozen.host.identity.dataRoot, 'workbench.sqlite'), { readOnly: true }); try { return db.prepare("SELECT payload FROM records WHERE kind='engine_run' AND session_id=?").all(sessionId).map(row => JSON.parse(row.payload)); } finally { db.close(); } } async function captureEngine(sessionId, prefix) { const jobs = engineJobs(sessionId); save(prefix + '/engine-jobs.json', jobs); const proof = []; for (const job of jobs) { const base = prefix + '/engines/' + job.id; const input = await artifact(sessionId, job.inputArtifactId); assert.equal(sha(input), job.inputSha256); save(base + '/input.json', input); if (job.reportArtifactId) { const report = await api('/sessions/' + sessionId + '/engine-runs/' + job.id + '/download'); assert.equal(sha(JSON.stringify(report)), job.reportSha256); save(base + '/report.json', JSON.stringify(report)); } if (job.seedContext) save(base + '/seed.json', await artifact(sessionId, job.seedContext.artifactId)); for (const request of job.requests) { const text = await artifact(sessionId, request.inputArtifactId); assert.equal(sha(text), request.inputSha256); const input = JSON.parse(text); assertPurgeModelInput(input.native, job.installation.strategy); const metrics = requestMetrics(input.body); assert.equal(metrics.sha256, request.requestSha256); assert.equal(metrics.bytes, request.requestBytes); assert.equal(input.view.mode, 'full'); assert.equal(input.view.inputBudget, null); const variants = requestVariants(input.body); assert.equal(variants.length, 1); assert.ok(variants.every(body => ['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)))); save(base + '/model-' + request.id + '.json', text); if (input.view.sourceArtifactId) save(base + '/source-' + request.id + '.json', await artifact(sessionId, input.view.sourceArtifactId)); proof.push({ jobId: job.id, requestId: request.id, requestSha256: metrics.sha256, variants: variants.map(body => ({ bodySha256: sha(JSON.stringify(body)), toolSha256: sha(JSON.stringify(body.tools)), toolCount: body.tools.length })), nativeSha256: request.sourceSha256, fullHistory: true, outputTokenCap: false }); } for (const call of job.calls) for (const type of ['input', 'output']) { if (call[type + 'ArtifactId']) save(base + '/tool-' + sha(call.sourceCallId) + '-' + type + '.json', await artifact(sessionId, call[type + 'ArtifactId'])); } } save(prefix + '/engine-proof.json', proof); return { jobs, proof }; } try { const host = await api('/host/status'), center = await api('/model-center'); assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0); assert.equal(boot.version, '1.25.8'); compareConfig(frozen.config, boot.config); assert.deepEqual(center.selected, frozen.center.selected); assert.deepEqual(center.pools, frozen.center.pools); original = center.connections.find(row => row.id === frozen.connection.id); assert.deepEqual(connectionDefinition(original), connectionDefinition(frozen.connection)); save('batch-start.json', { order, goal, purgePrompt, limits, config: boot.config, connection: original, localCapsChanged: false, startedAt: new Date().toISOString() }); configured = await api('/model-center/connections', { ...connectionInput(original), modelIdentityPolicy: 'require-match' }); assert.equal(configured.hasKey, original.hasKey); assert.equal(configured.credentialRevision, original.credentialRevision); await api('/model-center/select', { type: 'connection', id: original.id }); selected = true; save('active-model.json', { connection: configured, config: (await api('/bootstrap')).config }); let baseHead; for (const sample of order) { const prefix = sample.id; assert.equal(fs.existsSync(path.join(root, prefix)), false); const store = (file, value) => save(prefix + '/' + file, value), startedAt = Date.now(); let own, session, worker, group, workerState, failure, result, route; try { const installation = frozen.availabilityAuthority === 'host-api' ? await api('/engines/deepseek/controlled-availability?root=' + encodeURIComponent(frozen.installation.root)) : resolveControlledInstallation(process.cwd(), frozen.installation.root); const source = frozen.availabilityAuthority === 'host-api' ? await api('/engines/purge/source?kind=' + ORIGINAL_PURGE_KIND + '&root=' + encodeURIComponent(frozen.source.root) + '&harnessRoot=' + encodeURIComponent(installation.root)) : resolveOriginalPurgeSource(process.cwd(), frozen.source.root, installation); assert.equal(installation.controlledSha256, frozen.installation.controlledSha256); assert.equal(source.sourceSha256, frozen.source.sourceSha256); assert.equal(source.runtimeSha256, frozen.source.runtimeSha256); const current = await api('/model-center'); const currentHost = await api('/host/status'), currentBoot = await api('/bootstrap'); assert.equal(currentHost.identity.pid, frozen.host.identity.pid); assert.equal(currentHost.identity.instanceId, frozen.host.identity.instanceId); assert.equal(currentHost.identity.runtimeRoot, frozen.host.identity.runtimeRoot); assert.equal(currentBoot.version, '1.25.8'); compareConfig(read('active-model.json').config, currentBoot.config); assert.deepEqual(connectionDefinition(current.connections.find(row => row.id === original.id)), connectionDefinition(configured)); assert.deepEqual(current.selected, { type: 'connection', id: original.id }); own = prepareNode(path.join(root, prefix), store); baseHead ??= own.initialHead; assert.equal(own.initialHead, baseHead, 'Every comparison starts with identical original Git content.'); const before = own.checks('baseline'); assert.equal(before.test.stats.tests, 9); assert.equal(before.test.stats.fail, 9); assert.equal(before.build.status, 0); const project = await api('/projects', { root: own.sourceRoot }); session = await api('/sessions', { projectId: project.id }); route = '/sessions/' + session.id; currentRoute = route; owned.push({ projectId: project.id, sessionId: session.id }); store('owned-parent.json', { project, session }); const planned = await api(route + '/collaborations', { goal, engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256, ...(sample.enabled ? { strategy: { kind: ORIGINAL_PURGE_KIND, root: source.root, sourceSha256: source.sourceSha256, runtimeSha256: source.runtimeSha256, prompt: purgePrompt } } : {}) }, children: [{ label: 'expense-candidate', goal, paths: allowed }] }); group = planned.group; store('group-planned.json', group); assert.deepEqual(group.budget, { maxTokens: 0, maxSteps: 0, maxToolCalls: 0, maxDurationMs: 0 }); const groupRoute = route + '/collaborations/' + group.id, fingerprint = group.fingerprint; await api(groupRoute + '/prepare', { fingerprint }, 120000); group = (await waitParent(route, ['preparing'])).group; assert.equal(group.status, 'ready', group.summary); worker = group.children[0]; owned.push({ projectId: worker.projectId, sessionId: worker.sessionId }); store('owned-worker.json', worker); workerState = await api('/sessions/' + worker.sessionId); store('worker-prepared.json', workerState); const workerProject = (await api('/bootstrap')).projects.find(row => row.id === worker.projectId); assert.ok(workerProject); store('worker-project.json', workerProject); await api(groupRoute + '/start', { fingerprint }, 120000); const runningAt = Date.now(); let novelAt = runningAt, signature = '', previous = '', latestMessageAt = 0; const observed = []; for (;;) { const parent = await api(route); group = parent.execution.collaborations.at(-1); workerState = await api('/sessions/' + worker.sessionId); own.protect(); const changes = allowed.map(name => sha(fs.readFileSync(path.join(workerProject.root, name)))); const completedTools = workerState.events.filter(row => row.kind === 'tool' && row.phase === 'completed'); const facts = completedTools.filter(row => ['read_file', 'read_artifact', 'list_files', 'search_files', 'query_code'].includes(row.tool)) .map(row => row.tool + JSON.stringify(row.input) + sha(row.output ?? '')); const next = JSON.stringify({ changes, facts: [...new Set(facts)].sort(), checks: workerState.execution.checks.map(row => ({ status: row.status, source: row.sourceFingerprint })) }); if (signature !== next) { signature = next; novelAt = Date.now(); } const observation = { sample: prefix, group: group.status, worker: workerState.status, requests: workerState.execution.task?.usage.requests, tokens: workerState.execution.task?.usage.totalTokens, changes: workerState.changes.length, checks: workerState.execution.checks.map(row => row.status), engine: workerState.execution.engineRuns?.at(-1)?.status, lastTool: completedTools.at(-1)?.tool, elapsedMs: Date.now() - runningAt }; const key = JSON.stringify({ ...observation, elapsedMs: 0 }); if (key !== previous || Date.now() - latestMessageAt > 45000) { observed.push(observation); console.log(JSON.stringify(observation)); previous = key; latestMessageAt = Date.now(); } if (!['starting', 'running', 'stopping'].includes(group.status)) break; if (Date.now() - runningAt > limits.maximumMs || Date.now() - novelAt > limits.noNewFactMs) { failure = { reason: 'experiment_observation_ended', elapsedMs: Date.now() - runningAt, noNewFactMs: Date.now() - novelAt, productCapsChanged: false }; await settle(); break; } await delay(1000); } store('observations.json', observed); group = (await api(route)).execution.collaborations.at(-1); if (!failure && group.status === 'ready_to_collect') { await api(groupRoute + '/collect', { fingerprint }, 120000); group = (await waitParent(route, ['collecting'])).group; assert.equal(group.status, 'collected', group.summary); await api(groupRoute + '/merge', { fingerprint }, 120000); group = (await waitParent(route, ['merge_preparing'])).group; assert.equal(group.status, 'merge_prepared', group.summary); const checkState = await api(route + '/check-plans', { names: ['test', 'build'] }); const plan = checkState.execution.plans.at(-1); await api(route + '/check-jobs', { planId: plan.id }); let state; do { state = await api(route); if (['queued', 'running'].includes(state.execution.checks.at(-1).status)) await delay(500); else break; } while (Date.now() - startedAt < 900000); assert.equal(state.execution.checks.at(-1).status, 'passed'); const deliveryState = await api(route + '/delivery-plans', { message: 'Complete frozen AF expense engineering' }); await api(route + '/delivery-jobs', { planId: deliveryState.execution.deliveryPlans.at(-1).id }); do { state = await api(route); if (['queued', 'preparing', 'prepared', 'publishing', 'committed', 'reporting'].includes(state.execution.deliveries.at(-1).status)) await delay(500); else break; } while (Date.now() - startedAt < 900000); assert.equal(state.execution.deliveries.at(-1).status, 'completed'); await api(groupRoute + '/finalize', { fingerprint }, 120000); group = (await api(route)).execution.collaborations.at(-1); assert.equal(group.status, 'completed'); } else failure ??= { reason: 'model_or_engine_stopped', groupStatus: group.status, summary: group.summary }; const final = own.checks('independent-final'), commit = own.git(['rev-parse', 'HEAD']).trim(); const committedPaths = own.git(['diff', '--name-only', own.initialHead, commit]).trim().split(/\r?\n/).filter(Boolean).sort(); const committedCurrent = allowed.every(name => sha(Buffer.from(own.git(['show', commit + ':' + name]))) === sha(fs.readFileSync(path.join(own.sourceRoot, name)))); result = { sample: prefix, enabled: sample.enabled, success: !failure && group.status === 'completed' && final.test.stats.pass === 9 && final.test.stats.fail === 0 && final.build.status === 0 && commit !== own.initialHead && committedCurrent && JSON.stringify(committedPaths) === JSON.stringify(allowed.slice().sort()), failure: failure ?? null, final, commit, initialHead: own.initialHead, committedPaths, committedCurrent, parentSessionId: session.id, workerSessionId: worker.sessionId, ownerBytesAndIndexPreserved: true, supervisorApprovals: 0, supervisorContinuations: 0, automaticParentCollectionAndChecks: true }; } catch (error) { result = { sample: prefix, enabled: sample.enabled, success: false, failure: { error: error.stack, retry: false }, parentSessionId: session?.id, workerSessionId: worker?.sessionId }; } finally { const cleanupErrors = []; try { await settle(); } catch (error) { cleanupErrors.push({ phase: 'settle', error: error.stack }); } if (worker) { try { const captured = await captureSession(api, worker.sessionId, prefix + '/worker'), engine = await captureEngine(worker.sessionId, prefix + '/worker'); const identities = captured.rows.length > 0 && captured.rows.every(row => row.status === 'completed' && row.connectionId === original.id && row.modelIdentity?.complete && row.modelIdentity.status === 'matched'); const nativePassed = engine.jobs.length === 1 && engine.jobs.every(job => job.status === 'completed' && !job.requiresVerification && job.verified && job.native?.cleanup?.rootExited && job.native?.cleanup?.treeConfirmed && (!sample.enabled ? !job.native.originalPurge : job.native.originalPurge?.readiness.originalEntryLoaded && job.native.originalPurge?.readiness.originalApplyCalled)); result = { ...result, success: result.success && identities && nativePassed && captured.state.execution.task.usage.unknownRequests === 0, identitiesPassed: identities, nativePassed, requests: captured.rows.length, usage: captured.state.execution.task.usage, reportedCachedInputTokens: captured.rows.reduce((sum, row) => sum + (row.usage?.prompt_tokens_details?.cached_tokens ?? 0), 0), elapsedMs: Date.now() - startedAt, engineElapsedMs: engine.jobs.reduce((sum, job) => sum + Date.parse(job.finishedAt) - Date.parse(job.startedAt), 0), modelIdentityPolicy: 'require-match', captures: captured.captures, engineRequestProof: engine.proof }; } catch (error) { cleanupErrors.push({ phase: 'capture-worker', error: error.stack }); } } if (session) try { await captureSession(api, session.id, prefix + '/parent'); } catch (error) { cleanupErrors.push({ phase: 'capture-parent', error: error.stack }); } if (own) { try { own.protect(); store('git-status.txt', own.git(['status', '--porcelain=v1'])); store('git-log.txt', own.git(['log', '--stat', own.initialHead + '..HEAD'])); } catch (error) { cleanupErrors.push({ phase: 'owner', error: error.stack }); } } if (cleanupErrors.length) result = { ...result, success: false, cleanupErrors }; store('result.json', result); save('owned-' + prefix + '.json', owned); currentRoute = null; results.push(result); console.log(JSON.stringify({ sample: prefix, success: result.success, requests: result.requests, tokens: result.usage?.totalTokens, failure: result.failure, cleanupErrors: result.cleanupErrors })); if (cleanupErrors.length || result.failure?.error) { finalFailure = 'Driver or capture failed; later samples not dispatched.'; break; } } } } catch (error) { finalFailure = error.stack; process.exitCode = 1; } finally { const errors = [], restoration = {}; try { await settle(); } catch (error) { errors.push({ phase: 'settle', error: error.stack }); } try { if (configured) { const center = await api('/model-center'), current = center.connections.find(row => row.id === original.id); if (current.revision === configured.revision) restoration.connection = await api('/model-center/connections', { ...connectionInput(original), revision: current.revision, modelIdentityPolicy: original.modelIdentityPolicy ?? 'record', modelIdentityAliases: original.modelIdentityAliases ?? [], modelIdentityAliasSource: original.modelIdentityAliasSource ?? '' }); else { restoration.externalConnectionChangePreserved = true; errors.push({ phase: 'connection', error: 'External connection revision changed; preserved.' }); } } if (selected) { const center = await api('/model-center'); if (center.selected.type === 'connection' && center.selected.id === original.id) { await api('/model-center/select', frozen.center.selected); restoration.selection = frozen.center.selected; } else { restoration.externalSelectionChangePreserved = true; errors.push({ phase: 'selection', error: 'External selection changed; preserved.' }); } } restoration.config = (await api('/bootstrap')).config; restoration.comparison = compareConfig(frozen.config, restoration.config); } catch (error) { errors.push({ phase: 'restore', error: error.stack }); } restoration.keysReadOrResaved = false; restoration.errors = errors; save('restoration.json', restoration); save('owned.json', owned); save('result.json', { success: results.length === 4 && results.every(row => row.success) && !errors.length && !finalFailure, samples: results, finalFailure: finalFailure ?? null, restorationErrors: errors, purgeBenefit: 'not yet assessed', historicalSamplesReplayed: false }); await close(); } if (finalFailure) console.log(JSON.stringify({ finalFailure }));