From 2cb6c019fabdeff96f3fc929b45676de98bb9481 Mon Sep 17 00:00:00 2001 From: Agent Workbench Dev Date: Thu, 5 Nov 2026 09:41:16 +0800 Subject: [PATCH] test(live): add AC driver re-running the sealed AB task on 1.25.4 to observe truncation recovery --- scripts/live-ac/common.mjs | 75 +++++++++++++ scripts/live-ac/initial.mjs | 26 +++++ scripts/live-ac/node.mjs | 217 ++++++++++++++++++++++++++++++++++++ scripts/live-ac/state.mjs | 1 + 4 files changed, 319 insertions(+) create mode 100644 scripts/live-ac/common.mjs create mode 100644 scripts/live-ac/initial.mjs create mode 100644 scripts/live-ac/node.mjs create mode 100644 scripts/live-ac/state.mjs diff --git a/scripts/live-ac/common.mjs b/scripts/live-ac/common.mjs new file mode 100644 index 00000000..47aebcd0 --- /dev/null +++ b/scripts/live-ac/common.mjs @@ -0,0 +1,75 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { DatabaseSync } from 'node:sqlite'; +import { Agent, fetch } from 'undici'; +// Recovery sample AC: re-run the sealed AB expense task on the formally upgraded +// 1.25.4 host to observe whether bounded output-truncation recovery lets the model +// split its proposals and progress, where AB (1.25.3) ended at the 2048 ceiling. +export const origin = 'http://127.0.0.1:4318'; +export const batch = path.resolve('.qa/live-recovery-ac'); +// Same DeepSeek connection and low-reasoning parameters as the frozen AB sample. +export const connectionId = '1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd'; +export const experiment = { key: 'deepseek', id: connectionId, parameters: { thinking: { type: 'enabled' }, reasoning_effort: 'low' } }; +export const root = batch; +// The AB sample froze the connection's model/protocol/credentialRevision; reuse it +// so AC targets exactly the same real route without re-reading any credential. +const abCurrent = path.resolve('.qa/live-delivery-ab/current.json'); +if (fs.existsSync(abCurrent)) { + const frozen = JSON.parse(fs.readFileSync(abCurrent)); + experiment.model = frozen.experiment.model; experiment.protocol = frozen.experiment.protocol; + const connection = frozen.center.connections.find(c => c.id === connectionId); + experiment.credentialRevision = connection?.credentialRevision ?? 1; + experiment.abConnectionRevision = frozen.connectionRevision; +} +export const sha = value => createHash('sha256').update(value).digest('hex'); +export const saveAt = (directory, relative, value) => { + const file = path.join(directory, relative); fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' }); +}; +export const save = (relative, value) => saveAt(root, relative, value); +export const readRoot = () => JSON.parse(fs.readFileSync(path.join(root, 'current.json'))); +export const dataRoot = () => JSON.parse(fs.readFileSync(path.join(batch, 'current.json'))).host.identity.dataRoot; +export function meta(name) { + assert.ok(['config', 'modelRegistry', 'modelAttempts', 'modelNetworkDiagnostics'].includes(name)); + const db = new DatabaseSync(path.join(dataRoot(), 'workbench.sqlite'), { readOnly: true }); + try { const row = db.prepare('SELECT value FROM metadata WHERE key=?').get(name); return row ? JSON.parse(row.value) : null; } + finally { db.close(); } +} +export async function client() { + const dispatcher = new Agent({ connect: { autoSelectFamily: true, timeout: 5000 } }); let token; + const api = async (route, input, timeout = 30000) => { + const response = await fetch(origin + '/api' + route, { dispatcher, + method: input === undefined ? 'GET' : 'POST', redirect: 'error', signal: AbortSignal.timeout(timeout), + headers: { ...(token ? { 'X-Workbench-Token': token } : {}), 'Content-Type': 'application/json', Connection: 'close' }, + ...(input === undefined ? {} : { body: JSON.stringify(input) }) }); + const value = await response.json(); + if (!response.ok) throw Object.assign(new Error(route + ': ' + (value.error ?? response.status)), { status: response.status, response: value }); + return value; + }; + const boot = await api('/bootstrap'); token = boot.token; + return { api, boot, close: () => dispatcher.close() }; +} +export async function captureAttempts(api, attempts, relative) { + const captures = []; + for (const attempt of attempts) for (const kind of ['request', 'response']) { + const id = attempt[kind + 'ArtifactId']; if (!id) continue; + let content = '', offset = 0, page; + do { page = await api('/artifacts/' + id + '?session=' + attempt.sessionId + '&offset=' + offset + '&limit=16000'); + assert.ok(page.content.length || !page.truncated); content += page.content; offset += page.content.length; + } while (page.truncated); + assert.equal(sha(content), page.sha256); assert.equal(page.sha256, attempt[kind + 'Sha256']); + const name = `${relative}/${attempt.id}-${kind}.${kind === 'request' ? 'json' : attempt.streaming ? 'sse' : 'txt'}`; + save(name, content); captures.push({ attemptId: attempt.id, kind, artifactId: id, path: name, sha256: sha(content), bytes: Buffer.byteLength(content) }); + } + return captures; +} +// AC keeps its own independent request budget, separate from the sealed AA/AB batch. +export function acUsage() { + const before = new Set(JSON.parse(fs.readFileSync(path.join(root, 'original-attempts.json'))).map(row => row.id)); + const attempts = (meta('modelAttempts') ?? []).filter(row => !before.has(row.id)); + const tokens = attempts.reduce((sum, row) => sum + (row.usage?.total_tokens ?? 0), 0); + return { requests: attempts.length, tokens, unknown: attempts.filter(row => row.usageStatus !== 'reported').length }; +} +export function assertAcBudget() { const usage = acUsage(); assert.ok(usage.requests < 40 && usage.tokens < 400000, 'AC sample budget exhausted'); return usage; } diff --git a/scripts/live-ac/initial.mjs b/scripts/live-ac/initial.mjs new file mode 100644 index 00000000..b53d7bdd --- /dev/null +++ b/scripts/live-ac/initial.mjs @@ -0,0 +1,26 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { publicSnapshot } from './state.mjs'; +import { batch, experiment, connectionId, sha, client, saveAt, meta } from './common.mjs'; +// Freeze the AC baseline once on the formally upgraded 1.25.4 host. Never replay. +assert.equal(fs.existsSync(batch), false, 'AC freezes once; never replay.'); +assert.equal(execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8', windowsHide: true }).trim(), ''); +const { api, boot, close } = await client(); +try { + const host = await api('/host/status'), center = await api('/model-center'), connection = center.connections.find(c => c.id === connectionId); + assert.equal(boot.version, '1.25.4'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0); + assert.ok(connection.enabled && connection.hasKey); assert.equal(connection.streaming, true); + assert.equal(connection.contextLimit, 16000); assert.equal(connection.maxOutputTokens, 2048); + assert.equal(connection.model, experiment.model); assert.equal(connection.protocol, experiment.protocol); + const current = { host, config: boot.config, projects: boot.projects, center, + baseline: await publicSnapshot('http://127.0.0.1:4318', host.identity.dataRoot), + experiment: { ...experiment, model: connection.model, protocol: connection.protocol }, connectionRevision: connection.revision, + reviewedHead: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(), observedAt: new Date().toISOString() }; + saveAt(batch, 'current.json', current); + saveAt(batch, 'original-attempts.json', meta('modelAttempts') ?? []); + saveAt(batch, 'original-diagnostics.json', meta('modelNetworkDiagnostics') ?? []); + console.log(JSON.stringify({ frozen: true, version: boot.version, pid: host.identity.pid, projects: current.baseline.projects, + sessions: current.baseline.exports.length, route: connection.name, parameters: experiment.parameters, modelRequests: 0 })); +} finally { await close(); } diff --git a/scripts/live-ac/node.mjs b/scripts/live-ac/node.mjs new file mode 100644 index 00000000..366f6d0b --- /dev/null +++ b/scripts/live-ac/node.mjs @@ -0,0 +1,217 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { setTimeout as delay } from 'node:timers/promises'; +import { gitEnvironment } from '../../src/core/git-process.mjs'; +import { publicSnapshot, canonical } from './state.mjs'; +import { root, origin, dataRoot, connectionId, save, meta, client, sha, captureAttempts, readRoot, experiment } from './common.mjs'; +// Bounded real Node engineering on 1.25.4 with supervisor-in-the-loop review, +// identical to the sealed AB task. Purpose: observe whether the new output- +// truncation recovery lets the model split its proposals past the 2048 ceiling +// and progress, where AB on 1.25.3 ended with length/0-delivery. The model +// implements the three allowed modules; the supervisor only approves the sealed +// checks, reviews/applies the agent's own complete change set, and confirms Git. +assert.equal(fs.existsSync(path.join(root, 'node')), false, 'Never replay or overwrite a live sample.'); +const original = readRoot(); +experiment.model = original.experiment.model; experiment.protocol = original.experiment.protocol; +const { api, boot, close } = await client(); +const allowed = ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']; +const budgetFields = ['contextLimit', 'maxSteps', 'maxOutputTokens', 'maxRunTokens', 'maxTaskTokens', 'maxRunDurationMs']; +const connectionFields = ['id', 'name', 'brand', 'model', 'baseUrl', 'enabled', 'requiresKey', 'streaming', 'protocol', + 'capabilityDeclarations', 'contextLimit', 'maxOutputTokens', 'concurrency', 'nativeParameters', 'outputTokenField', + 'modelIdentityPolicy', 'modelIdentityAliases', 'modelIdentityAliasSource', 'parameterProfile']; +const connectionInput = value => Object.fromEntries(connectionFields.filter(key => Object.hasOwn(value, key)).map(key => [key, value[key]])); +const originalIdentity = value => ({ modelIdentityPolicy: value.modelIdentityPolicy ?? 'record', + modelIdentityAliases: value.modelIdentityAliases ?? [], modelIdentityAliasSource: value.modelIdentityAliasSource ?? '' }); +let before, centerBefore, publicMetadata, originalAttempts, connection, configured, project, session, state, taskId, route, budgetWritten = false, selectionWritten = false; +let result, delivery, succeeded = false, failure; +const approvals = [], applications = [], continuations = [], observations = [], recoveries = [], start = Date.now(); +const sourceRoot = path.join(root, 'node/expense-project'); +const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', sourceRoot, ...args], { encoding: 'utf8', windowsHide: true, env: gitEnvironment() }); +let protectedBytes, stagedBefore; +const protect = () => { + for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(sourceRoot, name)), bytes, name); + assert.equal(git(['show', ':owner-staged.txt']), stagedBefore, 'Preserve original index and working-copy states.'); +}; +const waitFor = async (predicate, timeout = 30000) => { + const deadline = Date.now() + timeout; + while (!await predicate()) { assert.ok(Date.now() < deadline, 'Local operation deadline exceeded; verify outcome, never replay.'); await delay(350); } +}; +// The supervisor auto-approves only the agent's own sealed proposal, recording a +// substantive rationale. This mirrors the AB supervisor decision, made inline. +const review = async (kind, value, reason) => { + const relative = `node/reviews/${kind}-${applications.length + 1}`; + const fingerprint = value.fingerprint ?? sha(JSON.stringify(value)); + save(relative + '/input.json', { ...value, fingerprint }); + const decision = { approve: true, fingerprint, reason, decidedAt: new Date().toISOString() }; + save(relative + '/decision.json', decision); + console.log(JSON.stringify({ reviewApproved: true, kind, path: relative + '/decision.json', fingerprint })); + return decision; +}; +const remainingRun = async () => { + const { assertAcBudget } = await import('./common.mjs'); assertAcBudget(); + const requests = state?.execution.task.usage.requests ?? 0; + assert.ok(requests < 30, 'The AC request budget is exhausted.'); + const budgets = { contextLimit: 16000, maxOutputTokens: 2048, maxSteps: 30 - requests, + maxRunTokens: 140000, maxTaskTokens: 140000, maxRunDurationMs: 600000 }; + await api('/model-center/runtime-budgets', budgets); budgetWritten = true; + continuations.push({ beforeRequests: requests, remainingSteps: budgets.maxSteps, at: new Date().toISOString() }); +}; +try { + assert.equal(boot.version, '1.25.4'); + const host = await api('/host/status'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false); + before = await publicSnapshot(origin, dataRoot()); save('node/before.json', before); + centerBefore = await api('/model-center'); connection = centerBefore.connections.find(value => value.id === connectionId); + assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.revision, original.connectionRevision); + assert.equal(connection.model, experiment.model); assert.equal(connection.protocol, experiment.protocol); assert.equal(connection.contextLimit, 16000); + assert.deepEqual(originalIdentity(connection), originalIdentity(original.center.connections.find(value => value.id === connectionId))); + publicMetadata = { config: meta('config'), registry: meta('modelRegistry') }; + save('node/diagnostics-before.json', meta('modelNetworkDiagnostics') ?? []); + save('node/original-public-metadata.json', publicMetadata); + originalAttempts = meta('modelAttempts') ?? []; save('node/original-attempts.json', originalAttempts); + save('node/contract.json', { kind: 'recovery-real-Node-engineering', version: boot.version, connectionId, connection, + maxRequests: 30, maxTokens: 140000, contextLimit: 16000, maxOutputTokens: 2048, maxDurationMs: 600000, + allowed, checks: ['test', 'build'], expectedTests: 9, commands: 'Only model-planned npm test/build, no arbitrary run_command', + review: 'Model seals all three files, supervisor reviews and applies; model plans Git, supervisor confirms.', + purpose: 'Observe bounded output-truncation recovery on the same AB task; 1.25.3 AB ended at the 2048 output ceiling with zero delivery.', + originalImplementationSupplied: false, automaticRetry: false, poolSwitching: false, + originalSelected: centerBefore.selected, originalBudgets: Object.fromEntries(budgetFields.map(key => [key, boot.config[key]])), + restore: 'Original budget values, native parameters, identity source and selection restored. Revision counters remain monotonic.' }); + fs.cpSync('benchmarks/live-delivery-v1/node', sourceRoot, { recursive: true, errorOnExist: true }); + fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# Expense report acceptance\nRepair the three source modules using the immutable original nine tests; do not edit checks or owner files.\n', { flag: 'wx' }); + fs.writeFileSync(path.join(sourceRoot, '.gitignore'), 'dist/\n.awb-*\nnode_modules/\n', { flag: 'wx' }); + for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) fs.writeFileSync(path.join(sourceRoot, name), 'Owner original baseline\n', { flag: 'wx' }); + git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); + git(['config', 'user.name', 'Workbench QA']); git(['config', 'user.email', 'qa@workbench.invalid']); + git(['add', '--', '.']); git(['commit', '--no-gpg-sign', '-m', 'Seal expense engineering baseline and original checks']); + fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner staged change\n'); git(['add', '--', 'owner-staged.txt']); + stagedBefore = git(['show', ':owner-staged.txt']); + fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner working copy differs from staging\n'); + fs.writeFileSync(path.join(sourceRoot, 'owner-unstaged.txt'), 'Owner unstaged change\n'); + fs.writeFileSync(path.join(sourceRoot, 'owner-untracked.txt'), 'Owner untracked change\n', { flag: 'wx' }); + const protectedNames = ['README.md', '.gitignore', 'package.json', 'test/expense.test.mjs', 'scripts/build.mjs', 'owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt']; + protectedBytes = Object.fromEntries(protectedNames.map(name => [name, fs.readFileSync(path.join(sourceRoot, name))])); + const sourceFiles = [...protectedNames, ...allowed]; + for (const name of sourceFiles) save('node/source-before/' + name, fs.readFileSync(path.join(sourceRoot, name))); + save('node/source-identity.json', { head: git(['rev-parse', 'HEAD']).trim(), status: git(['status', '--porcelain=v1']), stagedBefore, + files: Object.fromEntries(sourceFiles.map(name => [name, sha(fs.readFileSync(path.join(sourceRoot, name)))])) }); + project = await api('/projects', { root: sourceRoot }); session = await api('/sessions', { projectId: project.id }); route = '/sessions/' + session.id; + save('node/owned.json', { project, session }); + configured = await api('/model-center/connections', { ...connectionInput(connection), revision: connection.revision, + nativeParameters: experiment.parameters, modelIdentityPolicy: 'require-match', modelIdentityAliases: [], modelIdentityAliasSource: '' }); + assert.equal(configured.credentialRevision, connection.credentialRevision); assert.equal(configured.hasKey, true); + save('node/connection-frozen.json', configured); + await remainingRun(); await api('/model-center/select', { type: 'connection', id: connectionId }); selectionWritten = true; + const goal = '修复这个Node费用报表工程的amount、summary、report三个模块。金额用精确分单位处理,接受测试约定的十进制字符串、有限数字、退款、及空白;拒绝格式错误、超精度和不安全金额。按独立币种规范化、排序汇总并保留原数组,计数准确、安全求和,报表两位小数含负分金额。以原九项测试为完整验收约定。' + + '先inspect_project,再plan_checks选择["test","build"]并run_checks保留真实基线失败,读取原测试和相关源码。只允许修改src/amount.mjs、src/summary.mjs、src/report.mjs;不改测试、配置、README、构建脚本、owner文件或原暂存,不使用run_command,不直接提交Git。' + + 'propose_file/propose_patch准备完整三个模块后,用prepare_change_set一次封存全部三文件,等待审查应用。应用后继续同一个Task,重新plan_checks/run_checks,通过原9项测试和build后,用plan_delivery为三个文件准备本地Git预览并等待确认。模型必须自己实现,禁止用口头完成替代检查。预算最多30次模型请求、140000tokens,输出2048,上下文16K,请合并独立工具调用并避免重复观察。'; + save('node/goal.txt', goal); protect(); + await api(route + '/messages', { content: goal }); + while (Date.now() - start < 600000) { + state = await api(route); taskId ??= state.execution.task.id; assert.equal(state.execution.task.id, taskId); protect(); + const usage = state.execution.task.usage; assert.ok(usage.requests <= 30); assert.ok(usage.totalTokens <= 140000); + // Observe bounded output-truncation recovery events emitted by 1.25.4. + for (const event of state.events.filter(ev => ev.kind === 'context' && /输出被上限截断/.test(ev.label ?? ''))) { + if (!recoveries.some(r => r.eventId === event.id)) { recoveries.push({ eventId: event.id, label: event.label, output: event.output, at: event.createdAt }); console.log(JSON.stringify({ recoveryObserved: event.output })); } + } + const progress = { status: state.status, requests: usage.requests, tokens: usage.totalTokens, lastTool: state.events.filter(event => event.kind === 'tool').at(-1)?.tool, + checks: state.execution.checks.map(job => job.status), recoveries: recoveries.length, elapsedSeconds: Math.round((Date.now() - start) / 1000) }; + if (JSON.stringify({ ...progress, elapsedSeconds: 0 }) !== JSON.stringify({ ...observations.at(-1), elapsedSeconds: 0 })) { observations.push(progress); console.log(JSON.stringify(progress)); } + for (const change of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(change.path), 'Out-of-scope proposal: ' + change.path); + for (const approval of state.approvals.filter(item => item.status === 'pending')) { + const plan = state.execution.plans.find(value => value.id === approval.checkPlanId); + const allow = !!plan && plan.taskId === taskId && plan.checks.length === 2 && plan.checks.every(check => ['test', 'build'].includes(check.name) && !check.lifecycle?.length) + && approval.cwd === sourceRoot && approval.command === plan.checks.map(check => check.command).join('\n'); + approvals.push({ approvalId: approval.id, planId: plan?.id, command: approval.command, allow }); + await api(route + '/approvals/' + approval.id, { allow }); assert.equal(allow, true, 'Only the sealed original checks are authorized.'); + } + if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await delay(650); continue; } + if (state.status === 'failed') throw new Error(state.failure?.message ?? 'Actual-model task failed'); + const plan = state.execution.deliveryPlans?.findLast(value => value.status === 'planned' && value.taskId === taskId); + if (plan) { + assert.deepEqual(plan.paths.slice().sort(), allowed.slice().sort()); const requests = usage.requests; + assert.ok(state.events.some(event => event.tool === 'plan_delivery' && event.phase === 'completed')); + const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); + assert.equal(passing?.results.find(value => value.name === 'test')?.tests?.passed, 9); + await review('git', { plan, fingerprint: plan.fingerprint, checks: state.execution.checks, source: state.changes, + status: git(['status', '--porcelain=v1']), stagedBefore, stagedAfter: git(['show', ':owner-staged.txt']) }, + 'Delivery plan covers exactly the three allowed modules; the sealed nine tests pass and build is green. Owner index and working copy are unchanged. Approving the local Git preview.'); + await api(route + '/delivery-jobs', { planId: plan.id }); + await waitFor(async () => { state = await api(route); delivery = state.execution.deliveries.at(-1); return !!delivery?.reportArtifactId; }); + assert.equal(delivery.status, 'completed'); assert.equal(state.execution.task.usage.requests, requests); break; + } + const pending = state.changes.filter(change => change.status === 'pending'); + const changeBatch = state.execution.changeSets?.findLast(value => value.taskId === taskId && value.status === 'prepared'); + if (pending.length) { + assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(value => value.tests?.failed > 0)), 'Model must preserve the original failing tests.'); + assert.equal(changeBatch?.source, 'agent', 'Model must seal its own complete change set.'); + assert.deepEqual(changeBatch.files.map(file => file.changeId).sort(), pending.map(change => change.id).sort()); + if (!applications.length) assert.deepEqual(changeBatch.files.map(file => file.path).sort(), allowed.slice().sort()); + assert.ok(applications.length < 3); + await review('changes', { fingerprint: changeBatch.fingerprint, batch: changeBatch, changes: pending, checks: state.execution.checks }, + 'Agent sealed its own complete change set for the allowed modules only; the original failing tests are preserved before application. Applying the reviewed set.'); + await api(route + '/change-sets/' + changeBatch.id + '/apply', { fingerprint: changeBatch.fingerprint }); + await waitFor(async () => (await api(route + '/change-sets/' + changeBatch.id)).status === 'completed'); + for (const change of pending) assert.equal(sha(fs.readFileSync(path.join(sourceRoot, change.path))), change.sha256); + applications.push({ id: changeBatch.id, fingerprint: changeBatch.fingerprint, paths: changeBatch.files.map(file => file.path), source: changeBatch.source }); protect(); + await remainingRun(); await api(route + '/continue-reviewed', {}); continue; + } + throw new Error('Model stopped before verified Git delivery: ' + state.status + '/' + state.execution.run.stopReason); + } + assert.ok(delivery, 'The engineering deadline ended without complete delivery.'); + const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passing); + const tests = passing.results.find(value => value.name === 'test')?.tests; assert.equal(tests?.total, 9); assert.equal(tests.passed, 9); assert.equal(tests.failed, 0); + assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), allowed.slice().sort()); protect(); + succeeded = true; + result = { success: true, commit: delivery.commit, tests, delivery, checkJobId: passing.id }; +} catch (error) { + failure = { name: error.name, message: error.message, stack: error.stack, at: new Date().toISOString(), automaticReplay: false }; + result = { success: false, failure }; process.exitCode = 1; +} finally { + if (route) { + state = await api(route); + if (['running', 'stopping', 'awaiting_approval'].includes(state.status)) { await api(route + '/stop', {}); state = await api(route); } + taskId ??= state.execution.task?.id; + save('node/session.json', state); save('node/export.json', await api(route + '/export')); + const attempts = (meta('modelAttempts') ?? []).filter(value => value.sessionId === session.id); save('node/attempts.json', attempts); + const captures = await captureAttempts(api, attempts, 'node/captures'); + const identitiesValid = attempts.every(value => value.connectionId === connectionId && value.poolId === null + && (value.status === 'completed' || value.status === 'failed')); + result = { ...result, version: boot.version, sessionId: session.id, projectId: project.id, taskId, sourceRoot, + connectionId, requestedModel: experiment.model, usage: state.execution.task?.usage, state: state.status, stopReason: state.execution.run?.stopReason, + attempts: attempts.length, identitiesValid, captures, approvals, applications, continuations, observations, + outputTruncationRecoveries: recoveries, recoveryCount: recoveries.length, + modelImplementationSuppliedBySupervisor: false, modelWeightsAuthenticated: false, originalPurgeBenefitTested: false, + elapsedSeconds: Math.round((Date.now() - start) / 1000) }; + save('node/model-outcome.json', result); + for (const name of allowed) if (fs.existsSync(path.join(sourceRoot, name))) save('node/source-after/' + name, fs.readFileSync(path.join(sourceRoot, name))); + save('node/git-final.json', { head: git(['rev-parse', 'HEAD']).trim(), status: git(['status', '--porcelain=v1']), stagedBefore, stagedAfter: git(['show', ':owner-staged.txt']), diff: git(['diff', 'HEAD', '--', ...allowed]) }); + } + const host = await api('/host/status'); assert.equal(host.activity.active, false, 'Do not overwrite another active task.'); + if (configured) { + const center = await api('/model-center'), current = center.connections.find(value => value.id === connectionId); + assert.equal(current.revision, configured.revision, 'Do not overwrite an external connection edit.'); + const restored = await api('/model-center/connections', { ...connectionInput(connection), nativeParameters: connection.nativeParameters ?? {}, ...originalIdentity(connection), revision: current.revision }); + assert.equal(restored.credentialRevision, connection.credentialRevision); save('node/connection-restored.json', restored); + assert.deepEqual(originalIdentity(restored), originalIdentity(connection)); + } + if (budgetWritten) await api('/model-center/runtime-budgets', Object.fromEntries(budgetFields.map(key => [key, boot.config[key]]))); + if (selectionWritten) await api('/model-center/select', centerBefore.selected); + const after = await publicSnapshot(origin, dataRoot()); save('node/after.json', after); + if (before) { + for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item, 'Original export: ' + item.id); + for (const table of ['sessions', 'records', 'events', 'outbox']) { + const rows = new Map(after.database.tables[table].rows.map(row => [row.id, row])); + for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, 'Original ' + table + '/' + row.id); + } + const artifactMap = new Map(after.persistent.artifacts.map(value => [value.path, value])); + for (const item of before.persistent.artifacts) assert.deepEqual(artifactMap.get(item.path), item, 'Original artifact: ' + item.path); + save('node/restoration.json', { passed: true, originalExports: before.exports.length, originalArtifactFiles: before.persistent.artifacts.length, + originalRowsExact: ['sessions', 'records', 'events', 'outbox'], budgetsAndSelectionRestored: true, + pid: after.pid, version: after.version, noKeyExtractedOrResaved: true }); + } + save('node/result.json', result ?? { success: false, failure, noModelDispatch: true }); await close(); +} +console.log(JSON.stringify({ success: result.success, attempts: result.attempts, usage: result.usage, state: result.state, + stopReason: result.stopReason, recoveryCount: result.recoveryCount, commit: result.commit, tests: result.tests, error: result.failure?.message })); diff --git a/scripts/live-ac/state.mjs b/scripts/live-ac/state.mjs new file mode 100644 index 00000000..d632a150 --- /dev/null +++ b/scripts/live-ac/state.mjs @@ -0,0 +1 @@ +export { hash, canonical, digest, databaseSnapshot, publicSnapshot, compareOriginalState } from '../retained-host-state.mjs';