import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { setTimeout as delay } from 'node:timers/promises'; import { gitEnvironment } from '../../src/core/git-process.mjs'; import { publicSnapshot, canonical } from './state.mjs'; import { root, origin, dataRoot, connectionId, save, meta, client, sha, captureAttempts, readRoot } from './common.mjs'; // Bounded real Node engineering with supervisor-in-the-loop review. Model // implements the three allowed modules autonomously; supervisor only approves // the original checks, reviews/applies the allowed change set and confirms Git. assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'preflight/result.json'))).passed, true); assert.equal(fs.existsSync(path.join(root, 'node')), false, 'Never replay or overwrite a live sample.'); const original = readRoot(); const { api, boot, close } = await client(); const allowed = ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']; const budgetFields = ['contextLimit', 'maxSteps', 'maxOutputTokens', 'maxRunTokens', 'maxTaskTokens', 'maxRunDurationMs']; const connectionFields = ['id', 'name', 'brand', 'model', 'baseUrl', 'enabled', 'requiresKey', 'streaming', 'protocol', 'capabilityDeclarations', 'contextLimit', 'maxOutputTokens', 'concurrency', 'nativeParameters', 'outputTokenField', 'modelIdentityPolicy', 'modelIdentityAliases', 'modelIdentityAliasSource']; const connectionInput = value => Object.fromEntries(connectionFields.filter(key => Object.hasOwn(value, key)).map(key => [key, value[key]])); const originalIdentity = value => ({ modelIdentityPolicy: value.modelIdentityPolicy ?? 'record', modelIdentityAliases: value.modelIdentityAliases ?? [], modelIdentityAliasSource: value.modelIdentityAliasSource ?? '' }); let before, centerBefore, publicMetadata, originalAttempts, connection, configured, project, session, state, taskId, route, budgetWritten = false, selectionWritten = false; let result, delivery, succeeded = false, failure; const approvals = [], applications = [], continuations = [], observations = [], start = Date.now(); const sourceRoot = path.join(root, 'node/expense-project'); const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', sourceRoot, ...args], { encoding: 'utf8', windowsHide: true, env: gitEnvironment() }); let protectedBytes, stagedBefore; const protect = () => { for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(sourceRoot, name)), bytes, name); assert.equal(git(['show', ':owner-staged.txt']), stagedBefore, 'Preserve original index and working-copy states.'); }; const waitFor = async (predicate, timeout = 30000) => { const deadline = Date.now() + timeout; while (!await predicate()) { assert.ok(Date.now() < deadline, 'Local operation deadline exceeded; verify outcome, never replay.'); await delay(350); } }; const review = async (kind, value) => { const relative = `node/reviews/${kind}-${applications.length + 1}`; const fingerprint = value.fingerprint ?? sha(JSON.stringify(value)); save(relative + '/input.json', { ...value, fingerprint }); console.log(JSON.stringify({ reviewRequired: true, kind, path: relative + '/input.json', fingerprint })); const decisionPath = path.join(root, relative, 'decision.json'); await waitFor(async () => fs.existsSync(decisionPath), Math.max(1, 600000 - (Date.now() - start))); const decision = JSON.parse(fs.readFileSync(decisionPath)); assert.equal(decision.approve, true, 'Supervisor declined the original model proposal.'); assert.ok(typeof decision.reason === 'string' && decision.reason.length > 20, 'Record substantive review.'); assert.equal(decision.fingerprint, fingerprint); return decision; }; const remainingRun = async () => { const requests = state?.execution.task.usage.requests ?? 0; assert.ok(requests < 8, 'The sealed eight-request budget is exhausted.'); const budgets = { contextLimit: 16000, maxOutputTokens: 2048, maxSteps: 8 - requests, maxRunTokens: 90000, maxTaskTokens: 90000, maxRunDurationMs: 600000 }; await api('/model-center/runtime-budgets', budgets); budgetWritten = true; continuations.push({ beforeRequests: requests, remainingSteps: budgets.maxSteps, at: new Date().toISOString() }); }; try { assert.equal(boot.version, '1.25.2'); const host = await api('/host/status'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false); before = await publicSnapshot(origin, dataRoot()); save('node/before.json', before); centerBefore = await api('/model-center'); connection = centerBefore.connections.find(value => value.id === connectionId); assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.revision, original.connectionRevision); assert.equal(connection.model, 'gpt-6.1-sol'); assert.equal(connection.protocol, 'chat-completions'); assert.equal(connection.contextLimit, 16000); assert.deepEqual(originalIdentity(connection), { modelIdentityPolicy: 'record', modelIdentityAliases: [], modelIdentityAliasSource: '' }); publicMetadata = { config: meta('config'), registry: meta('modelRegistry') }; save('node/original-public-metadata.json', publicMetadata); originalAttempts = meta('modelAttempts') ?? []; save('node/original-attempts.json', originalAttempts); save('node/contract.json', { kind: 'new-real-Node-engineering', version: boot.version, connectionId, connection, maxRequests: 8, maxTokens: 90000, contextLimit: 16000, maxOutputTokens: 2048, maxDurationMs: 600000, allowed, checks: ['test', 'build'], expectedTests: 9, commands: 'Only model-planned npm test/build, no arbitrary run_command', review: 'Model seals all three files, supervisor reviews and applies; model plans Git, supervisor confirms.', continuation: 'Only review-action continuation of the same task with remaining steps, no extra implementation prompts.', originalImplementationSupplied: false, automaticRetry: false, poolSwitching: false, originalSelected: centerBefore.selected, originalBudgets: Object.fromEntries(budgetFields.map(key => [key, boot.config[key]])), restore: 'Original budget values and selection restored. Original absent identity fields become explicit record/[]/empty-string defaults; raw pre-state retained and only this declared normalization accepted. Revision counters remain monotonic.' }); fs.cpSync('benchmarks/live-delivery-v1/node', sourceRoot, { recursive: true, errorOnExist: true }); fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# Expense report acceptance\nRepair the three source modules using the immutable original nine tests; do not edit checks or owner files.\n', { flag: 'wx' }); fs.writeFileSync(path.join(sourceRoot, '.gitignore'), 'dist/\n.awb-*\nnode_modules/\n', { flag: 'wx' }); for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) fs.writeFileSync(path.join(sourceRoot, name), 'Owner original baseline\n', { flag: 'wx' }); git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); git(['config', 'user.name', 'Workbench QA']); git(['config', 'user.email', 'qa@workbench.invalid']); git(['add', '--', '.']); git(['commit', '--no-gpg-sign', '-m', 'Seal expense engineering baseline and original checks']); fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner staged change\n'); git(['add', '--', 'owner-staged.txt']); stagedBefore = git(['show', ':owner-staged.txt']); fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner working copy differs from staging\n'); fs.writeFileSync(path.join(sourceRoot, 'owner-unstaged.txt'), 'Owner unstaged change\n'); fs.writeFileSync(path.join(sourceRoot, 'owner-untracked.txt'), 'Owner untracked change\n', { flag: 'wx' }); const protectedNames = ['README.md', '.gitignore', 'package.json', 'test/expense.test.mjs', 'scripts/build.mjs', 'owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt']; protectedBytes = Object.fromEntries(protectedNames.map(name => [name, fs.readFileSync(path.join(sourceRoot, name))])); const sourceFiles = [...protectedNames, ...allowed]; for (const name of sourceFiles) save('node/source-before/' + name, fs.readFileSync(path.join(sourceRoot, name))); save('node/source-identity.json', { head: git(['rev-parse', 'HEAD']).trim(), status: git(['status', '--porcelain=v1']), stagedBefore, files: Object.fromEntries(sourceFiles.map(name => [name, sha(fs.readFileSync(path.join(sourceRoot, name)))])) }); project = await api('/projects', { root: sourceRoot }); session = await api('/sessions', { projectId: project.id }); route = '/sessions/' + session.id; save('node/owned.json', { project, session }); configured = await api('/model-center/connections', { ...connectionInput(connection), revision: connection.revision, modelIdentityPolicy: 'require-match', modelIdentityAliases: [], modelIdentityAliasSource: '' }); assert.equal(configured.credentialRevision, connection.credentialRevision); assert.equal(configured.hasKey, true); save('node/connection-frozen.json', configured); await remainingRun(); await api('/model-center/select', { type: 'connection', id: connectionId }); selectionWritten = true; const goal = '修复这个Node费用报表工程的amount、summary、report三个模块。金额用精确分单位处理,接受测试约定的十进制字符串、有限数字、退款、及空白;拒绝格式错误、超精度和不安全金额。按独立币种规范化、排序汇总并保留原数组,计数准确、安全求和,报表两位小数含负分金额。以原九项测试为完整奥约。' + '先inspect_project,再plan_checks选择["test","build"]并run_checks保留真实基线失败,读取原测试和相关源码。只允许修改src/amount.mjs、src/summary.mjs、src/report.mjs;不改测试、配置、README、构建脚本、owner文件或原暂存,不使用run_command,不直接提交Git。' + 'propose_file/propose_patch准备完整三个模块后,用prepare_change_set一次封存全部三文件,等待审查应用。应用后继续同一个Task,重新plan_checks/run_checks,通过原9项测试和build后,用plan_delivery为三个文件准备本地Git预览并等待确认。模型必须自己实现,禁止用口头完成替代检查。预算最多8次模型请求、90000tokens,输出2048,上下文16K,请合并独立工具调用并避免重复观察。'; save('node/goal.txt', goal); protect(); await api(route + '/messages', { content: goal }); while (Date.now() - start < 600000) { state = await api(route); taskId ??= state.execution.task.id; assert.equal(state.execution.task.id, taskId); protect(); const usage = state.execution.task.usage; assert.ok(usage.requests <= 8); assert.ok(usage.totalTokens <= 90000); const progress = { status: state.status, requests: usage.requests, tokens: usage.totalTokens, lastTool: state.events.filter(event => event.kind === 'tool').at(-1)?.tool, checks: state.execution.checks.map(job => job.status), elapsedSeconds: Math.round((Date.now() - start) / 1000) }; if (JSON.stringify({ ...progress, elapsedSeconds: 0 }) !== JSON.stringify({ ...observations.at(-1), elapsedSeconds: 0 })) { observations.push(progress); console.log(JSON.stringify(progress)); } for (const change of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(change.path), 'Out-of-scope proposal: ' + change.path); for (const approval of state.approvals.filter(item => item.status === 'pending')) { const plan = state.execution.plans.find(value => value.id === approval.checkPlanId); const allow = !!plan && plan.taskId === taskId && plan.checks.length === 2 && plan.checks.every(check => ['test', 'build'].includes(check.name) && !check.lifecycle?.length) && approval.cwd === sourceRoot && approval.command === plan.checks.map(check => check.command).join('\n'); approvals.push({ approvalId: approval.id, planId: plan?.id, command: approval.command, allow }); await api(route + '/approvals/' + approval.id, { allow }); assert.equal(allow, true, 'Only the sealed original checks are authorized.'); } if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await delay(650); continue; } if (state.status === 'failed') throw new Error(state.failure?.message ?? 'Actual-model task failed'); const plan = state.execution.deliveryPlans?.findLast(value => value.status === 'planned' && value.taskId === taskId); if (plan) { assert.deepEqual(plan.paths.slice().sort(), allowed.slice().sort()); const requests = usage.requests; assert.ok(state.events.some(event => event.tool === 'plan_delivery' && event.phase === 'completed')); const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.equal(passing?.results.find(value => value.name === 'test')?.tests?.passed, 9); await review('git', { plan, fingerprint: plan.fingerprint, checks: state.execution.checks, source: state.changes, status: git(['status', '--porcelain=v1']), stagedBefore, stagedAfter: git(['show', ':owner-staged.txt']) }); await api(route + '/delivery-jobs', { planId: plan.id }); await waitFor(async () => { state = await api(route); delivery = state.execution.deliveries.at(-1); return !!delivery?.reportArtifactId; }); assert.equal(delivery.status, 'completed'); assert.equal(state.execution.task.usage.requests, requests); break; } const pending = state.changes.filter(change => change.status === 'pending'); const batch = state.execution.changeSets?.findLast(value => value.taskId === taskId && value.status === 'prepared'); if (pending.length) { assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(value => value.tests?.failed > 0)), 'Model must preserve the original failing tests.'); assert.equal(batch?.source, 'agent', 'Model must seal its own complete change set.'); assert.deepEqual(batch.files.map(file => file.changeId).sort(), pending.map(change => change.id).sort()); if (!applications.length) assert.deepEqual(batch.files.map(file => file.path).sort(), allowed.slice().sort()); assert.ok(applications.length < 3); await review('changes', { fingerprint: batch.fingerprint, batch, changes: pending, checks: state.execution.checks }); await api(route + '/change-sets/' + batch.id + '/apply', { fingerprint: batch.fingerprint }); await waitFor(async () => (await api(route + '/change-sets/' + batch.id)).status === 'completed'); for (const change of pending) assert.equal(sha(fs.readFileSync(path.join(sourceRoot, change.path))), change.sha256); applications.push({ id: batch.id, fingerprint: batch.fingerprint, paths: batch.files.map(file => file.path), source: batch.source }); protect(); await remainingRun(); await api(route + '/continue-reviewed', {}); continue; } throw new Error('Model stopped before verified Git delivery: ' + state.status + '/' + state.execution.run.stopReason); } assert.ok(delivery, 'The sealed engineering deadline ended without complete delivery.'); const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passing); const tests = passing.results.find(value => value.name === 'test')?.tests; assert.equal(tests?.total, 9); assert.equal(tests.passed, 9); assert.equal(tests.failed, 0); assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), allowed.slice().sort()); protect(); succeeded = true; result = { success: true, commit: delivery.commit, tests, delivery, checkJobId: passing.id }; } catch (error) { failure = { name: error.name, message: error.message, stack: error.stack, at: new Date().toISOString(), automaticReplay: false }; result = { success: false, failure }; process.exitCode = 1; } finally { if (route) { state = await api(route); if (['running', 'stopping', 'awaiting_approval'].includes(state.status)) { await api(route + '/stop', {}); state = await api(route); } taskId ??= state.execution.task?.id; save('node/session.json', state); save('node/export.json', await api(route + '/export')); const attempts = (meta('modelAttempts') ?? []).filter(value => value.sessionId === session.id); save('node/attempts.json', attempts); const captures = await captureAttempts(api, attempts, 'node/captures'); const identitiesValid = attempts.every(value => value.connectionId === connectionId && value.poolId === null && value.ordinal === 1 && value.status === 'completed' && value.modelIdentity?.status === 'matched' && value.modelIdentity.complete === true && value.modelIdentity.policy === 'require-match'); result = { ...result, version: boot.version, sessionId: session.id, projectId: project.id, taskId, sourceRoot, connectionId, requestedModel: 'gpt-6.1-sol', usage: state.execution.task?.usage, state: state.status, stopReason: state.execution.run?.stopReason, attempts: attempts.length, identitiesValid, captures, approvals, applications, continuations, observations, modelImplementationSuppliedBySupervisor: false, modelWeightsAuthenticated: false, originalPurgeBenefitTested: false, elapsedSeconds: Math.round((Date.now() - start) / 1000) }; if (succeeded) assert.equal(identitiesValid, true); save('node/model-outcome.json', result); for (const name of allowed) save('node/source-after/' + name, fs.readFileSync(path.join(sourceRoot, name))); save('node/git-final.json', { head: git(['rev-parse', 'HEAD']).trim(), status: git(['status', '--porcelain=v1']), stagedBefore, stagedAfter: git(['show', ':owner-staged.txt']), diff: git(['diff', 'HEAD', '--', ...allowed]) }); } const host = await api('/host/status'); assert.equal(host.activity.active, false, 'Do not overwrite another active task.'); if (configured) { const center = await api('/model-center'), current = center.connections.find(value => value.id === connectionId); assert.equal(current.revision, configured.revision, 'Do not overwrite an external connection edit.'); const restored = await api('/model-center/connections', { ...connectionInput(connection), ...originalIdentity(connection), revision: current.revision }); assert.equal(restored.credentialRevision, connection.credentialRevision); save('node/connection-restored.json', restored); assert.deepEqual(originalIdentity(restored), originalIdentity(connection)); } if (budgetWritten) await api('/model-center/runtime-budgets', Object.fromEntries(budgetFields.map(key => [key, boot.config[key]]))); if (selectionWritten) await api('/model-center/select', centerBefore.selected); const after = await publicSnapshot(origin, dataRoot()); save('node/after.json', after); if (before) { for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item, 'Original export: ' + item.id); for (const table of ['sessions', 'records', 'events', 'outbox']) { const rows = new Map(after.database.tables[table].rows.map(row => [row.id, row])); for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, 'Original ' + table + '/' + row.id); } const artifactMap = new Map(after.persistent.artifacts.map(value => [value.path, value])); for (const item of before.persistent.artifacts) assert.deepEqual(artifactMap.get(item.path), item, 'Original artifact: ' + item.path); const finalBoot = await api('/bootstrap'), normalized = { ...finalBoot.config, configRevision: boot.config.configRevision }; assert.deepEqual(canonical(normalized), canonical(boot.config)); assert.deepEqual(finalBoot.projects.filter(value => value.id !== project?.id), boot.projects, 'Every original public project, excluding only the new owned sample.'); assert.deepEqual(after.modelCenter.selected, centerBefore.selected); assert.deepEqual(after.modelCenter.pools, centerBefore.pools); for (const item of centerBefore.connections.filter(value => ![connectionId, 'legacy-default'].includes(value.id))) assert.deepEqual(after.modelCenter.connections.find(value => value.id === item.id), item); const finalMetadata = { config: meta('config'), registry: meta('modelRegistry') }; save('node/final-public-metadata.json', finalMetadata); const configNormalized = { ...finalMetadata.config, configRevision: publicMetadata.config.configRevision }; assert.deepEqual(canonical(configNormalized), canonical(publicMetadata.config), 'Raw public disk config except its monotonic revision.'); const registryNormalized = structuredClone(finalMetadata.registry); assert.ok(registryNormalized.revision > publicMetadata.registry.revision); registryNormalized.revision = publicMetadata.registry.revision; const originalConnection = publicMetadata.registry.connections.find(value => value.id === connectionId); const finalConnection = registryNormalized.connections.find(value => value.id === connectionId); assert.equal(finalConnection.revision, originalConnection.revision + 2); finalConnection.revision = originalConnection.revision; finalConnection.updatedAt = originalConnection.updatedAt; const registryBefore = structuredClone(publicMetadata.registry); const normalizedBeforeConnection = registryBefore.connections.find(value => value.id === connectionId); const normalization = Object.fromEntries(Object.entries(originalIdentity(normalizedBeforeConnection)).filter(([key]) => !Object.hasOwn(normalizedBeforeConnection, key))); Object.assign(normalizedBeforeConnection, normalization); save('node/declared-identity-default-normalization.json', { connectionId, added: normalization, originalRawStatePreserved: true, originalSemanticsRestored: true, reason: 'The public partial-update API retains missing fields; restore explicit original default semantics rather than silently retaining require-match.' }); for (const value of [registryBefore, registryNormalized]) value.connections.sort((a, b) => a.id.localeCompare(b.id)); assert.deepEqual(canonical(registryNormalized), canonical(registryBefore), 'Registry except declared revision/timestamp/order and explicit original identity defaults.'); const finalAttempts = meta('modelAttempts') ?? []; for (const item of originalAttempts) assert.deepEqual(finalAttempts.find(value => value.id === item.id), item, 'Original complete model attempt: ' + item.id); const metadataRows = new Map(after.database.tables.metadata.rows.map(row => [row.id, row])); for (const row of before.database.tables.metadata.rows.filter(row => !['config', 'modelRegistry', 'modelAttempts'].includes(row.id))) assert.deepEqual(metadataRows.get(row.id), row); save('node/restoration.json', { passed: true, originalExports: before.exports.length, originalArtifactFiles: before.persistent.artifacts.length, originalRowsExact: ['sessions', 'records', 'events', 'outbox'], budgetsAndSelectionRestored: true, poolsUnchanged: true, originalPublicProjectsExact: true, originalAttemptsExact: originalAttempts.length, rawConfigExceptRevisionExact: true, registryExceptDeclaredRevisionTimestampOrderAndIdentityDefaultsExact: true, identityDefaultNormalization: normalization, otherMetadataRowsExact: true, originalCredentialRevision: connection?.credentialRevision, finalCredentialRevision: after.modelCenter.connections.find(value => value.id === connectionId)?.credentialRevision, changedOnlyMonotonicRevisionsObservedHealthAndDeclaredIdentityDefaults: true, pid: after.pid, version: after.version, noKeyExtractedOrResaved: true }); } save('node/result.json', result ?? { success: false, failure, noModelDispatch: true }); await close(); } console.log(JSON.stringify({ success: result.success, attempts: result.attempts, usage: result.usage, state: result.state, stopReason: result.stopReason, commit: result.commit, tests: result.tests, error: result.failure?.message }));