test: freeze AB delivery with verified DeepSeek low reasoning

This commit is contained in:
Agent Workbench Dev
2026-11-05 03:23:44 +08:00
parent 6726ce1448
commit 658a5c4e37
11 changed files with 515 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
# AB:已验证 DeepSeek 低强度模式的新工程验证
AA计划4a7af1a6、结果6726ce14和162份证据封存不变:三条关闭模式未通过,AA工程未启动。用户真实验证及全盘开发授权继续有效。本批依据AA新增观测另冻DeepSeek开启低强度;不是更改AA门槛、重放失败或补写旧样本。
固定1.25.3/原PID36552/原Provider,DeepSeek连接1a7a55a9、revision3/credential1、deepseek-flash、Chat SSE。参数thinking.enabled+reasoning_effort=low;不关闭推理,不改型号/端点,不启用停用线路。先核对AAmanifest与当前同修订,再一次所属DNS/TLS诊断、一个新的严格身份无工具单请求预检(16K总预算/2048输出/320秒)。不重复读取上游目录,复用同修订AA目录事实及未知能力。
通过才运行一个全新Node,三源码/原九测试/build/owner staged+unstaged+untracked保持;仅给业务要求和通用工具流程,模型自行实现、封存完整变更集,监督阅读后审查应用、批准原检查和模型生成的Git计划。24请求/140000tokens/600秒/16K2048;不自动重试或换线,不在无进展/step_limit/length/协议/网络/未知消费停止后续跑。应用审查后仅continue-reviewed,剩余请求和Task总预算保持。人工不写实现、不改测试、不发追加实现提示。
Node闭环后再新React/Python,再原版Purge1.1.78/Harness0.2.1-rc.2、purge-original-restricted-v1的ABBA四例off/on/on/off;每例固定源/模型/低强度参数/预算/检查,逐请求原版加载与清理、真实输入/响应/usage封存。React/Python每例24请求/140000tokens,Purge每例24/105000;按AA+AB合计100请求/700000tokens(AA已6请求/771报告tokens+1unknown原预留2235)分配剩余额度,派发前限制,不扩大上限。前置失败则后续明确未执行,不能把提前失败当Purge节省。
恢复原预算、池/选择、身份来源和原生参数语义,允许且声明API缺省字段显式化、单调修订/更新时间/观察健康;不提取、读取、复制、解密、重存Key,不重启正式,不覆盖外部编辑。每阶段核对原行/导出/工件/完整尝试、封存失败与未测,验证/回顾/本地提交,不委派、不推送。与AA、旧8次/关闭推理/其它供应商等条件不同,不作单因素收益比较。
+4
View File
@@ -1,5 +1,9 @@
# 统一执行进度
## AB 新工程冻结
- AA按原门未执行工程且封存提交6726ce14。依用户真实验证/全盘授权,使用已通过DeepSeek低强度另冻AB;旧关闭失败和unknown不改。AB新预检+新Node,不复用或续跑AA Task;跨AA/AB100请求700000tokens上限包含AA未知预留2235。驱动复用已验证单请求载荷,只有新目录/低强度固定参数、总预算和新身份冻结适配,不改产品或原测试。首次规划补丁标题不匹配被拒、无文件变化,按实际标题补入。
## AA 真实验证开始
- AA最终原状态对账通过:91原导出/3926原工件/252原尝试/原配置与凭据均精确;162份179580919字节封存和离线核验通过,三兼容模式失败与四次验收器拒绝保持。QA与阶段回顾见docs/QA-LIVE-VALIDATION-AA.md。当前81项目97会话,空闲/窗口0/9229关闭;另冻AB,不把通过的开启短答当工程交付。
+73
View File
@@ -0,0 +1,73 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
import { Agent, fetch } from 'undici';
export const origin = 'http://127.0.0.1:4318';
export const batch = path.resolve('.qa/live-delivery-ab');
export const routes = [
{ key: 'deepseek', id: '1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd', off: { thinking: { type: 'disabled' } }, on: { thinking: { type: 'enabled' }, reasoning_effort: 'low' } },
{ key: 'claude', id: '9ae348a5-d0a6-42ed-8664-0da5f9effe34', off: { thinking: { type: 'disabled' } }, on: { thinking: { type: 'enabled', budget_tokens: 1024 }, output_config: { effort: 'low' } } },
{ key: 'gpt', id: '93d0cf6d-4f32-4918-9da8-6f125aa8a6e5', off: { reasoning_effort: 'none' }, on: { reasoning_effort: 'low' } },
];
export const target = 'deepseek';
export const experiment = routes.find(row => row.key === target);
export const root = batch;
experiment.parameters = experiment.on;
export const connectionId = experiment?.id;
if (experiment && fs.existsSync(path.join(root, 'current.json'))) {
const frozen = JSON.parse(fs.readFileSync(path.join(root, 'current.json'))).experiment;
experiment.model = frozen.model; experiment.protocol = frozen.protocol;
const connection = JSON.parse(fs.readFileSync(path.join(root, 'current.json'))).center.connections.find(c => c.id === connectionId);
experiment.credentialRevision = connection.credentialRevision;
}
export const sha = value => createHash('sha256').update(value).digest('hex');
export const saveAt = (directory, relative, value) => {
const file = path.join(directory, relative); fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
};
export const save = (relative, value) => saveAt(root, relative, value);
export const readRoot = () => JSON.parse(fs.readFileSync(path.join(root, 'current.json')));
export const dataRoot = () => JSON.parse(fs.readFileSync(path.join(batch, 'current.json'))).host.identity.dataRoot;
export function meta(name) {
assert.ok(['config', 'modelRegistry', 'modelAttempts', 'modelNetworkDiagnostics'].includes(name));
const db = new DatabaseSync(path.join(dataRoot(), 'workbench.sqlite'), { readOnly: true });
try { const row = db.prepare('SELECT value FROM metadata WHERE key=?').get(name); return row ? JSON.parse(row.value) : null; }
finally { db.close(); }
}
export async function client() {
const dispatcher = new Agent({ connect: { autoSelectFamily: true, timeout: 5000 } }); let token;
const api = async (route, input, timeout = 30000) => {
const response = await fetch(origin + '/api' + route, { dispatcher,
method: input === undefined ? 'GET' : 'POST', redirect: 'error', signal: AbortSignal.timeout(timeout),
headers: { ...(token ? { 'X-Workbench-Token': token } : {}), 'Content-Type': 'application/json', Connection: 'close' },
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const value = await response.json();
if (!response.ok) throw Object.assign(new Error(route + ': ' + (value.error ?? response.status)), { status: response.status, response: value });
return value;
};
const boot = await api('/bootstrap'); token = boot.token;
return { api, boot, close: () => dispatcher.close() };
}
export async function captureAttempts(api, attempts, relative) {
const captures = [];
for (const attempt of attempts) for (const kind of ['request', 'response']) {
const id = attempt[kind + 'ArtifactId']; if (!id) continue;
let content = '', offset = 0, page;
do { page = await api('/artifacts/' + id + '?session=' + attempt.sessionId + '&offset=' + offset + '&limit=16000');
assert.ok(page.content.length || !page.truncated); content += page.content; offset += page.content.length;
} while (page.truncated);
assert.equal(sha(content), page.sha256); assert.equal(page.sha256, attempt[kind + 'Sha256']);
const name = `${relative}/${attempt.id}-${kind}.${kind === 'request' ? 'json' : attempt.streaming ? 'sse' : 'txt'}`;
save(name, content); captures.push({ attemptId: attempt.id, kind, artifactId: id, path: name, sha256: sha(content), bytes: Buffer.byteLength(content) });
}
return captures;
}
export function batchUsage() {
const before = new Set(JSON.parse(fs.readFileSync(path.resolve('docs/live-validation-aa-evidence/original-attempts.json'))).map(row => row.id));
const attempts = (meta('modelAttempts') ?? []).filter(row => !before.has(row.id));
const tokens = attempts.reduce((sum, row) => sum + (row.usage?.total_tokens ?? 0), 0);
return { requests: attempts.length, tokens, unknown: attempts.filter(row => row.usageStatus !== 'reported').length, historicalUnknownReservation: 2235, accountedTokens: tokens + 2235 };
}
export function assertBatchBudget() { const usage = batchUsage(); assert.ok(usage.requests < 100 && usage.accountedTokens < 700000, 'Combined AA/AB batch budget exhausted'); return usage; }
+19
View File
@@ -0,0 +1,19 @@
import assert from 'node:assert/strict';
import { setTimeout as delay } from 'node:timers/promises';
import { client, save, readRoot, connectionId } from './common.mjs';
const { api, close } = await client(); const original = readRoot(); let id;
try {
const center = await api('/model-center'), connection = center.connections.find(c => c.id === connectionId);
assert.equal(connection.revision, original.connectionRevision);
const started = await api('/model-center/network-diagnostics', { id: connectionId, revision: connection.revision });
id = started.job.id; save('network-start.json', started); const deadline = Date.now() + 25000;
let state;
do { state = await api('/model-center/network-diagnostics'); if (!state.active) break; assert.ok(Date.now() < deadline); await delay(250); } while (true);
const job = state.jobs.find(job => job.id === id); save('network.json', job);
assert.equal(job.httpRequests, 0); assert.equal(job.modelRequests, 0); assert.equal(job.credentialAccess, false);
console.log(JSON.stringify({ status: job.status, modelRequests: 0, httpRequests: 0, retry: false }));
} finally {
const state = await api('/model-center/network-diagnostics');
if (id && state.active && ['pending', 'running'].includes(state.jobs.find(job => job.id === id)?.status)) save('network-stop.json', await api('/model-center/network-diagnostics/' + id + '/stop', {}));
await close();
}
+30
View File
@@ -0,0 +1,30 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { publicSnapshot } from './state.mjs';
import { batch, experiment, sha, client, saveAt } from './common.mjs';
assert.equal(fs.existsSync(batch), false, 'AB freezes once; never replay.');
assert.equal(execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8', windowsHide: true }).trim(), '');
const aaManifest = fs.readFileSync('docs/live-validation-aa-evidence/MANIFEST.json');
assert.equal(sha(aaManifest), '4dc3847b63bcf6b738f02390345f45337d88e903739295b5c02cdf92eea38612');
const aaSummary = JSON.parse(fs.readFileSync('docs/live-validation-aa-evidence/compatibility-summary.json'));
assert.equal(aaSummary.engineeringRoute, null);
assert.equal(aaSummary.observations.find(row => row.key === experiment.key).modes.on.passed, true);
const { api, boot, close } = await client();
try {
const host = await api('/host/status'), center = await api('/model-center'), connection = center.connections.find(c => c.id === experiment.id);
assert.equal(boot.version, '1.25.3'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0);
assert.ok(connection.enabled && connection.hasKey); assert.equal(connection.streaming, true); assert.equal(connection.contextLimit, 16000); assert.equal(connection.maxOutputTokens, 2048);
const original = JSON.parse(fs.readFileSync('docs/live-validation-aa-evidence/deepseek/current.json'));
assert.equal(connection.revision, original.connectionRevision); assert.equal(connection.credentialRevision, original.experiment.credentialRevision ?? 1);
const runtimeManifest = fs.readFileSync(path.join(boot.runtimeRoot, 'MANIFEST.json'));
assert.equal(sha(runtimeManifest), JSON.parse(fs.readFileSync('docs/model-catalog-1.25.3-release-identity.json')).portableManifestSha256);
const current = { host, config: boot.config, projects: boot.projects, center, baseline: await publicSnapshot('http://127.0.0.1:4318', host.identity.dataRoot),
experiment: { ...experiment, model: connection.model, protocol: connection.protocol }, connectionRevision: connection.revision,
reviewedHead: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(), runtimeManifestSha256: sha(runtimeManifest),
aaManifestSha256: sha(aaManifest), observedAt: new Date().toISOString() };
saveAt(batch, 'current.json', current);
const { meta } = await import('./common.mjs'); saveAt(batch, 'original-attempts.json', meta('modelAttempts') ?? []); saveAt(batch, 'original-diagnostics.json', meta('modelNetworkDiagnostics') ?? []);
console.log(JSON.stringify({ frozen: true, version: boot.version, pid: host.identity.pid, projects: current.baseline.projects, sessions: current.baseline.exports.length, route: connection.name, parameters: experiment.parameters, modelRequests: 0 }));
} finally { await close(); }
+1
View File
@@ -0,0 +1 @@
export * from '../live-aa/inspector.mjs';
+248
View File
@@ -0,0 +1,248 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { setTimeout as delay } from 'node:timers/promises';
import { gitEnvironment } from '../../src/core/git-process.mjs';
import { publicSnapshot, canonical } from './state.mjs';
import { root, origin, dataRoot, connectionId, save, meta, client, sha, captureAttempts, readRoot, experiment } from './common.mjs';
// Bounded real Node engineering with supervisor-in-the-loop review. Model
// implements the three allowed modules autonomously; supervisor only approves
// the original checks, reviews/applies the allowed change set and confirms Git.
assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'preflight/result.json'))).passed, true);
assert.equal(fs.existsSync(path.join(root, 'node')), false, 'Never replay or overwrite a live sample.');
const original = readRoot();
experiment.model = original.experiment.model; experiment.protocol = original.experiment.protocol;
const { api, boot, close } = await client();
const allowed = ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs'];
const budgetFields = ['contextLimit', 'maxSteps', 'maxOutputTokens', 'maxRunTokens', 'maxTaskTokens', 'maxRunDurationMs'];
const connectionFields = ['id', 'name', 'brand', 'model', 'baseUrl', 'enabled', 'requiresKey', 'streaming', 'protocol',
'capabilityDeclarations', 'contextLimit', 'maxOutputTokens', 'concurrency', 'nativeParameters', 'outputTokenField',
'modelIdentityPolicy', 'modelIdentityAliases', 'modelIdentityAliasSource', 'parameterProfile'];
const connectionInput = value => Object.fromEntries(connectionFields.filter(key => Object.hasOwn(value, key)).map(key => [key, value[key]]));
const originalIdentity = value => ({ modelIdentityPolicy: value.modelIdentityPolicy ?? 'record',
modelIdentityAliases: value.modelIdentityAliases ?? [], modelIdentityAliasSource: value.modelIdentityAliasSource ?? '' });
let before, centerBefore, publicMetadata, originalAttempts, connection, configured, project, session, state, taskId, route, budgetWritten = false, selectionWritten = false;
let result, delivery, succeeded = false, failure;
const approvals = [], applications = [], continuations = [], observations = [], start = Date.now();
const sourceRoot = path.join(root, 'node/expense-project');
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', sourceRoot, ...args], { encoding: 'utf8', windowsHide: true, env: gitEnvironment() });
let protectedBytes, stagedBefore;
const protect = () => {
for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(sourceRoot, name)), bytes, name);
assert.equal(git(['show', ':owner-staged.txt']), stagedBefore, 'Preserve original index and working-copy states.');
};
const waitFor = async (predicate, timeout = 30000) => {
const deadline = Date.now() + timeout;
while (!await predicate()) { assert.ok(Date.now() < deadline, 'Local operation deadline exceeded; verify outcome, never replay.'); await delay(350); }
};
const review = async (kind, value) => {
const relative = `node/reviews/${kind}-${applications.length + 1}`;
const fingerprint = value.fingerprint ?? sha(JSON.stringify(value));
save(relative + '/input.json', { ...value, fingerprint });
console.log(JSON.stringify({ reviewRequired: true, kind, path: relative + '/input.json', fingerprint }));
const decisionPath = path.join(root, relative, 'decision.json');
await waitFor(async () => fs.existsSync(decisionPath), Math.max(1, 600000 - (Date.now() - start)));
const decision = JSON.parse(fs.readFileSync(decisionPath));
assert.equal(decision.approve, true, 'Supervisor declined the original model proposal.');
assert.ok(typeof decision.reason === 'string' && decision.reason.length > 20, 'Record substantive review.');
assert.equal(decision.fingerprint, fingerprint);
return decision;
};
const remainingRun = async () => {
const { assertBatchBudget } = await import('./common.mjs'); assertBatchBudget();
const requests = state?.execution.task.usage.requests ?? 0;
assert.ok(requests < 24, 'The sealed 24-request budget is exhausted.');
const budgets = { contextLimit: 16000, maxOutputTokens: 2048, maxSteps: 24 - requests,
maxRunTokens: 140000, maxTaskTokens: 140000, maxRunDurationMs: 600000 };
await api('/model-center/runtime-budgets', budgets); budgetWritten = true;
continuations.push({ beforeRequests: requests, remainingSteps: budgets.maxSteps, at: new Date().toISOString() });
};
try {
assert.equal(boot.version, '1.25.3');
const host = await api('/host/status'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false);
before = await publicSnapshot(origin, dataRoot()); save('node/before.json', before);
centerBefore = await api('/model-center'); connection = centerBefore.connections.find(value => value.id === connectionId);
assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.revision, original.connectionRevision);
assert.equal(connection.model, experiment.model); assert.equal(connection.protocol, experiment.protocol); assert.equal(connection.contextLimit, 16000);
assert.deepEqual(originalIdentity(connection), originalIdentity(original.center.connections.find(value => value.id === connectionId)));
publicMetadata = { config: meta('config'), registry: meta('modelRegistry') };
save('node/diagnostics-before.json', meta('modelNetworkDiagnostics') ?? []);
save('node/original-public-metadata.json', publicMetadata);
originalAttempts = meta('modelAttempts') ?? []; save('node/original-attempts.json', originalAttempts);
save('node/contract.json', { kind: 'new-real-Node-engineering', version: boot.version, connectionId, connection,
maxRequests: 24, maxTokens: 140000, contextLimit: 16000, maxOutputTokens: 2048, maxDurationMs: 600000,
allowed, checks: ['test', 'build'], expectedTests: 9, commands: 'Only model-planned npm test/build, no arbitrary run_command',
review: 'Model seals all three files, supervisor reviews and applies; model plans Git, supervisor confirms.',
continuation: 'Only review-action continuation of the same task with remaining steps, no extra implementation prompts.',
originalImplementationSupplied: false, automaticRetry: false, poolSwitching: false,
originalSelected: centerBefore.selected, originalBudgets: Object.fromEntries(budgetFields.map(key => [key, boot.config[key]])),
restore: 'Original budget values, native parameters, identity source and selection restored. Original absent fields may become explicit API defaults; raw pre-state retained and only declared default normalization accepted. Revision counters remain monotonic.' });
fs.cpSync('benchmarks/live-delivery-v1/node', sourceRoot, { recursive: true, errorOnExist: true });
fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# Expense report acceptance\nRepair the three source modules using the immutable original nine tests; do not edit checks or owner files.\n', { flag: 'wx' });
fs.writeFileSync(path.join(sourceRoot, '.gitignore'), 'dist/\n.awb-*\nnode_modules/\n', { flag: 'wx' });
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) fs.writeFileSync(path.join(sourceRoot, name), 'Owner original baseline\n', { flag: 'wx' });
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']);
git(['config', 'user.name', 'Workbench QA']); git(['config', 'user.email', 'qa@workbench.invalid']);
git(['add', '--', '.']); git(['commit', '--no-gpg-sign', '-m', 'Seal expense engineering baseline and original checks']);
fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner staged change\n'); git(['add', '--', 'owner-staged.txt']);
stagedBefore = git(['show', ':owner-staged.txt']);
fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner working copy differs from staging\n');
fs.writeFileSync(path.join(sourceRoot, 'owner-unstaged.txt'), 'Owner unstaged change\n');
fs.writeFileSync(path.join(sourceRoot, 'owner-untracked.txt'), 'Owner untracked change\n', { flag: 'wx' });
const protectedNames = ['README.md', '.gitignore', 'package.json', 'test/expense.test.mjs', 'scripts/build.mjs', 'owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'];
protectedBytes = Object.fromEntries(protectedNames.map(name => [name, fs.readFileSync(path.join(sourceRoot, name))]));
const sourceFiles = [...protectedNames, ...allowed];
for (const name of sourceFiles) save('node/source-before/' + name, fs.readFileSync(path.join(sourceRoot, name)));
save('node/source-identity.json', { head: git(['rev-parse', 'HEAD']).trim(), status: git(['status', '--porcelain=v1']), stagedBefore,
files: Object.fromEntries(sourceFiles.map(name => [name, sha(fs.readFileSync(path.join(sourceRoot, name)))])) });
project = await api('/projects', { root: sourceRoot }); session = await api('/sessions', { projectId: project.id }); route = '/sessions/' + session.id;
save('node/owned.json', { project, session });
configured = await api('/model-center/connections', { ...connectionInput(connection), revision: connection.revision,
nativeParameters: experiment.parameters, modelIdentityPolicy: 'require-match', modelIdentityAliases: [], modelIdentityAliasSource: '' });
assert.equal(configured.credentialRevision, connection.credentialRevision); assert.equal(configured.hasKey, true);
save('node/connection-frozen.json', configured);
await remainingRun(); await api('/model-center/select', { type: 'connection', id: connectionId }); selectionWritten = true;
const goal = '修复这个Node费用报表工程的amount、summary、report三个模块。金额用精确分单位处理,接受测试约定的十进制字符串、有限数字、退款、及空白;拒绝格式错误、超精度和不安全金额。按独立币种规范化、排序汇总并保留原数组,计数准确、安全求和,报表两位小数含负分金额。以原九项测试为完整验收约定。'
+ '先inspect_project,再plan_checks选择["test","build"]并run_checks保留真实基线失败,读取原测试和相关源码。只允许修改src/amount.mjs、src/summary.mjs、src/report.mjs;不改测试、配置、README、构建脚本、owner文件或原暂存,不使用run_command,不直接提交Git。'
+ 'propose_file/propose_patch准备完整三个模块后,用prepare_change_set一次封存全部三文件,等待审查应用。应用后继续同一个Task,重新plan_checks/run_checks,通过原9项测试和build后,用plan_delivery为三个文件准备本地Git预览并等待确认。模型必须自己实现,禁止用口头完成替代检查。预算最多24次模型请求、140000tokens,输出2048,上下文16K,请合并独立工具调用并避免重复观察。';
save('node/goal.txt', goal); protect();
await api(route + '/messages', { content: goal });
while (Date.now() - start < 600000) {
state = await api(route); taskId ??= state.execution.task.id; assert.equal(state.execution.task.id, taskId); protect();
const usage = state.execution.task.usage; assert.ok(usage.requests <= 24); assert.ok(usage.totalTokens <= 140000);
const progress = { status: state.status, requests: usage.requests, tokens: usage.totalTokens, lastTool: state.events.filter(event => event.kind === 'tool').at(-1)?.tool,
checks: state.execution.checks.map(job => job.status), elapsedSeconds: Math.round((Date.now() - start) / 1000) };
if (JSON.stringify({ ...progress, elapsedSeconds: 0 }) !== JSON.stringify({ ...observations.at(-1), elapsedSeconds: 0 })) { observations.push(progress); console.log(JSON.stringify(progress)); }
for (const change of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(change.path), 'Out-of-scope proposal: ' + change.path);
for (const approval of state.approvals.filter(item => item.status === 'pending')) {
const plan = state.execution.plans.find(value => value.id === approval.checkPlanId);
const allow = !!plan && plan.taskId === taskId && plan.checks.length === 2 && plan.checks.every(check => ['test', 'build'].includes(check.name) && !check.lifecycle?.length)
&& approval.cwd === sourceRoot && approval.command === plan.checks.map(check => check.command).join('\n');
approvals.push({ approvalId: approval.id, planId: plan?.id, command: approval.command, allow });
await api(route + '/approvals/' + approval.id, { allow }); assert.equal(allow, true, 'Only the sealed original checks are authorized.');
}
if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await delay(650); continue; }
if (state.status === 'failed') throw new Error(state.failure?.message ?? 'Actual-model task failed');
const plan = state.execution.deliveryPlans?.findLast(value => value.status === 'planned' && value.taskId === taskId);
if (plan) {
assert.deepEqual(plan.paths.slice().sort(), allowed.slice().sort()); const requests = usage.requests;
assert.ok(state.events.some(event => event.tool === 'plan_delivery' && event.phase === 'completed'));
const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1);
assert.equal(passing?.results.find(value => value.name === 'test')?.tests?.passed, 9);
await review('git', { plan, fingerprint: plan.fingerprint, checks: state.execution.checks, source: state.changes,
status: git(['status', '--porcelain=v1']), stagedBefore, stagedAfter: git(['show', ':owner-staged.txt']) });
await api(route + '/delivery-jobs', { planId: plan.id });
await waitFor(async () => { state = await api(route); delivery = state.execution.deliveries.at(-1); return !!delivery?.reportArtifactId; });
assert.equal(delivery.status, 'completed'); assert.equal(state.execution.task.usage.requests, requests); break;
}
const pending = state.changes.filter(change => change.status === 'pending');
const batch = state.execution.changeSets?.findLast(value => value.taskId === taskId && value.status === 'prepared');
if (pending.length) {
assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(value => value.tests?.failed > 0)), 'Model must preserve the original failing tests.');
assert.equal(batch?.source, 'agent', 'Model must seal its own complete change set.');
assert.deepEqual(batch.files.map(file => file.changeId).sort(), pending.map(change => change.id).sort());
if (!applications.length) assert.deepEqual(batch.files.map(file => file.path).sort(), allowed.slice().sort());
assert.ok(applications.length < 3);
await review('changes', { fingerprint: batch.fingerprint, batch, changes: pending, checks: state.execution.checks });
await api(route + '/change-sets/' + batch.id + '/apply', { fingerprint: batch.fingerprint });
await waitFor(async () => (await api(route + '/change-sets/' + batch.id)).status === 'completed');
for (const change of pending) assert.equal(sha(fs.readFileSync(path.join(sourceRoot, change.path))), change.sha256);
applications.push({ id: batch.id, fingerprint: batch.fingerprint, paths: batch.files.map(file => file.path), source: batch.source }); protect();
await remainingRun(); await api(route + '/continue-reviewed', {}); continue;
}
throw new Error('Model stopped before verified Git delivery: ' + state.status + '/' + state.execution.run.stopReason);
}
assert.ok(delivery, 'The sealed engineering deadline ended without complete delivery.');
const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passing);
const tests = passing.results.find(value => value.name === 'test')?.tests; assert.equal(tests?.total, 9); assert.equal(tests.passed, 9); assert.equal(tests.failed, 0);
assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), allowed.slice().sort()); protect();
succeeded = true;
result = { success: true, commit: delivery.commit, tests, delivery, checkJobId: passing.id };
} catch (error) {
failure = { name: error.name, message: error.message, stack: error.stack, at: new Date().toISOString(), automaticReplay: false };
result = { success: false, failure }; process.exitCode = 1;
} finally {
if (route) {
state = await api(route);
if (['running', 'stopping', 'awaiting_approval'].includes(state.status)) { await api(route + '/stop', {}); state = await api(route); }
taskId ??= state.execution.task?.id;
save('node/session.json', state); save('node/export.json', await api(route + '/export'));
const attempts = (meta('modelAttempts') ?? []).filter(value => value.sessionId === session.id); save('node/attempts.json', attempts);
const captures = await captureAttempts(api, attempts, 'node/captures');
const identitiesValid = attempts.every(value => value.connectionId === connectionId && value.poolId === null && value.ordinal === 1
&& value.status === 'completed' && value.modelIdentity?.status === 'matched' && value.modelIdentity.complete === true && value.modelIdentity.policy === 'require-match');
result = { ...result, version: boot.version, sessionId: session.id, projectId: project.id, taskId, sourceRoot,
connectionId, requestedModel: experiment.model, usage: state.execution.task?.usage, state: state.status, stopReason: state.execution.run?.stopReason,
attempts: attempts.length, identitiesValid, captures, approvals, applications, continuations, observations,
modelImplementationSuppliedBySupervisor: false, modelWeightsAuthenticated: false, originalPurgeBenefitTested: false,
elapsedSeconds: Math.round((Date.now() - start) / 1000) };
if (succeeded) assert.equal(identitiesValid, true);
save('node/model-outcome.json', result);
for (const name of allowed) save('node/source-after/' + name, fs.readFileSync(path.join(sourceRoot, name)));
save('node/git-final.json', { head: git(['rev-parse', 'HEAD']).trim(), status: git(['status', '--porcelain=v1']), stagedBefore, stagedAfter: git(['show', ':owner-staged.txt']), diff: git(['diff', 'HEAD', '--', ...allowed]) });
}
const host = await api('/host/status'); assert.equal(host.activity.active, false, 'Do not overwrite another active task.');
if (configured) {
const center = await api('/model-center'), current = center.connections.find(value => value.id === connectionId);
assert.equal(current.revision, configured.revision, 'Do not overwrite an external connection edit.');
const restored = await api('/model-center/connections', { ...connectionInput(connection), nativeParameters: connection.nativeParameters ?? {}, ...originalIdentity(connection), revision: current.revision });
assert.equal(restored.credentialRevision, connection.credentialRevision); save('node/connection-restored.json', restored);
assert.deepEqual(originalIdentity(restored), originalIdentity(connection));
}
if (budgetWritten) await api('/model-center/runtime-budgets', Object.fromEntries(budgetFields.map(key => [key, boot.config[key]])));
if (selectionWritten) await api('/model-center/select', centerBefore.selected);
const after = await publicSnapshot(origin, dataRoot()); save('node/after.json', after);
if (before) {
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item, 'Original export: ' + item.id);
for (const table of ['sessions', 'records', 'events', 'outbox']) {
const rows = new Map(after.database.tables[table].rows.map(row => [row.id, row]));
for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, 'Original ' + table + '/' + row.id);
}
const artifactMap = new Map(after.persistent.artifacts.map(value => [value.path, value]));
for (const item of before.persistent.artifacts) assert.deepEqual(artifactMap.get(item.path), item, 'Original artifact: ' + item.path);
const finalBoot = await api('/bootstrap'), normalized = { ...finalBoot.config, configRevision: boot.config.configRevision };
assert.deepEqual(canonical(normalized), canonical(boot.config));
assert.deepEqual(finalBoot.projects.filter(value => value.id !== project?.id), boot.projects, 'Every original public project, excluding only the new owned sample.');
assert.deepEqual(after.modelCenter.selected, centerBefore.selected); assert.deepEqual(after.modelCenter.pools, centerBefore.pools);
for (const item of centerBefore.connections.filter(value => ![connectionId, 'legacy-default'].includes(value.id))) assert.deepEqual(after.modelCenter.connections.find(value => value.id === item.id), item);
const finalMetadata = { config: meta('config'), registry: meta('modelRegistry') };
save('node/final-public-metadata.json', finalMetadata);
const configNormalized = { ...finalMetadata.config, configRevision: publicMetadata.config.configRevision };
assert.deepEqual(canonical(configNormalized), canonical(publicMetadata.config), 'Raw public disk config except its monotonic revision.');
const registryNormalized = structuredClone(finalMetadata.registry);
assert.ok(registryNormalized.revision > publicMetadata.registry.revision);
registryNormalized.revision = publicMetadata.registry.revision;
const originalConnection = publicMetadata.registry.connections.find(value => value.id === connectionId);
const finalConnection = registryNormalized.connections.find(value => value.id === connectionId);
assert.equal(finalConnection.revision, originalConnection.revision + 2);
finalConnection.revision = originalConnection.revision; finalConnection.updatedAt = originalConnection.updatedAt;
const registryBefore = structuredClone(publicMetadata.registry);
const normalizedBeforeConnection = registryBefore.connections.find(value => value.id === connectionId);
if (!Object.hasOwn(normalizedBeforeConnection, 'outputTokenField') && Object.hasOwn(finalConnection, 'outputTokenField')) assert.equal(finalConnection.outputTokenField, 'max_tokens', 'Only the documented Chat/Messages default can be normalized.');
const identityNormalization = Object.fromEntries(Object.entries(originalIdentity(normalizedBeforeConnection)).filter(([key]) => !Object.hasOwn(normalizedBeforeConnection, key)));
const normalization = { ...identityNormalization, ...(!Object.hasOwn(normalizedBeforeConnection, 'nativeParameters') ? { nativeParameters: {} } : {}), ...(!Object.hasOwn(normalizedBeforeConnection, 'outputTokenField') && Object.hasOwn(finalConnection, 'outputTokenField') ? { outputTokenField: finalConnection.outputTokenField } : {}) };
Object.assign(normalizedBeforeConnection, normalization);
save('node/declared-identity-default-normalization.json', { connectionId, added: normalization,
originalRawStatePreserved: true, originalSemanticsRestored: true, reason: 'The public partial-update API retains missing fields; restore explicit original default semantics rather than silently retaining require-match.' });
for (const value of [registryBefore, registryNormalized]) value.connections.sort((a, b) => a.id.localeCompare(b.id));
assert.deepEqual(canonical(registryNormalized), canonical(registryBefore), 'Registry except declared revision/timestamp/order and explicit original identity defaults.');
const finalAttempts = meta('modelAttempts') ?? [];
for (const item of originalAttempts) assert.deepEqual(finalAttempts.find(value => value.id === item.id), item, 'Original complete model attempt: ' + item.id);
const metadataRows = new Map(after.database.tables.metadata.rows.map(row => [row.id, row]));
const diagnosticsBefore = JSON.parse(fs.readFileSync(path.join(root, 'node/diagnostics-before.json')));
const diagnosticsAfter = meta('modelNetworkDiagnostics') ?? [];
assert.deepEqual(diagnosticsAfter.slice(0, diagnosticsBefore.length), diagnosticsBefore, 'Every original diagnostic field and order.');
save('node/diagnostics-after.json', { originalExact: true, jobs: diagnosticsAfter, externalAdditions: diagnosticsAfter.slice(diagnosticsBefore.length) });
for (const row of before.database.tables.metadata.rows.filter(row => !['config', 'modelRegistry', 'modelAttempts', 'modelNetworkDiagnostics'].includes(row.id))) assert.deepEqual(metadataRows.get(row.id), row);
save('node/restoration.json', { passed: true, originalExports: before.exports.length, originalArtifactFiles: before.persistent.artifacts.length,
originalRowsExact: ['sessions', 'records', 'events', 'outbox'], budgetsAndSelectionRestored: true, poolsUnchanged: true,
originalPublicProjectsExact: true, originalAttemptsExact: originalAttempts.length, rawConfigExceptRevisionExact: true,
registryExceptDeclaredRevisionTimestampOrderAndIdentityDefaultsExact: true, identityDefaultNormalization: normalization, otherMetadataRowsExact: true,
originalCredentialRevision: connection?.credentialRevision, finalCredentialRevision: after.modelCenter.connections.find(value => value.id === connectionId)?.credentialRevision,
changedOnlyMonotonicRevisionsObservedHealthAndDeclaredIdentityDefaults: true, pid: after.pid, version: after.version, noKeyExtractedOrResaved: true });
}
save('node/result.json', result ?? { success: false, failure, noModelDispatch: true }); await close();
}
console.log(JSON.stringify({ success: result.success, attempts: result.attempts, usage: result.usage, state: result.state,
stopReason: result.stopReason, commit: result.commit, tests: result.tests, error: result.failure?.message }));
+76
View File
@@ -0,0 +1,76 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { publicSnapshot, hash } from './state.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
import { root, origin, dataRoot, connectionId, save, meta, client, captureAttempts, readRoot, experiment } from './common.mjs';
// Single no-tool provider identity preflight. One request max, strict identity, stop on failure.
assert.equal(fs.existsSync(path.join(root, 'preflight')), false, 'Never replay or overwrite a real sample.');
const original = readRoot();
assert.equal(experiment.key, 'deepseek');
const nativeParameters = experiment.parameters;
const prefix = 'preflight';
const { assertBatchBudget } = await import('./common.mjs');
assertBatchBudget();
const { api, boot, close } = await client();
let fatal;
try {
await inspectorClosed();
assert.equal(boot.version, '1.25.3');
const host = await api('/host/status'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false);
const before = await publicSnapshot(origin, dataRoot()); save(prefix + '/before.json', before);
assert.deepEqual(before.active, []);
const connection = before.modelCenter.connections.find(value => value.id === connectionId);
assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.credentialRevision, experiment.credentialRevision);
assert.equal(connection.protocol, experiment.protocol); assert.equal(connection.model, experiment.model);
assert.equal(connection.revision, original.connectionRevision);
assert.equal(connection.contextLimit, 16000); assert.equal(connection.modelIdentityPolicy ?? 'record', 'record');
const originalAttempts = meta('modelAttempts') ?? []; save(prefix + '/original-attempts.json', originalAttempts);
const targetName = connection.name + ' · low';
const nonce = randomUUID(), prompt = `这是独立模型连接身份预检,不是工程任务。只回复 OK ${nonce},不要调用工具、读写文件或执行命令。`;
save(prefix + '/contract.json', { kind: 'single-real-provider-identity-preflight', connectionId, connection,
version: boot.version, runtimeRoot: boot.runtimeRoot, maxRequests: 1, maxAttempts: 1, maxOutputTokens: 2048, nativeParameters,
maxTokens: 16000, contextLimit: 16000, toolDefinitions: 0, toolExecution: 0,
poolSwitching: false, automaticReplay: false, modelIdentityPolicy: 'require-match in this experiment snapshot only',
noGlobalConfigurationWrite: true, noCredentialExtractionOrResave: true, prompt, nonce,
originalBaselineSha256: hash(fs.readFileSync(path.join(root, prefix + '/before.json'))) });
const projectRoot = path.join(root, prefix + '/identity-only'); fs.mkdirSync(projectRoot);
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# AB provider identity preflight\nNo engineering or command execution.\n', { flag: 'wx' });
const project = await api('/projects', { root: projectRoot });
const session = await api('/sessions', { projectId: project.id }); save(prefix + '/owned.json', { project, session });
const input = { pid: host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, sessionId: session.id, projectId: project.id,
nativeParameters, connectionId, connectionRevision: connection.revision, credentialRevision: connection.credentialRevision,
baseUrl: connection.baseUrl, model: connection.model, protocol: connection.protocol, title: `AB · ${targetName}单次预检`, prompt };
save(prefix + '/input.json', input);
console.log(JSON.stringify({ stage: 'single-provider-preflight', connectionId, model: connection.model, protocol: connection.protocol, baseUrl: connection.baseUrl, maxRequests: 1, toolDefinitions: 0 }));
const invoked = await invokeProbe(input); save(prefix + '/expression.js', invoked.expression); save(prefix + '/host-result.json', invoked.result);
const previous = new Set(originalAttempts.map(value => value.id));
const attempts = (meta('modelAttempts') ?? []).filter(value => !previous.has(value.id)); save(prefix + '/attempts.json', attempts);
assert.ok(attempts.length <= 1); for (const attempt of attempts) { assert.equal(attempt.sessionId, session.id); assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.poolId, null); }
const state = await api('/sessions/' + session.id); save(prefix + '/session.json', state);
save(prefix + '/export.json', await api('/sessions/' + session.id + '/export'));
const captures = await captureAttempts(api, attempts, prefix + '/captures');
const attempt = attempts[0];
const passed = attempts.length === 1 && attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched'
&& attempt.modelIdentity.complete === true && attempt.modelIdentity.policy === 'require-match' && attempt.responseRedacted === false
&& attempt.usageStatus === 'reported' && attempt.usage.total_tokens <= 16000 && captures.length === 2
&& invoked.result.actualTools === 0 && state.status === 'completed' && invoked.result.message?.content?.includes(nonce);
const result = { passed, engineeringTestAllowed: passed, testedAt: new Date().toISOString(), version: boot.version, nativeParameters, reasoningMode: 'on-low',
connectionId, sessionId: session.id, projectId: project.id, taskId: invoked.result.taskId,
requestedModel: connection.model, reportedModels: attempt?.modelIdentity?.reportedModels ?? [], identity: attempt?.modelIdentity,
requestUrl: attempt?.requestUrl, requests: attempts.length, usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched',
actualTools: invoked.result.actualTools, stopReason: invoked.result.stopReason, captures,
noRetryOrPoolSwitch: attempts.every(value => value.ordinal === 1 && value.poolId === null),
modelWeightsAuthenticated: false, engineeringDeliveryTested: false, purgeBenefitTested: false,
defaultSelectionChanged: false, connectionConfigChanged: false, credentialValuesReadOrSaved: false };
save(prefix + '/result.json', result); console.log(JSON.stringify({ passed, requests: result.requests, usage: result.usage, reportedModels: result.reportedModels, stopReason: result.stopReason }));
} catch (error) {
fatal = { name: error.name, message: error.message, stack: error.stack, recordedAt: new Date().toISOString(), automaticReplay: false };
save(prefix + '/driver-error.json', fatal); process.exitCode = 1;
} finally {
save(prefix + '/after.json', await publicSnapshot(origin, dataRoot()));
save(prefix + '/inspector-final.json', { closed: await inspectorClosed() });
await close();
}
if (fatal) console.error(fatal.message);
+1
View File
@@ -0,0 +1 @@
export * from '../live-aa/state.mjs';
+43
View File
@@ -0,0 +1,43 @@
// Offline preservation and wire checks for one already-completed AB preflight.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { root, save, sha, experiment } from './common.mjs';
const mode = 'on-low';
const prefix = 'preflight';
const read = file => JSON.parse(fs.readFileSync(path.join(root, prefix, file)));
const before = read('before.json'), after = read('after.json'), result = read('result.json');
const owned = read('owned.json'), attempts = read('attempts.json'), session = read('session.json');
for (const field of ['pid', 'version', 'runtimeRoot', 'hasKey', 'configSha256']) assert.equal(after[field], before[field], field);
assert.deepEqual(before.active, []); assert.deepEqual(after.active, []);
assert.equal(after.projects, before.projects + 1); assert.equal(after.exports.length, before.exports.length + 1);
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item);
for (const table of ['sessions', 'records', 'events', 'outbox']) {
assert.equal(after.database.tables[table].schemaSha256, before.database.tables[table].schemaSha256);
const rows = new Map(after.database.tables[table].rows.map(value => [value.id, value]));
for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, table + '/' + row.id);
}
const metadata = new Map(after.database.tables.metadata.rows.map(value => [value.id, value]));
for (const row of before.database.tables.metadata.rows.filter(value => value.id !== 'modelAttempts')) assert.deepEqual(metadata.get(row.id), row);
const artifacts = new Map(after.persistent.artifacts.map(value => [value.path, value]));
for (const row of before.persistent.artifacts) assert.deepEqual(artifacts.get(row.path), row);
assert.deepEqual(after.modelCenter.selected, before.modelCenter.selected); assert.deepEqual(after.modelCenter.pools, before.modelCenter.pools);
assert.equal(attempts.length, result.requests); assert.ok(attempts.length <= 1);
for (const attempt of attempts) {
assert.equal(attempt.sessionId, owned.session.id); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
assert.equal(attempt.model, experiment.model); assert.equal(attempt.protocol, experiment.protocol);
}
assert.equal(result.actualTools, 0); assert.equal(session.execution.run.toolCalls, 0); assert.equal(read('inspector-final.json').closed, true);
for (const capture of result.captures) {
const bytes = fs.readFileSync(path.join(root, capture.path)); assert.equal(bytes.length, capture.bytes); assert.equal(sha(bytes), capture.sha256);
if (capture.kind === 'request') {
const request = JSON.parse(bytes); assert.equal(request.max_tokens, 2048); assert.ok(!request.tools?.length);
for (const [key, expected] of Object.entries(experiment.parameters)) assert.deepEqual(request[key], expected);
}
}
const verification = { passed: true, route: experiment.key, reasoningMode: mode, modelPassed: result.passed,
originalExportsExact: before.exports.length, originalArtifactFilesExact: before.persistent.artifacts.length,
originalRowsExact: ['sessions', 'records', 'events', 'outbox'], metadataExactExceptNewAttempt: true,
budgetsSelectionPoolsAndCredentialsUnchanged: true, originalPidPreserved: after.pid, inspectorClosed: true,
preflightRequests: result.requests, reportedTokens: result.usage?.total_tokens ?? null, toolsExecuted: 0, verificationModelRequests: 0 };
save(prefix + '/verification.json', verification); console.log(JSON.stringify(verification));
+9
View File
@@ -1,5 +1,14 @@
# Agent Workbench:核心代理能力研究与产品计划
## 当前:AB 已验证低强度模式的新工程验证
AA已封存提交6726ce14,原关闭门不满足且不续跑。AB使用同revision3/credential1的DeepSeek开启低强度,新冻结/新单请求/新Node;AA+AB合计100请求700000tokens,计入AA未知预留2235,不扩大总额。
1. [in_progress] 冻结AB计划/驱动、核对AAmanifest和正式原状态;一次新诊断/单请求预检。
2. [pending] 新Node24请求140000tokens、原九测试/build/模型变更集/审查与本地Git。
3. [pending] 具前置后React/Python和原版Purge ABBA,否则明确未执行。
4. [pending] 原状态恢复核对、异常与未测保留、离线封存/阶段回顾及本地提交。
## 当前:AA 真实兼容、自主交付与 Purge 验证
用户明确要求进入真实验证,保留历史异常。计划docs/LIVE-VALIDATION-AA-PLAN.md;原1.25.3/PID36552/Provider/Key保持,不续跑旧样本、不委派/推送。