Files
Agent-Workbench/scripts/verify-controlled-release.mjs
T

200 lines
21 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import http from 'node:http';
import assert from 'node:assert/strict';
import { spawn, execFileSync } from 'node:child_process';
import { createHash, randomUUID } from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { gitEnvironment } from '../src/core/git-process.mjs';
import { assertToolPairs, call } from '../tests/helpers.mjs';
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
const wait = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds));
async function until(predicate, timeout = 45000) {
const deadline = Date.now() + timeout;
while (!await predicate()) { if (Date.now() > deadline) throw new Error('Owned compiled-engine acceptance timed out.'); await wait(50); }
}
// This verifier supplies a declared local response algorithm. The executing
// Runtime/Store/permissions/native bridge all belong to the compiled host.
export async function verifyControlledApi({ api, url, token, directory, appRoot, contained = false }) {
const origin = new URL(url);
assert.equal(origin.hostname, '127.0.0.1'); assert.notEqual(origin.port, '4318');
const expectedVersion = JSON.parse(fs.readFileSync(path.join(appRoot, fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json'), 'utf8')).version;
assert.ok(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(expectedVersion));
const bootstrap = await api('/bootstrap'); assert.equal(bootstrap.version, expectedVersion);
assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
const root = path.join(directory, 'compiled-controlled-node'), hooks = path.join(directory, 'disabled-hooks');
fs.mkdirSync(root); fs.mkdirSync(hooks);
fs.cpSync(path.join(appRoot, 'benchmarks/workers-delivery-v1/node'), root, { recursive: true });
fs.writeFileSync(path.join(root, '.gitignore'), 'dist/\nnode_modules/\n.awb-*\n');
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) fs.writeFileSync(path.join(root, name), 'Owner baseline\n');
const git = args => execFileSync('git', ['-c', 'core.hooksPath=' + hooks, ...args], { cwd: root, env: gitEnvironment(), encoding: 'utf8', windowsHide: true, timeout: 15000 });
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); git(['config', 'user.name', 'Workbench QA']); git(['config', 'user.email', 'qa@workbench.invalid']);
git(['add', '--', '.']); git(['commit', '--no-gpg-sign', '-m', 'Seal compiled controlled-engine acceptance']);
const initialHead = git(['rev-parse', 'HEAD']).trim(), writable = ['src/add.mjs', 'src/range.mjs'];
const protectedFiles = Object.fromEntries(git(['ls-files', '-z']).split('\0').filter(name => name && !writable.includes(name) && !name.startsWith('owner-'))
.map(name => [name, fs.readFileSync(path.join(root, name))]));
fs.writeFileSync(path.join(root, 'owner-staged.txt'), 'Owner staged change\n'); git(['add', '--', 'owner-staged.txt']);
fs.writeFileSync(path.join(root, 'owner-staged.txt'), 'Owner working copy after staging\n'); fs.writeFileSync(path.join(root, 'owner-unstaged.txt'), 'Owner unstaged change\n');
fs.writeFileSync(path.join(root, 'owner-untracked.txt'), 'Owner untracked file\n');
const owner = () => ({ staged: git(['show', ':owner-staged.txt']), stagedDiff: git(['diff', '--cached', '--binary', '--', 'owner-staged.txt']),
files: Object.fromEntries(['owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(root, name)).toString('base64')])) });
const originalOwner = owner(), verifyProtected = target => { for (const [name, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(target, name)), bytes, name); };
const requests = [], reviews = [], cases = [], originalRuns = [], exports = [];
const mark = text => { cases.push(text); console.log('PASS ' + text); };
let phase = 'new', childRoute, parentRoute, group;
const modelServer = http.createServer(async (request, response) => {
try {
assert.equal(request.url, '/v1/chat/completions'); assert.equal(request.headers.authorization, 'Bearer owned-compiled-engine-fixture');
const chunks = []; for await (const chunk of request) chunks.push(chunk);
const input = JSON.parse(Buffer.concat(chunks)); assert.equal(input.model, 'owned-compiled-engineering'); assertToolPairs(input.messages);
const session = await api(childRoute), job = session.execution.checks.at(-1), plan = session.execution.plans.at(-1);
let calls = [];
if (session.changes.some(value => value.status === 'pending')) { /* Host review follows native completion. */ }
else if (phase !== 'fixed' && job) {
assert.equal(job.status, 'failed'); assert.ok(job.results.some(value => value.tests?.failed > 0)); phase = 'fixed';
const fixes = { 'src/add.mjs': 'export function add(left, right) {\n if (!Number.isFinite(left) || !Number.isFinite(right)) throw new TypeError("finite numeric inputs required");\n return left + right;\n}\n',
'src/range.mjs': 'export function inclusiveRange(start, end) {\n if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || end < start || end - start >= 10000) throw new RangeError("bounded ascending integer range required");\n return Array.from({ length: end - start + 1 }, (_, index) => start + index);\n}\n' };
calls = Object.entries(fixes).map(([name, content]) => call(randomUUID(), 'propose_file', { path: name, content, reason: 'Declared local acceptance repair; retain protected tests and configuration.' }));
} else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, 8);
else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: ['test', 'build'] })];
else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })];
requests.push({ input, phase, calls });
response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: calls.length ? '' : 'Owned compiled native run complete; host proof remains required.',
...(calls.length ? { tool_calls: calls } : {}) }, finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 123, completion_tokens: 45, total_tokens: 168 } }));
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
});
await new Promise(resolve => modelServer.listen(0, '127.0.0.1', resolve));
try {
await api('/config', { mode: 'compatible', model: 'owned-compiled-engineering', baseUrl: 'http://127.0.0.1:' + modelServer.address().port + '/v1', apiKey: 'owned-compiled-engine-fixture', contextLimit: 64000 });
const project = await api('/projects', { root }), parent = await api('/sessions', { projectId: project.id }); parentRoute = '/sessions/' + parent.id;
const installation = await api('/engines/deepseek/controlled-availability'); assert.equal(installation.controlledStatus, 'available', installation.message);
assert.equal(installation.version, '0.2.1-rc.2');
if (contained) assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
const source = await api('/engines/purge/source'); assert.equal(source.status, 'available', source.message); assert.equal(source.defaultEnabled, false);
const planned = await api(parentRoute + '/collaborations', { goal: 'Deliver compiled controlled-engine Node library with retained failures, review, checks and Git.',
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256,
strategy: { kind: 'purge-prompt-experiment', root: source.root, sourceSha256: source.sourceSha256, prompt: '保留原测试与 {{compiled_literal}},按实际失败修复已允许源码,等待宿主审批。' } },
budget: { maxTokens: 200000, maxSteps: 64, maxToolCalls: 128, maxDurationMs: 600000 },
children: [{ label: 'engine-compiled-node', goal: 'Repair two source modules and pass the original 8 tests and build.', paths: writable }] });
group = planned.group;
const groupState = async () => (await api(parentRoute + '/collaborations')).groups.find(value => value.id === group.id);
const operation = name => api(parentRoute + '/collaborations/' + group.id + '/' + name, { fingerprint: group.fingerprint });
await operation('prepare'); await until(async () => { group = await groupState(); return !['preparing', 'planned'].includes(group.status); });
assert.equal(group.status, 'ready', group.summary); const child = group.children[0]; childRoute = '/sessions/' + child.sessionId;
const childProject = (await api('/bootstrap')).projects.find(value => value.id === child.projectId);
assert.notEqual(childProject.root, root); verifyProtected(childProject.root); assert.deepEqual(owner(), originalOwner); assert.equal(requests.length, 0);
mark('compiled host seals actual Harness/source identities and prepares one owned worktree while retaining parent bytes and staging');
const apply = async (route, set) => { reviews.push({ action: 'apply reviewed change set', id: set.id, fingerprint: set.fingerprint, paths: set.files.map(value => value.path) });
await api(route + '/change-sets/' + set.id + '/apply', { fingerprint: set.fingerprint });
let state; await until(async () => { state = await api(route + '/change-sets/' + set.id); return !['prepared', 'applying'].includes(state.status); }); assert.equal(state.status, 'completed', state.summary); };
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
let current = await api(childRoute); const pending = current.changes.filter(value => value.status === 'pending');
if (pending.length) { assert.equal(group.status, 'awaiting_review'); const next = await api(childRoute + '/change-sets', { changeIds: pending.map(value => value.id) }); await apply(childRoute, next.execution.changeSets.at(-1)); }
await operation('start');
await until(async () => {
current = await api(childRoute);
for (const approval of current.approvals.filter(value => value.status === 'pending')) {
const event = current.events.find(value => value.id === approval.eventId); assert.equal(event.tool, 'run_checks');
const plan = current.execution.plans.at(-1); assert.deepEqual(plan.checks.map(value => value.name), ['test', 'build']);
reviews.push({ action: 'approve original check', id: approval.id, taskId: approval.taskId, invocationId: approval.invocationId, planId: plan.id, contractSha256: plan.contractSha256 });
await api(childRoute + '/approvals/' + approval.id, { allow: true });
}
group = await groupState(); return ['ready_to_collect', 'awaiting_review', 'paused', 'unknown', 'failed', 'stale'].includes(group.status);
}, 120000);
const run = (await api(childRoute + '/engine-runs')).at(-1); assert.equal(run.status, 'completed', run.error); assert.equal(run.nativeSummary.treeConfirmed, true);
const response = await fetch(url + '/api' + childRoute + '/engine-runs/' + run.id + '/download', { headers: { 'X-Workbench-Token': token, Connection: 'close' } });
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), run.reportSha256);
const original = JSON.parse(bytes); assert.equal(original.native.stderr, ''); assert.equal(original.native.cleanup.treeConfirmed, true);
assert.equal(original.native.activity.endEvents[0].data.reason.kind, 'completed');
assert.equal(original.native.readiness.promptExperimentSha256, run.installation.strategy.sha256);
const saved = path.join(directory, 'compiled-engine-original-' + run.id + '.json'); fs.writeFileSync(saved, bytes); originalRuns.push({ run, original, reportPath: saved, sha256: sha(bytes) });
}
assert.equal(group.status, 'ready_to_collect', group.summary); const current = await api(childRoute);
assert.ok(current.execution.checks.some(value => value.status === 'failed' && value.results.some(result => result.tests?.failed > 0)));
assert.equal(current.execution.checks.at(-1).status, 'passed'); assert.equal(current.execution.checks.at(-1).results[0].tests.passed, 8);
assert.equal(current.execution.task.usage.requests, requests.length); assert.equal(current.execution.task.usage.totalTokens, requests.length * 168);
for (const request of requests) assert.equal(request.input.messages.filter(value => value.role === 'system').map(value => value.content).join('').split(group.children[0].engine.strategy.section.text).length - 1, 1);
verifyProtected(childProject.root); assert.deepEqual(owner(), originalOwner);
mark('compiled native execution retains failing tests, requires original check approvals, seals literal prompt/model/receipts/results and accounts each owned HTTP request once');
await operation('collect'); await until(async () => { group = await groupState(); return group.status !== 'collecting'; }); assert.equal(group.status, 'collected', group.summary);
const collection = await api(parentRoute + '/collaborations/' + group.id + '/collection'); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
await operation('merge'); await until(async () => { group = await groupState(); return ['merge_prepared', 'unknown', 'stale'].includes(group.status); }); assert.equal(group.status, 'merge_prepared', group.summary);
await apply(parentRoute, (await api(parentRoute + '/change-sets/' + group.merge.changeSetId)));
const checkPlan = (await api(parentRoute + '/check-plans', { names: ['test', 'build'] })).execution.plans.at(-1);
const accepted = (await api(parentRoute + '/check-jobs', { planId: checkPlan.id })).execution.checks.at(-1); let check;
await until(async () => { check = await api(parentRoute + '/check-jobs/' + accepted.id); return !['queued', 'running'].includes(check.status); });
assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, 8); verifyProtected(root);
mark('compiled parent review and immutable merge pass the original 8 functional tests and actual ESM build');
const deliveryPlan = (await api(parentRoute + '/delivery-plans', { message: 'feat: deliver compiled controlled-engine Node repair' })).execution.deliveryPlans.at(-1);
assert.deepEqual(deliveryPlan.paths, writable);
const deliveryId = (await api(parentRoute + '/delivery-jobs', { planId: deliveryPlan.id })).execution.deliveries.at(-1).id; let delivery;
await until(async () => { delivery = await api(parentRoute + '/delivery-jobs/' + deliveryId); return ['completed', 'failed', 'cancelled', 'unknown'].includes(delivery.status); });
assert.equal(delivery.status, 'completed', delivery.summary); assert.equal(git(['rev-parse', 'HEAD^']).trim(), initialHead);
assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split('\n'), writable);
assert.deepEqual(owner(), originalOwner); verifyProtected(root); await operation('finalize'); group = await groupState(); assert.equal(group.status, 'completed', group.summary);
for (const route of [parentRoute, childRoute]) { const value = await api(route); const file = path.join(directory, value.id + '-session.json'); fs.writeFileSync(file, JSON.stringify(value, null, 2)); exports.push({ id: value.id, file, sha256: sha(fs.readFileSync(file)) }); }
mark('compiled Git delivery contains only reviewed modules, preserves staged/unstaged/untracked owner bytes and completes the parent with sealed original evidence');
return { success: true, version: bootstrap.version, appRoot, contained, cases, root, group, requests, reviews, originalRuns, exports,
protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([name, bytes]) => [name, sha(bytes)])), owner: originalOwner, installation, purgeSource: source,
checks: check, delivery, fixtureModelRequests: requests.length, realModelRequests: 0,
scope: 'Actual compiled host, local native Harness, checks/reviews/Git and an explicitly owned HTTP response fixture. No configured model or upstream Purge plugin executes; real inference benefit remains unmeasured.' };
} catch (error) {
const sessions = [];
for (const route of [parentRoute, childRoute].filter(Boolean)) { try { sessions.push(await api(route)); } catch {} }
fs.writeFileSync(path.join(directory, 'compiled-api-failure.json'), JSON.stringify({ success: false, error: error.stack, cases, group, requests, reviews, originalRuns, sessions, realModelRequests: 0 }, null, 2));
throw error;
} finally {
if (group?.status === 'running') { try { await api(parentRoute + '/collaborations/' + group.id + '/stop', { fingerprint: group.fingerprint }); } catch {} }
modelServer.closeAllConnections(); await new Promise(resolve => modelServer.close(resolve));
}
}
export async function verifyControlledRelease(appRoot = process.cwd()) {
appRoot = fs.realpathSync.native(appRoot);
// Keep the synthetic Git project independent of the installation/extraction
// depth. The production workspace path guard remains responsible for users'
// actual project roots. Retain this exclusive directory as original evidence.
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'awb-controlled-'));
const node = fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath;
const env = { ...gitEnvironment(), ...Object.fromEntries(['AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_ROOT'].filter(name => process.env[name]).map(name => [name, process.env[name]])),
AGENT_WORKBENCH_APP_DIR: appRoot, AGENT_WORKBENCH_DATA_DIR: path.join(directory, 'data'), AGENT_WORKBENCH_PORT: '0' };
const child = spawn(node, [path.join(appRoot, 'dist-server/server.mjs')], { cwd: appRoot, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '', errors = '', url = '', result, bootstrap;
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
const api = async (route, input) => {
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', signal: AbortSignal.timeout(30000),
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const value = await response.json(); assert.equal(response.ok, true, value.error); return value;
};
try {
await until(() => url || child.exitCode !== null, 15000); assert.ok(url, errors);
bootstrap = await (await fetch(url + '/api/bootstrap', { signal: AbortSignal.timeout(5000) })).json();
const health = await api('/health'); assert.equal(health.pid, child.pid);
result = await verifyControlledApi({ api, url, token: bootstrap.token, directory, appRoot, contained: node !== process.execPath });
} catch (error) { result = { success: false, appRoot, directory, error: error.stack, realModelRequests: 0 }; throw error; }
finally {
try { if (url && bootstrap && child.exitCode === null) { await api('/shutdown', {}); await until(() => child.exitCode !== null || child.signalCode !== null, 15000); } }
finally {
if (child.exitCode === null && child.signalCode === null) {
if (process.platform === 'win32') { try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000 }); } catch { child.kill(); } } else child.kill();
}
fs.writeFileSync(path.join(directory, 'host-stdout-log.txt'), output); fs.writeFileSync(path.join(directory, 'host-stderr-log.txt'), errors);
Object.assign(result ??= { success: false }, { directory, pid: child.pid, rootExited: child.exitCode !== null || child.signalCode !== null, exit: { code: child.exitCode, signal: child.signalCode } });
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
console.log('EVIDENCE ' + directory);
}
}
assert.equal(result.rootExited, true); assert.equal(result.exit.code, 0); return result;
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
const args = process.argv.slice(2);
if (args.length === 1 && args[0] === '--help') { console.log('node scripts/verify-controlled-release.mjs [--app-root DIR] - run an actual compiled local host/native Harness/owned model/Node repair/check/review/Git/prompt acceptance in independent data. Requires explicit Harness and Purge source paths.'); process.exit(0); }
assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]), 'Unsupported argument; use --help.');
const result = await verifyControlledRelease(args[1]); console.log(JSON.stringify({ success: result.success, version: result.version, directory: result.directory, cases: result.cases.length,
rootExited: result.rootExited, fixtureModelRequests: result.fixtureModelRequests, realModelRequests: 0 }));
}