Files
Agent-Workbench/docs/milestone-audit-deployment-verifier.mjs

122 lines
12 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
import { upgradeRunning } from '../scripts/upgrade-running.mjs';
const targetVersion = '1.12.1', previousVersion = '1.12.0';
const read = filename => JSON.parse(fs.readFileSync(filename, 'utf8'));
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
const digest = value => hash(JSON.stringify(canonical(value)));
const release = read('releases/latest.json'), source = read('.qa/source-package-validation.json'), archive = read('.qa/archives-validation.json'), portable = read('.qa/release-validation.json');
const preflight = read('docs/milestone-audit-service-preflight.json');
assert.equal(preflight.version, previousVersion); assert.equal(preflight.registeredPid, preflight.pid); assert.equal(preflight.busySessions, 0);
assert.ok(Number.isInteger(preflight.pid) && preflight.pid > 0);
for (const [name, result] of Object.entries({ source, archive, portable })) { assert.equal(result.success, true, name); assert.equal(result.version, targetVersion, name); }
assert.equal(read(path.join(release.portableRoot, 'MANIFEST.json')).version, targetVersion);
assert.equal(path.resolve(portable.appRoot), path.resolve(release.portableRoot));
assert.ok(source.tests.tests > 357); assert.equal(source.tests.pass, source.tests.tests);
for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(source.tests[name], 0);
assert.equal(source.buildMatchedFiles, 6); assert.ok(source.actualRea.success && source.actualEngine.success && source.actualBrowser.success);
for (const kind of ['portable', 'source']) {
assert.equal(archive.archives.find(item => item.kind === kind).sha256, release[kind].sha256);
assert.equal(hash(fs.readFileSync(release[kind].path)), release[kind].sha256);
}
for (const input of source.inputs) {
for (const root of [process.cwd(), release.sourceRoot]) {
const bytes = fs.readFileSync(path.join(root, input.path)); assert.equal(bytes.length, input.bytes, input.path); assert.equal(hash(bytes), input.sha256, input.path);
}
}
const files = (root, relative) => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))
.flatMap(entry => { assert.equal(entry.isSymbolicLink(), false); const filename = path.join(relative, entry.name); return entry.isDirectory() ? files(root, filename) : [filename]; });
const compared = [], repackedMetadata = [];
function compare(leftRoot, rightRoot, relative) {
const names = files(leftRoot, relative); assert.deepEqual(names, files(rightRoot, relative));
for (const filename of names) {
const left = fs.readFileSync(path.join(leftRoot, filename)), right = fs.readFileSync(path.join(rightRoot, filename));
if (!left.equals(right) && filename.replaceAll('\\', '/') === 'runtime/REA-RUNTIME.json') {
const { builtAt: leftBuiltAt, ...a } = JSON.parse(left), { builtAt: rightBuiltAt, ...b } = JSON.parse(right);
assert.ok(Number.isFinite(Date.parse(leftBuiltAt)) && Number.isFinite(Date.parse(rightBuiltAt))); assert.deepEqual(a, b);
repackedMetadata.push({ path: filename.replaceAll('\\', '/'), leftBuiltAt, rightBuiltAt, leftSha256: hash(left), rightSha256: hash(right), difference: 'builtAt only; every other field/hash equal' });
} else assert.ok(left.equals(right), filename);
compared.push(filename.replaceAll('\\', '/'));
}
}
for (const folder of ['src', 'tests', 'scripts', 'public', 'benchmarks', 'runtime']) compare(release.sourceRoot, source.sourceRoot, folder);
for (const folder of ['dist-client', 'dist-server']) { compare(release.portableRoot, source.sourceRoot, folder); compare(process.cwd(), release.portableRoot, folder); }
const ui = read('docs/milestone-audit-all-ui-validation.json'), security = read('docs/milestone-audit-security-ui-validation.json');
assert.equal(ui.success, true); assert.equal(ui.version, targetVersion); assert.equal(ui.groups, 150); assert.equal(ui.results.length, 20);
assert.equal(security.success, true); assert.equal(security.assertions.length, 8); assert.equal(security.realModelRequests, 0);
const directory = path.resolve('.qa/milestone-deployment-' + randomUUID()); fs.mkdirSync(directory);
const url = 'http://127.0.0.1:4318'; let token;
async function api(route) {
const response = await fetch(url + '/api' + route, { headers: { Connection: 'close', ...(token ? { 'X-Workbench-Token': token } : {}) }, signal: AbortSignal.timeout(20000) });
assert.ok(response.ok, route + ': HTTP ' + response.status); return await response.json();
}
const before = await api('/bootstrap'), health = await api('/health'); token = before.token;
assert.equal(before.version, previousVersion); assert.equal(health.pid, preflight.pid); assert.equal(typeof before.config.hasKey, 'boolean');
assert.ok(before.sessions.every(value => !['running', 'awaiting_approval', 'stopping'].includes(value.status)));
const info = read(path.join(process.env.LOCALAPPDATA, 'AgentWorkbench/service-info.json')); assert.equal(info.pid, health.pid);
assert.equal(fs.realpathSync.native(info.appRoot), fs.realpathSync.native(preflight.appRoot));
assert.equal(fs.realpathSync.native(before.runtimeRoot), fs.realpathSync.native(preflight.runtimeRoot));
assert.equal(fs.realpathSync.native(info.dataRoot), fs.realpathSync.native(preflight.dataRoot));
const modelCenterBefore = await api('/model-center');
assert.equal(modelCenterBefore.waiting, 0); assert.ok(modelCenterBefore.connections.every(value => value.activeRequests === 0));
function snapshotDb() {
const db = new DatabaseSync(path.join(info.dataRoot, 'workbench.sqlite'), { readOnly: true }); db.exec('BEGIN');
try {
const tables = {};
for (const [table, order] of [['projects', 'id'], ['sessions', 'id'], ['records', 'kind,id'], ['events', 'seq']]) {
const rows = db.prepare('SELECT * FROM ' + table + ' ORDER BY ' + order).all();
tables[table] = { count: rows.length, sha256: digest(rows), rows: rows.map(row => ({ id: row.id, ...(row.kind ? { kind: row.kind } : {}), sha256: digest(row) })) };
}
const sessions = db.prepare("SELECT id,json_extract(payload,'$.archived') AS archived FROM sessions ORDER BY id").all();
const recordKinds = db.prepare('SELECT kind,COUNT(*) AS count FROM records GROUP BY kind ORDER BY kind').all();
return { tables, sessions, recordKinds };
} finally { db.exec('ROLLBACK'); db.close(); }
}
const beforeDb = snapshotDb(), exports = [];
for (const brief of beforeDb.sessions) { const value = await api('/sessions/' + brief.id + '/export'); exports.push({ id: brief.id, session: digest(value.session), project: digest(value.project), execution: digest(value.execution) }); }
fs.writeFileSync(path.join(directory, 'before.json'), JSON.stringify({ version: before.version, pid: health.pid, configSha256: digest(before.config), projectsSha256: digest(before.projects), sessionsSha256: digest(before.sessions), modelCenterSha256: digest(modelCenterBefore), database: beforeDb, exports }, null, 2));
assert.deepEqual(snapshotDb(), beforeDb, 'User state changed during preflight; do not use stale snapshots.');
console.log(JSON.stringify({ stage: 'pre-upgrade', version: before.version, pid: health.pid, hasKey: before.config.hasKey,
counts: Object.fromEntries(Object.entries(beforeDb.tables).map(([name, value]) => [name, value.count])), allSessions: beforeDb.sessions.length, inputs: source.inputFiles, comparedFiles: compared.length }));
const upgraded = await upgradeRunning({ url, pid: health.pid, currentRoot: info.appRoot, ...(before.runtimeRoot ? { handlerRoot: before.runtimeRoot } : {}), newRoot: release.portableRoot });
fs.writeFileSync(path.join(directory, 'upgrade.json'), JSON.stringify(upgraded, null, 2));
assert.equal(upgraded.version, targetVersion); assert.equal(upgraded.verified, true); assert.equal(upgraded.connectionPreserved, true);
const after = await api('/bootstrap'), afterHealth = await api('/health');
assert.equal(afterHealth.pid, health.pid); assert.equal(digest(after.config), digest(before.config));
assert.equal(digest(after.projects), digest(before.projects)); assert.equal(digest(after.sessions), digest(before.sessions));
assert.equal(digest(await api('/model-center')), digest(modelCenterBefore), 'All public connections, pools, imports, credential flags and attempts must remain unchanged.');
assert.deepEqual(snapshotDb(), beforeDb, 'Every persisted row must remain unchanged.');
for (const saved of exports) { const value = await api('/sessions/' + saved.id + '/export'); assert.equal(digest(value.session), saved.session); assert.equal(digest(value.project), saved.project); assert.equal(digest(value.execution), saved.execution); }
await assert.rejects(fetch('http://127.0.0.1:9229/json/list', { signal: AbortSignal.timeout(750) }));
const headers = Object.fromEntries((await fetch(url)).headers); assert.equal(headers['x-frame-options'], 'DENY'); assert.match(headers['content-security-policy'], /frame-ancestors 'none'/);
const rea = await api('/research/availability'); assert.equal(rea.status, 'available'); assert.equal(rea.version, '6.8.0');
assert.equal(fs.realpathSync.native(rea.packageRoot), fs.realpathSync.native(path.join(release.portableRoot, 'runtime/node_modules/rea-agents')));
assert.equal((await api('/engines/deepseek/availability')).status, 'available');
assert.equal((await api('/sessions/' + before.sessions[0].id + '/browser')).availability.status, 'available');
const result = { success: true, version: targetVersion, previousVersion, testedAt: new Date().toISOString(), ...upgraded,
directory, appRoot: release.portableRoot, dataRoot: info.dataRoot, originalAppRoot: info.appRoot,
model: after.config.model, hasKey: after.config.hasKey, allSessionCount: beforeDb.sessions.length, visibleSessionCount: before.sessions.length,
archivedSessionCount: beforeDb.sessions.filter(value => value.archived).length,
database: Object.fromEntries(Object.entries(beforeDb.tables).map(([name, value]) => [name, { count: value.count, sha256: value.sha256 }])),
recordKinds: beforeDb.recordKinds, preservedSessionSnapshots: exports, allRowsAndExportsPreserved: true, inspectorClosed: true,
comparedFiles: compared.length, repackedMetadata, frozenInputs: source.inputFiles, independentSourceRoot: source.sourceRoot,
independentSourceTests: source.tests, uiGroups: ui.groups, securityGroups: security.assertions.length, realModelRequests: 0, headers,
comparison: 'Exact canonical hashes of every persisted row and exported project/session/execution, with no omitted compatibility fields.',
preflight, historicalReleasePid: preflight.historicalReleasePid, preservesCurrentObservedProcess: true,
modelCenterPreserved: true, modelCenter: { sha256: digest(modelCenterBefore), connections: modelCenterBefore.connections.length,
pools: modelCenterBefore.pools.length, imports: modelCenterBefore.imports.length, attempts: modelCenterBefore.attempts.length },
credentialsHandling: 'Only public configuration/hasKey and data hashes; original Provider/credentials stay in the process. No model key read/copied/printed/saved and no model request.' };
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
fs.writeFileSync('docs/milestone-audit-upgrade-live-validation.json', JSON.stringify(result, null, 2) + '\n');
fs.writeFileSync('docs/deployment.json', JSON.stringify({ ready: true, url, version: targetVersion, mode: after.config.mode, model: after.config.model, hasKey: after.config.hasKey,
pid: health.pid, appRoot: release.portableRoot, originalAppRoot: info.appRoot, dataRoot: info.dataRoot, installedAt: new Date().toISOString(), verification: 'milestone-audit-upgrade-live-validation.json' }, null, 2) + '\n');
console.log(JSON.stringify({ success: true, version: targetVersion, pid: health.pid, hasKey: result.hasKey, directory, allSessions: result.allSessionCount,
counts: Object.fromEntries(Object.entries(result.database).map(([name, value]) => [name, value.count])), allRowsAndExportsPreserved: true, inspectorClosed: true, realModelRequests: 0 }));