136 lines
14 KiB
JavaScript
136 lines
14 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { upgradeRunning } from '../scripts/upgrade-running.mjs';
|
|
|
|
const targetVersion = '1.14.0', previousVersion = '1.13.0';
|
|
const read = filename => JSON.parse(fs.readFileSync(filename, 'utf8'));
|
|
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
|
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
|
const digest = value => hash(JSON.stringify(canonical(value)));
|
|
const release = read('releases/latest.json'), source = read('.qa/source-package-validation.json'), archive = read('.qa/archives-validation.json'), portable = read('.qa/release-validation.json');
|
|
const preflight = read('docs/controlled-engine-service-preflight.json');
|
|
const finalBaseline = read('docs/controlled-engine-final-release-baseline.json'), portableQa = read('docs/controlled-engine-portable-qa-change.json');
|
|
assert.equal(portableQa.success, true); assert.equal(portableQa.change.path, 'scripts/verify-release.mjs');
|
|
assert.deepEqual(source.inputs, finalBaseline.sourceValidatedInputs);
|
|
assert.equal(preflight.version, previousVersion); assert.equal(preflight.registeredPid, preflight.pid); assert.equal(preflight.busySessions, 0);
|
|
assert.ok(Number.isInteger(preflight.pid) && preflight.pid > 0);
|
|
for (const [name, result] of Object.entries({ source, archive, portable })) { assert.equal(result.success, true, name); assert.equal(result.version, targetVersion, name); }
|
|
assert.equal(read(path.join(release.portableRoot, 'MANIFEST.json')).version, targetVersion);
|
|
assert.equal(path.resolve(portable.appRoot), path.resolve(release.portableRoot));
|
|
assert.ok(source.tests.tests > 404); assert.equal(source.tests.pass, source.tests.tests);
|
|
for (const name of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(source.tests[name], 0);
|
|
assert.equal(source.buildMatchedFiles, 6); assert.ok(source.actualRea.success && source.actualEngine.success && source.actualBrowser.success && source.actualControlled.success);
|
|
assert.equal(source.actualControlled.cases, 4); assert.equal(source.actualControlled.realModelRequests, 0); assert.equal(source.actualControlled.rootExited, true);
|
|
assert.equal(portable.actualControlled.success, true); assert.equal(portable.actualControlled.cases.length, 4); assert.equal(portable.actualControlled.realModelRequests, 0);
|
|
for (const kind of ['portable', 'source']) {
|
|
assert.equal(archive.archives.find(item => item.kind === kind).sha256, release[kind].sha256);
|
|
assert.equal(hash(fs.readFileSync(release[kind].path)), release[kind].sha256);
|
|
}
|
|
for (const input of finalBaseline.inputs) {
|
|
for (const root of [process.cwd(), release.sourceRoot]) {
|
|
const bytes = fs.readFileSync(path.join(root, input.path)); assert.equal(bytes.length, input.bytes, input.path); assert.equal(hash(bytes), input.sha256, input.path);
|
|
}
|
|
}
|
|
const files = (root, relative) => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))
|
|
.flatMap(entry => { assert.equal(entry.isSymbolicLink(), false); const filename = path.join(relative, entry.name); return entry.isDirectory() ? files(root, filename) : [filename]; });
|
|
const compared = [], repackedMetadata = [];
|
|
function compare(leftRoot, rightRoot, relative) {
|
|
const names = files(leftRoot, relative); assert.deepEqual(names, files(rightRoot, relative));
|
|
for (const filename of names) {
|
|
const left = fs.readFileSync(path.join(leftRoot, filename)), right = fs.readFileSync(path.join(rightRoot, filename));
|
|
if (!left.equals(right) && filename.replaceAll('\\', '/') === 'runtime/REA-RUNTIME.json') {
|
|
const { builtAt: leftBuiltAt, ...a } = JSON.parse(left), { builtAt: rightBuiltAt, ...b } = JSON.parse(right);
|
|
assert.ok(Number.isFinite(Date.parse(leftBuiltAt)) && Number.isFinite(Date.parse(rightBuiltAt))); assert.deepEqual(a, b);
|
|
repackedMetadata.push({ path: filename.replaceAll('\\', '/'), leftBuiltAt, rightBuiltAt, leftSha256: hash(left), rightSha256: hash(right), difference: 'builtAt only; every other field/hash equal' });
|
|
} else if (filename.replaceAll('\\', '/') === portableQa.change.path) {
|
|
assert.equal(hash(left), portableQa.change.sha256); assert.equal(hash(right), portableQa.change.originalSha256);
|
|
} else assert.ok(left.equals(right), filename);
|
|
compared.push(filename.replaceAll('\\', '/'));
|
|
}
|
|
}
|
|
for (const folder of ['src', 'tests', 'scripts', 'public', 'benchmarks', 'runtime']) compare(release.sourceRoot, source.sourceRoot, folder);
|
|
for (const folder of ['dist-client', 'dist-server']) { compare(release.portableRoot, source.sourceRoot, folder); compare(process.cwd(), release.portableRoot, folder); }
|
|
const ui = read('docs/controlled-engine-all-ui-validation.json'), projects = read('docs/controlled-engine-final-projects-validation.json');
|
|
assert.equal(ui.success, true); assert.equal(ui.version, targetVersion); assert.equal(ui.groups, 178); assert.equal(ui.results.length, 23);
|
|
assert.equal(projects.success, true); assert.equal(projects.testedVersion, targetVersion); assert.equal(projects.targetVersion, targetVersion); assert.equal(projects.results.length, 3); assert.equal(projects.realModelRequests, 0); assert.ok(projects.results.every(value => value.success));
|
|
|
|
const directory = path.resolve('.qa/controlled-engine-deployment-' + randomUUID()); fs.mkdirSync(directory);
|
|
const url = 'http://127.0.0.1:4318'; let token;
|
|
async function api(route) {
|
|
const response = await fetch(url + '/api' + route, { headers: { Connection: 'close', ...(token ? { 'X-Workbench-Token': token } : {}) }, signal: AbortSignal.timeout(20000) });
|
|
assert.ok(response.ok, route + ': HTTP ' + response.status); return await response.json();
|
|
}
|
|
const before = await api('/bootstrap'), health = await api('/health'); token = before.token;
|
|
assert.equal(before.version, previousVersion); assert.equal(health.pid, preflight.pid); assert.equal(typeof before.config.hasKey, 'boolean');
|
|
assert.ok(before.sessions.every(value => !['running', 'awaiting_approval', 'stopping'].includes(value.status)));
|
|
const info = read(path.join(process.env.LOCALAPPDATA, 'AgentWorkbench/service-info.json')); assert.equal(info.pid, health.pid);
|
|
assert.equal(fs.realpathSync.native(info.appRoot), fs.realpathSync.native(preflight.appRoot));
|
|
assert.equal(fs.realpathSync.native(before.runtimeRoot), fs.realpathSync.native(preflight.runtimeRoot));
|
|
assert.equal(fs.realpathSync.native(info.dataRoot), fs.realpathSync.native(preflight.dataRoot));
|
|
const modelCenterBefore = await api('/model-center');
|
|
assert.equal(modelCenterBefore.waiting, 0); assert.ok(modelCenterBefore.connections.every(value => value.activeRequests === 0));
|
|
function snapshotDb() {
|
|
const db = new DatabaseSync(path.join(info.dataRoot, 'workbench.sqlite'), { readOnly: true }); db.exec('BEGIN');
|
|
try {
|
|
const tables = {};
|
|
for (const { name: table } of db.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all()) {
|
|
const rows = db.prepare('SELECT * FROM "' + table.replaceAll('"', '""') + '" ORDER BY rowid').all();
|
|
tables[table] = { count: rows.length, sha256: digest(rows), rows: rows.map((row, index) => ({ id: row.id ?? row.key ?? row.seq ?? index, ...(row.kind ? { kind: row.kind } : {}), sha256: digest(row) })) };
|
|
}
|
|
const sessions = db.prepare("SELECT id,json_extract(payload,'$.archived') AS archived FROM sessions ORDER BY id").all();
|
|
const recordKinds = db.prepare('SELECT kind,COUNT(*) AS count FROM records GROUP BY kind ORDER BY kind').all();
|
|
return { tables, sessions, recordKinds };
|
|
} finally { db.exec('ROLLBACK'); db.close(); }
|
|
}
|
|
const beforeDb = snapshotDb(), exports = [];
|
|
for (const brief of beforeDb.sessions) { const value = await api('/sessions/' + brief.id + '/export'); exports.push({ id: brief.id, session: digest(value.session), project: digest(value.project), execution: digest(value.execution) }); }
|
|
fs.writeFileSync(path.join(directory, 'before.json'), JSON.stringify({ version: before.version, pid: health.pid, configSha256: digest(before.config), projectsSha256: digest(before.projects), sessionsSha256: digest(before.sessions), modelCenterSha256: digest(modelCenterBefore), database: beforeDb, exports }, null, 2));
|
|
assert.deepEqual(snapshotDb(), beforeDb, 'User state changed during preflight; do not use stale snapshots.');
|
|
console.log(JSON.stringify({ stage: 'pre-upgrade', version: before.version, pid: health.pid, hasKey: before.config.hasKey,
|
|
counts: Object.fromEntries(Object.entries(beforeDb.tables).map(([name, value]) => [name, value.count])), allSessions: beforeDb.sessions.length, inputs: source.inputFiles, comparedFiles: compared.length }));
|
|
const upgraded = await upgradeRunning({ url, pid: health.pid, currentRoot: info.appRoot, ...(before.runtimeRoot ? { handlerRoot: before.runtimeRoot } : {}), newRoot: release.portableRoot });
|
|
fs.writeFileSync(path.join(directory, 'upgrade.json'), JSON.stringify(upgraded, null, 2));
|
|
assert.equal(upgraded.version, targetVersion); assert.equal(upgraded.verified, true); assert.equal(upgraded.connectionPreserved, true);
|
|
const after = await api('/bootstrap'), afterHealth = await api('/health');
|
|
assert.equal(afterHealth.pid, health.pid); assert.equal(digest(after.config), digest(before.config));
|
|
assert.equal(digest(after.projects), digest(before.projects)); assert.equal(digest(after.sessions), digest(before.sessions));
|
|
assert.equal(digest(await api('/model-center')), digest(modelCenterBefore), 'All public connections, pools, imports, credential flags and attempts must remain unchanged.');
|
|
assert.deepEqual(snapshotDb(), beforeDb, 'Every persisted row must remain unchanged.');
|
|
for (const saved of exports) { const value = await api('/sessions/' + saved.id + '/export'); assert.equal(digest(value.session), saved.session); assert.equal(digest(value.project), saved.project); assert.equal(digest(value.execution), saved.execution); }
|
|
await assert.rejects(fetch('http://127.0.0.1:9229/json/list', { signal: AbortSignal.timeout(750) }));
|
|
const headers = Object.fromEntries((await fetch(url)).headers); assert.equal(headers['x-frame-options'], 'DENY'); assert.match(headers['content-security-policy'], /frame-ancestors 'none'/);
|
|
const rea = await api('/research/availability'); assert.equal(rea.status, 'available'); assert.equal(rea.version, '6.8.0');
|
|
assert.equal(fs.realpathSync.native(rea.packageRoot), fs.realpathSync.native(path.join(release.portableRoot, 'runtime/node_modules/rea-agents')));
|
|
assert.equal((await api('/engines/deepseek/availability')).status, 'available');
|
|
const controlledAvailability = await api('/engines/deepseek/controlled-availability'); assert.equal(controlledAvailability.controlledStatus, 'available');
|
|
// Hot upgrade retains the host process and therefore its original execPath.
|
|
// Require that exact original executable, plus byte identity with the new bundle.
|
|
assert.equal(fs.realpathSync.native(controlledAvailability.node.path), fs.realpathSync.native(path.join(info.appRoot, 'runtime/node.exe')));
|
|
assert.equal(hash(fs.readFileSync(controlledAvailability.node.path)), hash(fs.readFileSync(path.join(release.portableRoot, 'runtime/node.exe'))));
|
|
assert.equal(controlledAvailability.node.version, '24.22.1');
|
|
const purgeSource = await api('/engines/purge/source'); assert.equal(purgeSource.status, 'available'); assert.equal(purgeSource.defaultEnabled, false);
|
|
assert.equal((await api('/sessions/' + before.sessions[0].id + '/browser')).availability.status, 'available');
|
|
const result = { success: true, version: targetVersion, previousVersion, testedAt: new Date().toISOString(), ...upgraded,
|
|
directory, appRoot: release.portableRoot, dataRoot: info.dataRoot, originalAppRoot: info.appRoot,
|
|
model: after.config.model, hasKey: after.config.hasKey, allSessionCount: beforeDb.sessions.length, visibleSessionCount: before.sessions.length,
|
|
archivedSessionCount: beforeDb.sessions.filter(value => value.archived).length,
|
|
database: Object.fromEntries(Object.entries(beforeDb.tables).map(([name, value]) => [name, { count: value.count, sha256: value.sha256 }])),
|
|
recordKinds: beforeDb.recordKinds, preservedSessionSnapshots: exports, allRowsAndExportsPreserved: true, inspectorClosed: true,
|
|
comparedFiles: compared.length, repackedMetadata, frozenInputs: source.inputFiles, independentSourceRoot: source.sourceRoot,
|
|
independentSourceTests: source.tests, uiGroups: ui.groups, projectSamples: projects.results.length, realModelRequests: 0, headers,
|
|
comparison: 'Exact canonical hashes of every persisted row and exported project/session/execution, with no omitted compatibility fields.',
|
|
preflight, historicalReleasePid: preflight.historicalReleasePid, preservesCurrentObservedProcess: true,
|
|
modelCenterPreserved: true, modelCenter: { sha256: digest(modelCenterBefore), connections: modelCenterBefore.connections.length,
|
|
pools: modelCenterBefore.pools.length, imports: modelCenterBefore.imports.length, attempts: modelCenterBefore.attempts.length },
|
|
credentialsHandling: 'Only public configuration/hasKey and data hashes; original Provider/credentials stay in the process. No model key read/copied/printed/saved and no model request.' };
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
|
fs.writeFileSync('docs/controlled-engine-upgrade-live-validation.json', JSON.stringify(result, null, 2) + '\n');
|
|
fs.writeFileSync('docs/deployment.json', JSON.stringify({ ready: true, url, version: targetVersion, mode: after.config.mode, model: after.config.model, hasKey: after.config.hasKey,
|
|
pid: health.pid, appRoot: release.portableRoot, originalAppRoot: info.appRoot, dataRoot: info.dataRoot, installedAt: new Date().toISOString(), verification: 'controlled-engine-upgrade-live-validation.json' }, null, 2) + '\n');
|
|
console.log(JSON.stringify({ success: true, version: targetVersion, pid: health.pid, hasKey: result.hasKey, directory, allSessions: result.allSessionCount,
|
|
counts: Object.fromEntries(Object.entries(result.database).map(([name, value]) => [name, value.count])), allRowsAndExportsPreserved: true, inspectorClosed: true, realModelRequests: 0 }));
|