183 lines
17 KiB
JavaScript
183 lines
17 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { spawn, execFileSync } from 'node:child_process';
|
|
import net from 'node:net';
|
|
import { createResearchSamples } from './research-fixtures.mjs';
|
|
import { verifyBrowserApi } from './verify-browser.mjs';
|
|
import { verifyControlledApi } from './verify-controlled-release.mjs';
|
|
import { gitEnvironment } from '../src/core/git-process.mjs';
|
|
|
|
if (process.argv.includes('--help')) { console.log('node scripts/verify-release.mjs [--native-engines] - validate the latest immutable portable release in an isolated host.'); process.exit(0); }
|
|
if (process.argv.slice(2).some(value => value !== '--native-engines')) throw new Error('Unsupported argument; use --help.');
|
|
|
|
const release = JSON.parse(fs.readFileSync('releases/latest.json', 'utf8'));
|
|
const appRoot = release.portableRoot;
|
|
const manifest = JSON.parse(fs.readFileSync(path.join(appRoot, 'MANIFEST.json'), 'utf8'));
|
|
for (const file of manifest.files) {
|
|
const absolute = path.resolve(appRoot, file.path);
|
|
assert.ok(absolute.startsWith(appRoot + path.sep));
|
|
const bytes = fs.readFileSync(absolute);
|
|
assert.equal(bytes.length, file.bytes);
|
|
assert.equal(createHash('sha256').update(bytes).digest('hex'), file.sha256, file.path);
|
|
}
|
|
const data = path.resolve('.qa', 'release-' + randomUUID());
|
|
fs.mkdirSync(data, { recursive: true });
|
|
const portProbe = net.createServer();
|
|
await new Promise((resolve, reject) => { portProbe.once('error', reject); portProbe.listen(0, '127.0.0.1', resolve); });
|
|
const port = portProbe.address().port;
|
|
await new Promise(resolve => portProbe.close(resolve));
|
|
const env = { ...gitEnvironment(), ...Object.fromEntries(['AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_ROOT'].filter(name => process.env[name]).map(name => [name, process.env[name]])), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: String(port) };
|
|
delete env.AGENT_WORKBENCH_API_KEY;
|
|
for (const name of Object.keys(env)) if (name.toLowerCase() === 'path') delete env[name];
|
|
const windowsRoot = process.env.SystemRoot || 'C:\\Windows';
|
|
env.PATH = [windowsRoot, path.join(windowsRoot, 'System32'), path.join(windowsRoot, 'System32', 'WindowsPowerShell', 'v1.0')].join(path.delimiter);
|
|
let externalGit = null;
|
|
if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1','1.21.0', '1.22.0'].includes(manifest.version)) {
|
|
// The controlled project requires external Git. Preserve its observed binary
|
|
// directory while keeping system Node/npm unavailable to the packaged host.
|
|
const binary = fs.realpathSync.native(execFileSync('where.exe', ['git.exe'], { env: gitEnvironment(), encoding: 'utf8', windowsHide: true, timeout: 10000 }).trim().split(/\r?\n/)[0]);
|
|
const directory = path.dirname(binary);
|
|
assert.equal(fs.existsSync(path.join(directory, 'node.exe')), false); assert.equal(fs.existsSync(path.join(directory, 'npm.cmd')), false);
|
|
env.PATH = directory + path.delimiter + env.PATH;
|
|
externalGit = { path: binary, version: execFileSync('git', ['--version'], { env, windowsHide: true, encoding: 'utf8', timeout: 10000 }).trim(),
|
|
sha256: createHash('sha256').update(fs.readFileSync(binary)).digest('hex') };
|
|
}
|
|
function runPowerShell(script, extra = []) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn('powershell.exe', ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', path.join(appRoot, 'scripts', script), '-Port', String(port), ...extra], { windowsHide: true, env, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
let output = '', errors = '';
|
|
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); });
|
|
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
|
const timeout = setTimeout(() => { child.kill(); reject(new Error('Launcher did not exit in time.')); }, 20000);
|
|
child.on('error', error => { clearTimeout(timeout); reject(error); });
|
|
// Windows descendants can inherit pipe handles after the launcher exits.
|
|
// Wait for the launcher itself, then close this test's readers.
|
|
child.on('exit', code => {
|
|
clearTimeout(timeout); child.stdout.destroy(); child.stderr.destroy();
|
|
if (code === 0) resolve(output); else reject(new Error('Launcher failed: ' + errors));
|
|
});
|
|
});
|
|
}
|
|
const launch = () => runPowerShell('Start-Workbench.ps1', ['-NoBrowser']);
|
|
const stop = () => runPowerShell('Stop-Workbench.ps1');
|
|
const url = 'http://127.0.0.1:' + port;
|
|
const until = async (predicate, timeoutMs = 10000) => {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (!(await predicate())) {
|
|
if (Date.now() > deadline) throw new Error('Release state timeout');
|
|
await new Promise(resolve => setTimeout(resolve, 50));
|
|
}
|
|
};
|
|
let started = false;
|
|
try {
|
|
console.log((await launch()).trim());
|
|
const firstInfo = JSON.parse(fs.readFileSync(path.join(data, 'service-info.json'), 'utf8'));
|
|
assert.equal(path.resolve(firstInfo.appRoot), path.resolve(appRoot));
|
|
const health = await (await fetch(url + '/api/health')).json();
|
|
assert.equal(health.pid, firstInfo.pid, 'The test must own the launched service.');
|
|
started = true;
|
|
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
|
const api = async (route, input) => {
|
|
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
|
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
|
const result = await response.json(); assert.ok(response.ok, result.error); return result;
|
|
};
|
|
assert.equal(bootstrap.version, manifest.version); assert.equal(bootstrap.config.mode, 'demo');
|
|
console.log((await launch()).trim());
|
|
assert.equal(JSON.parse(fs.readFileSync(path.join(data, 'service-info.json'), 'utf8')).pid, firstInfo.pid);
|
|
const example = bootstrap.projects.find(project => project.id === bootstrap.exampleProjectId);
|
|
if (['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1','1.21.0', '1.22.0'].includes(manifest.version)) {
|
|
assert.equal(path.resolve(example.root).toLowerCase(), path.join(data, 'projects', 'starter-project').toLowerCase());
|
|
const compareTemplate = relative => { for (const entry of fs.readdirSync(path.join(appRoot, 'examples/starter-project', relative), { withFileTypes: true })) {
|
|
const next = path.join(relative, entry.name); if (entry.isDirectory()) compareTemplate(next);
|
|
else assert.deepEqual(fs.readFileSync(path.join(example.root, next)), fs.readFileSync(path.join(appRoot, 'examples/starter-project', next)), next);
|
|
} }; compareTemplate('');
|
|
} else assert.ok(example.root.toLowerCase().startsWith(appRoot.toLowerCase() + path.sep));
|
|
const session = await api('/sessions', { projectId: example.id });
|
|
await api('/sessions/' + session.id + '/messages', { content: '便携包工具与随附 Node 验证' });
|
|
let current;
|
|
await until(async () => { current = await api('/sessions/' + session.id); return current.status === 'awaiting_approval'; });
|
|
assert.equal(current.approvals[0].command, 'node --version');
|
|
assert.equal(fs.existsSync(path.join(example.root, 'WORKBENCH-NOTES.md')), false);
|
|
await api('/sessions/' + session.id + '/approvals/' + current.approvals[0].id, { allow: true });
|
|
await until(async () => { current = await api('/sessions/' + session.id); return current.status === 'needs_review'; });
|
|
const command = current.events.find(event => event.tool === 'run_command');
|
|
assert.equal(command.phase, 'completed'); assert.match(command.output, new RegExp(manifest.nodeVersion.replaceAll('.', '\\.')));
|
|
for (const proposal of current.changes.filter(change => change.status === 'pending')) await api('/sessions/' + session.id + '/changes/' + proposal.id + '/reject', {});
|
|
const planned = await api('/sessions/' + session.id + '/check-plans', { names: ['test'] });
|
|
const plan = planned.execution.plans.at(-1); assert.ok(plan.checks.some(check => check.name === 'test'));
|
|
const accepted = await api('/sessions/' + session.id + '/check-jobs', { planId: plan.id }); const jobId = accepted.execution.checks.at(-1).id;
|
|
let checked; await until(async () => { checked = await api('/sessions/' + session.id + '/check-jobs/' + jobId); return !['queued', 'running'].includes(checked.status); });
|
|
assert.equal(checked.status, 'passed'); assert.equal(checked.results[0].tests.total, 4); assert.equal(checked.results[0].tests.passed, 4);
|
|
const checkLog = await api('/artifacts/' + checked.results[0].logArtifactId + '?session=' + session.id); assert.match(checkLog.content, /tests 4/);
|
|
assert.ok(fs.existsSync(path.join(appRoot, 'runtime/node_modules/npm/bin/npm-cli.js')));
|
|
const actualRea = [];
|
|
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1','1.21.0', '1.22.0'].includes(manifest.version)) {
|
|
const installation = await api('/research/availability'); assert.equal(installation.status, 'available'); assert.equal(installation.version, '6.8.0');
|
|
assert.equal(fs.realpathSync.native(installation.packageRoot), fs.realpathSync.native(path.join(appRoot, 'runtime/node_modules/rea-agents')));
|
|
assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
|
|
const runtimeManifest = JSON.parse(fs.readFileSync(path.join(appRoot, 'runtime/REA-RUNTIME.json'), 'utf8'));
|
|
assert.equal(runtimeManifest.version, '6.8.0'); assert.equal(runtimeManifest.protocolVersion, '2025-03-26');
|
|
assert.equal(createHash('sha256').update(fs.readFileSync(path.join(appRoot, 'runtime', runtimeManifest.installationLock.path))).digest('hex'), runtimeManifest.installationLock.sha256);
|
|
const projectRoot = path.join(data, 'research-project'); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned portable REA project\n');
|
|
const samples = createResearchSamples(path.join(data, 'research-samples')), researchProject = await api('/projects', { root: projectRoot });
|
|
const researchSession = await api('/sessions', { projectId: researchProject.id }), researchRoute = '/sessions/' + researchSession.id;
|
|
for (const [operation, filename] of [['javascript', samples.asar], ['pe-imports', samples.pe], ['pe-resources', samples.pe]]) {
|
|
const registered = await api(researchRoute + '/research/targets', { path: filename, label: 'Owned portable ' + operation });
|
|
const target = registered.target, startedJob = await api(researchRoute + '/research/jobs', { targetId: target.id, operation });
|
|
let job; await until(async () => { job = await api(researchRoute + '/research/jobs/' + startedJob.job.id); return !!job.finishedAt; }, 45000);
|
|
assert.equal(job.status, 'completed', job.summary); assert.equal(job.cleanup.rootExited, true);
|
|
const response = await fetch(url + '/api' + researchRoute + '/research/jobs/' + job.id + '/download?kind=bundle', { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } });
|
|
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer());
|
|
assert.equal(createHash('sha256').update(bytes).digest('hex'), job.bundle.documentSha256);
|
|
const evidence = JSON.parse(bytes.toString('utf8')).records.find(record => record.evidence_id === job.evidenceId);
|
|
assert.equal(evidence.subject.digest.sha256, target.sha256);
|
|
if (operation === 'javascript') assert.equal(evidence.normalized_result.statistics.parsed_javascript_files, 2);
|
|
if (operation === 'pe-imports') { assert.match(JSON.stringify(evidence.normalized_result), /KERNEL32\.dll/); assert.match(JSON.stringify(evidence.normalized_result), /ExitProcess/); }
|
|
if (operation === 'pe-resources') assert.match(JSON.stringify(evidence.normalized_result), new RegExp(createHash('sha256').update('REA-DATA').digest('hex')));
|
|
const reused = await api(researchRoute + '/research/jobs', { targetId: target.id, operation });
|
|
let reuseJob; await until(async () => { reuseJob = await api(researchRoute + '/research/jobs/' + reused.job.id); return !!reuseJob.finishedAt; });
|
|
assert.equal(reuseJob.reusedFromJobId, job.id); assert.equal(reuseJob.traceArtifactId, undefined);
|
|
actualRea.push({ operation, jobId: job.id, evidenceId: job.evidenceId, targetSha256: target.sha256, bundleSha256: job.bundle.documentSha256, rootExited: true, reusedJobId: reuseJob.id });
|
|
}
|
|
assert.equal((await api(researchRoute)).execution.task.usage.requests, 0); assert.equal(fs.existsSync(samples.marker), false);
|
|
console.log('PASS actual contained REA: ASAR, PE imports and PE resources; zero target execution/model requests');
|
|
}
|
|
let actualEngine = null;
|
|
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1','1.21.0', '1.22.0'].includes(manifest.version) && process.argv.includes('--native-engines')) {
|
|
const installation = await api('/engines/deepseek/availability'); assert.equal(installation.status, 'available', installation.message);
|
|
assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
|
|
const engineRoot = path.join(data, 'engine-project'); fs.mkdirSync(engineRoot); fs.writeFileSync(path.join(engineRoot, 'README.md'), '# Owned portable engine connection\n');
|
|
const project = await api('/projects', { root: engineRoot }), engineSession = await api('/sessions', { projectId: project.id }), route = '/sessions/' + engineSession.id;
|
|
const started = await api(route + '/engines/jobs', { root: installation.root, identitySha256: installation.identitySha256 });
|
|
let job; await until(async () => { job = await api(route + '/engines/jobs/' + started.job.id); return !!job.reportArtifactId; }, 30000);
|
|
assert.equal(job.status, 'completed', job.summary); assert.equal(job.reportSummary.rootExited, true); assert.equal(job.reportSummary.modelRequests, 0);
|
|
const response = await fetch(url + '/api' + route + '/engines/jobs/' + job.id + '/download', { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } });
|
|
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer());
|
|
assert.equal(createHash('sha256').update(bytes).digest('hex'), job.reportSha256);
|
|
const original = JSON.parse(bytes.toString('utf8')); assert.equal(original.owner.taskId, job.taskId); assert.equal(original.report.responses.length, 6);
|
|
assert.equal(original.report.providerRequests, 0); assert.equal(original.report.cleanup.exit.code, 0);
|
|
actualEngine = { packageVersion: installation.version, profile: installation.profile, installationSha256: installation.identitySha256,
|
|
reportSha256: job.reportSha256, rootExited: true, modelRequests: 0, nativeMethods: original.report.responses.map(value => value.method), node: installation.node };
|
|
console.log('PASS compiled portable engine adapter: actual local SDK, contained Node, six native methods and sealed original report');
|
|
}
|
|
assert.equal((await api('/research')).length, 4);
|
|
const page = await fetch(url); assert.equal(page.status, 200);
|
|
assert.match(await page.text(), /Agent Workbench/);
|
|
const actualBrowser = ['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1','1.21.0', '1.22.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null;
|
|
const actualControlled = ['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1','1.21.0', '1.22.0'].includes(manifest.version) ? await verifyControlledApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null;
|
|
console.log((await stop()).trim()); started = false;
|
|
await until(async () => { try { await fetch(url + '/api/health'); return false; } catch { return true; } });
|
|
for (const file of manifest.files) {
|
|
const bytes = fs.readFileSync(path.join(appRoot, file.path)); assert.equal(bytes.length, file.bytes);
|
|
assert.equal(createHash('sha256').update(bytes).digest('hex'), file.sha256, file.path);
|
|
}
|
|
const result = { success: true, date: new Date().toISOString(), appRoot, version: manifest.version, nodeVersion: manifest.nodeVersion,
|
|
verifiedFiles: manifest.files.length, url, reusedPid: firstInfo.pid, checks: { jobId, status: checked.status, tests: checked.results[0].tests }, actualRea, actualEngine, actualBrowser, actualControlled, externalGit,
|
|
steps: ['manifest hashes', 'bundled Node launch', 'same-service reuse', 'packaged example and static page', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1','1.21.0', '1.22.0'].includes(manifest.version) ? ['starter project copied to owned user data; original template bytes matched'] : []), 'explicit command consent', 'actual bundled Node version', 'actual bundled npm check without system Node/npm', 'check log artifact', ...(actualRea.length ? ['contained REA installation and Node identity', 'actual ASAR/imports/resources', 'original Bundle and target hash binding', 'verified reuse without producer replay'] : []), 'report availability', 'graceful stop', 'every original packaged file remains unchanged'], realModel: false };
|
|
fs.writeFileSync('.qa/release-validation.json', JSON.stringify(result, null, 2));
|
|
console.log(JSON.stringify(result, null, 2));
|
|
} finally { if (started) { try { await stop(); } catch {} } }
|