219 lines
20 KiB
JavaScript
219 lines
20 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { spawn, execFileSync } from 'node:child_process';
|
|
|
|
if (process.argv.includes('--help')) { console.log('node scripts/verify-source-package.mjs [--native-engines] - validate independently extracted source after verify-archives.py --extract-source; set the absolute QA Python and archive root.'); process.exit(0); }
|
|
if (process.argv.slice(2).some(value => value !== '--native-engines')) throw new Error('Unsupported argument; use --help.');
|
|
|
|
const workspace = process.cwd();
|
|
const evidence = JSON.parse(fs.readFileSync('.qa/archives-validation.json', 'utf8'));
|
|
assert.equal(evidence.success, true);
|
|
assert.ok(evidence.extractionRoot, 'Run python scripts/verify-archives.py --extract-source first.');
|
|
const sourceRoot = fs.realpathSync.native(evidence.extractionRoot);
|
|
const archiveQaRoot = path.resolve(process.env.AGENT_WORKBENCH_ARCHIVE_QA_ROOT || '.qa');
|
|
assert.ok(sourceRoot.startsWith(archiveQaRoot + path.sep));
|
|
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
|
assert.ok(fs.existsSync(npmCli), 'Use Node with its bundled npm.');
|
|
const files = (root, relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))
|
|
.flatMap(entry => entry.isDirectory() ? files(root, path.join(relative, entry.name)) : [path.join(relative, entry.name)]);
|
|
const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
const buildFolders = ['dist-client', 'dist-server', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version) ? ['dist-desktop'] : [])];
|
|
const expected = Object.fromEntries(buildFolders.flatMap(folder => files(path.join(workspace, folder)).map(relative => [path.join(folder, relative), digest(path.join(workspace, folder, relative))])));
|
|
const allowedEnv = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON|AGENT_WORKBENCH_DEEPSEEK_ROOT|AGENT_WORKBENCH_PURGE_ROOT)$/i;
|
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => allowedEnv.test(key)));
|
|
const inheritedPath = Object.entries(env).find(([key]) => key.toLowerCase() === 'path')?.[1] || '';
|
|
for (const key of Object.keys(env)) if (key.toLowerCase() === 'path') delete env[key];
|
|
env.PATH = [path.dirname(process.execPath), inheritedPath].filter(Boolean).join(path.delimiter);
|
|
const directory = path.join(sourceRoot, '.qa', 'package-validation-' + randomUUID());
|
|
fs.mkdirSync(directory, { recursive: true });
|
|
const inputFolders = ['src', 'tests', 'scripts', ...['benchmarks', 'public', 'examples', 'research'].filter(folder => fs.existsSync(path.join(sourceRoot, folder)))];
|
|
const inputNames = [...inputFolders.flatMap(folder => files(path.join(sourceRoot, folder)).map(relative => path.join(folder, relative))),
|
|
'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', ...(['1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version) ? ['mobile.html'] : []), ...['.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].filter(name => fs.existsSync(path.join(sourceRoot, name)))].sort();
|
|
const snapshot = () => inputNames.map(relative => ({ path: relative.replaceAll('\\', '/'), bytes: fs.statSync(path.join(sourceRoot, relative)).size, sha256: digest(path.join(sourceRoot, relative)) }));
|
|
const inputs = snapshot(); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2));
|
|
let stage = 'environment preflight', python = null;
|
|
const resultFile = path.join(workspace, '.qa/source-package-validation.json');
|
|
const logs = [];
|
|
function save(result) {
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
|
fs.writeFileSync(resultFile, JSON.stringify(result, null, 2));
|
|
}
|
|
|
|
function run(args, logName) {
|
|
console.log('Source package: ' + stage);
|
|
return new Promise((resolve, reject) => {
|
|
const logPath = path.join(directory, logName);
|
|
const outputFile = fs.createWriteStream(logPath);
|
|
const child = spawn(process.execPath, args, { cwd: sourceRoot, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
let tail = '', timedOut = false;
|
|
const capture = bytes => { outputFile.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-12000); };
|
|
child.stdout.on('data', capture); child.stderr.on('data', capture);
|
|
const timer = setTimeout(() => {
|
|
timedOut = true;
|
|
if (process.platform === 'win32' && child.pid) {
|
|
try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000, stdio: 'ignore' }); } catch { child.kill(); }
|
|
} else child.kill();
|
|
}, 30 * 60 * 1000);
|
|
child.once('error', error => { clearTimeout(timer); outputFile.end(); reject(error); });
|
|
child.once('close', code => { clearTimeout(timer); outputFile.end(() => {
|
|
logs.push({ stage, path: logPath, sha256: digest(logPath), bytes: fs.statSync(logPath).size });
|
|
code === 0 && !timedOut ? resolve(tail) : reject(new Error(stage + (timedOut ? ' timed out after 30 minutes' : ' failed (' + code + ')') + ': ' + tail));
|
|
}); });
|
|
});
|
|
}
|
|
|
|
try {
|
|
assert.equal(JSON.parse(fs.readFileSync(path.join(sourceRoot, 'package.json'), 'utf8')).version, evidence.version);
|
|
assert.match(process.version, /^v24\./, 'Source package validation requires Node.js 24.');
|
|
if (env.AGENT_WORKBENCH_TEST_PYTHON) {
|
|
assert.ok(path.isAbsolute(env.AGENT_WORKBENCH_TEST_PYTHON) && fs.existsSync(env.AGENT_WORKBENCH_TEST_PYTHON), 'AGENT_WORKBENCH_TEST_PYTHON must name an existing absolute QA interpreter.');
|
|
const executable = fs.realpathSync.native(env.AGENT_WORKBENCH_TEST_PYTHON);
|
|
python = JSON.parse(execFileSync(executable, ['-I', '-c', 'import json,sys,pytest; print(json.dumps({"version":sys.version.split()[0],"pytest":pytest.__version__}))'], { env, windowsHide: true, encoding: 'utf8', timeout: 10000 }));
|
|
assert.ok(Number(python.version.split('.')[0]) === 3 && Number(python.version.split('.')[1]) >= 10, 'QA Python must be 3.10 or newer.');
|
|
assert.equal(python.pytest, '8.4.2', 'Install the fixed QA requirements before validating the source package.');
|
|
python.path = executable; env.AGENT_WORKBENCH_TEST_PYTHON = executable;
|
|
}
|
|
console.log(JSON.stringify({ sourceRoot, directory, nodeVersion: process.version, python }));
|
|
stage = 'install';
|
|
await run([npmCli, 'ci', '--offline', '--ignore-scripts', '--no-audit', '--no-fund'], 'install.log');
|
|
stage = 'build'; await run([npmCli, 'run', 'build'], 'build.log');
|
|
const actualFiles = buildFolders.flatMap(folder => files(path.join(sourceRoot, folder)).map(relative => path.join(folder, relative)));
|
|
assert.deepEqual(actualFiles.slice().sort(), Object.keys(expected).sort());
|
|
for (const [relative, sha256] of Object.entries(expected)) assert.equal(digest(path.join(sourceRoot, relative)), sha256, relative);
|
|
let actualRea = null;
|
|
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'contained REA';
|
|
const reaOutput = await run(['scripts/verify-rea.mjs', '--app-root', sourceRoot, '--contained'], 'rea.log');
|
|
actualRea = JSON.parse(reaOutput.trim().split('\n').at(-1));
|
|
assert.equal(actualRea.success, true); assert.equal(actualRea.version, evidence.version); assert.equal(actualRea.results.length, 3);
|
|
assert.ok(actualRea.results.every(item => item.rootExited === true));
|
|
}
|
|
let actualEngine = null;
|
|
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version) && process.argv.includes('--native-engines')) {
|
|
stage = 'native DeepSeek SDK';
|
|
const output = await run(['scripts/verify-engines.mjs'], 'engines.log');
|
|
actualEngine = JSON.parse(output.trim().split('\n').at(-1));
|
|
assert.equal(actualEngine.success, true); assert.equal(actualEngine.version, evidence.version);
|
|
assert.equal(actualEngine.modelRequests, 0); assert.equal(actualEngine.rootExited, true); assert.equal(actualEngine.nativeMethods.length, 6);
|
|
}
|
|
let actualBrowser = null;
|
|
if (['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'contained browser runtime';
|
|
const output = await run(['scripts/verify-browser.mjs', '--app-root', sourceRoot, '--contained'], 'browser.log');
|
|
actualBrowser = JSON.parse(output.trim().split('\n').at(-1));
|
|
assert.equal(actualBrowser.success, true); assert.equal(actualBrowser.version, evidence.version);
|
|
assert.equal(actualBrowser.rootExited, true); assert.equal(actualBrowser.results.length, 2);
|
|
assert.ok(actualBrowser.results.every(value => value.status === 'passed' && value.browserExited));
|
|
assert.equal(actualBrowser.modelRequests, 0);
|
|
}
|
|
let actualControlled = null;
|
|
if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'compiled controlled-engine delivery';
|
|
const output = await run(['scripts/verify-controlled-release.mjs', '--app-root', sourceRoot], 'controlled-engine.log');
|
|
actualControlled = JSON.parse(output.trim().split('\n').at(-1));
|
|
assert.equal(actualControlled.success, true); assert.equal(actualControlled.version, evidence.version); assert.equal(actualControlled.rootExited, true);
|
|
assert.equal(actualControlled.realModelRequests, 0); assert.equal(actualControlled.cases, 4); assert.ok(actualControlled.fixtureModelRequests > 0);
|
|
}
|
|
let actualCompanion = null;
|
|
if (['1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'contained companion gateway and UI';
|
|
const output = await run(['scripts/verify-companion-ui.mjs', '--app-root', sourceRoot], 'companion-ui.log');
|
|
const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
|
|
assert.ok(location && path.resolve(location).startsWith(path.join(sourceRoot, '.qa') + path.sep));
|
|
actualCompanion = JSON.parse(fs.readFileSync(path.join(location, 'result.json'), 'utf8'));
|
|
assert.equal(actualCompanion.success, true); assert.equal(actualCompanion.version, evidence.version);
|
|
assert.equal(fs.realpathSync.native(actualCompanion.appRoot), sourceRoot); assert.equal(actualCompanion.containedRuntime, true);
|
|
assert.equal(actualCompanion.passed.length, 7); assert.deepEqual(actualCompanion.errors, []); assert.equal(actualCompanion.realModelRequests, 0);
|
|
}
|
|
let actualProtocols = null;
|
|
if (['1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'contained model protocols and mobile streams';
|
|
const output = await run(['scripts/verify-model-protocols-ui.mjs', '--app-root', sourceRoot], 'model-protocols-ui.log');
|
|
const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
|
|
assert.ok(location && path.resolve(location).startsWith(path.join(sourceRoot, '.qa') + path.sep));
|
|
actualProtocols = JSON.parse(fs.readFileSync(path.join(location, 'result.json'), 'utf8'));
|
|
assert.equal(actualProtocols.success, true); assert.equal(actualProtocols.version, evidence.version);
|
|
assert.equal(fs.realpathSync.native(actualProtocols.appRoot), sourceRoot); assert.equal(actualProtocols.containedRuntime, true);
|
|
assert.equal(actualProtocols.passed.length, 5); assert.deepEqual(actualProtocols.pageErrors, []); assert.deepEqual(actualProtocols.requestFailures, []);
|
|
}
|
|
let actualVision = null;
|
|
if (['1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'contained screenshot inputs and exact Task budget resume';
|
|
const output = await run(['scripts/verify-vision-ui.mjs', '--app-root', sourceRoot], 'vision-ui.log');
|
|
const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
|
|
assert.ok(location && path.resolve(location).startsWith(path.join(sourceRoot, '.qa') + path.sep));
|
|
actualVision = JSON.parse(fs.readFileSync(path.join(location, 'result.json'), 'utf8'));
|
|
assert.equal(actualVision.success, true); assert.equal(actualVision.version, evidence.version);
|
|
assert.equal(fs.realpathSync.native(actualVision.appRoot), sourceRoot); assert.equal(actualVision.containedRuntime, true);
|
|
assert.equal(actualVision.passed.length, 5); assert.deepEqual(actualVision.pageErrors, []); assert.equal(actualVision.realModelRequests, 0);
|
|
}
|
|
let actualProjectCache = null;
|
|
if (evidence.version === '1.19.0') {
|
|
stage = 'contained source-verified parsing cache controls';
|
|
const output = await run(['scripts/verify-project-cache-ui.mjs', '--app-root', sourceRoot], 'project-cache-ui.log');
|
|
const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
|
|
assert.ok(location && path.resolve(location).startsWith(path.join(sourceRoot, '.qa') + path.sep));
|
|
actualProjectCache = JSON.parse(fs.readFileSync(path.join(location, 'result.json'), 'utf8'));
|
|
assert.equal(actualProjectCache.success, true); assert.equal(actualProjectCache.version, evidence.version);
|
|
assert.equal(fs.realpathSync.native(actualProjectCache.appRoot), sourceRoot); assert.equal(actualProjectCache.passed.length, 7);
|
|
assert.equal(actualProjectCache.realModelRequests, 0); assert.deepEqual(actualProjectCache.pageErrors, []);
|
|
}
|
|
stage = 'tests';
|
|
const output = await run(['--test', '--test-concurrency=2', '--test-reporter=tap', ...files(path.join(sourceRoot, 'tests')).filter(file => !file.includes(path.sep) && file.endsWith('.test.mjs')).map(file => path.join('tests', file))], 'tests.tap');
|
|
const stats = Object.fromEntries([...output.matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])]));
|
|
assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0);
|
|
let actualDesktop = null;
|
|
if (['1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'actual contained Electron window'; await run(['scripts/verify-desktop-window.mjs'], 'desktop-window.log');
|
|
const window = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/desktop-window-latest.json'), 'utf8'));
|
|
assert.equal(window.success, true); assert.equal(path.resolve(window.appRoot), sourceRoot); assert.equal(window.results.length, 7);
|
|
stage = 'actual contained Electron engine UI'; await run(['scripts/verify-controlled-ui.mjs', '--desktop'], 'desktop-controlled-ui.log');
|
|
const uiLocation = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/desktop-controlled-ui-location.json'), 'utf8'));
|
|
const ui = JSON.parse(fs.readFileSync(uiLocation.result, 'utf8')); assert.equal(ui.passed, true); assert.equal(ui.testedVersion, evidence.version); assert.equal(ui.assertions.length, 9);
|
|
stage = 'actual contained Electron engineering projects'; await run(['scripts/verify-controlled-projects.mjs', '--desktop'], 'desktop-projects.log');
|
|
const projectLocation = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/desktop-projects-location.json'), 'utf8'));
|
|
const projects = JSON.parse(fs.readFileSync(projectLocation.result, 'utf8')); assert.equal(projects.success, true); assert.equal(projects.testedVersion, evidence.version); assert.equal(projects.cases.length, 16);
|
|
assert.deepEqual(projects.results.map(value => value.parentCheck.results[0].tests.passed), [8, 9, 10]);
|
|
actualDesktop = { window, ui, projects, scope: 'Independently rebuilt source and its shipped Electron binary; original window/engine/engineering assertions. Own credentials/HTTP fixtures; Windows native picker and complete host crash/stop have separate actual evidence.' };
|
|
}
|
|
let actualProtocolEngineering = null;
|
|
if (['1.17.0', '1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'Responses SSE actual Node React Python engineering';
|
|
await run(['scripts/verify-controlled-projects.mjs', '--responses'], 'responses-engineering.log');
|
|
const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/responses-engineering-location.json'), 'utf8'));
|
|
actualProtocolEngineering = JSON.parse(fs.readFileSync(location.result, 'utf8'));
|
|
assert.equal(actualProtocolEngineering.success, true); assert.equal(actualProtocolEngineering.testedVersion, evidence.version);
|
|
assert.equal(actualProtocolEngineering.protocol, 'responses-stream'); assert.equal(actualProtocolEngineering.realModelRequests, 0);
|
|
assert.deepEqual(actualProtocolEngineering.results.map(value => value.parentCheck.results[0].tests.passed), [8, 9, 10]);
|
|
}
|
|
let actualVisionEngineering = null;
|
|
if (['1.18.0', '1.19.0'].includes(evidence.version)) {
|
|
stage = 'screenshot Task Node React Python engineering';
|
|
await run(['scripts/verify-vision-projects.mjs'], 'vision-engineering.log');
|
|
const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/vision-engineering-location.json'), 'utf8'));
|
|
actualVisionEngineering = JSON.parse(fs.readFileSync(location.result, 'utf8'));
|
|
assert.equal(actualVisionEngineering.success, true); assert.equal(actualVisionEngineering.version, evidence.version);
|
|
assert.equal(actualVisionEngineering.realModelRequests, 0);
|
|
assert.deepEqual(actualVisionEngineering.results.map(value => value.check.results[0].tests.passed), [8, 9, 10]);
|
|
}
|
|
let actualCacheEngineering = null;
|
|
if (evidence.version === '1.19.0') {
|
|
stage = 'cached index original requests and Node React Python engineering';
|
|
await run(['scripts/verify-project-cache-projects.mjs'], 'project-cache-engineering.log');
|
|
const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/project-cache-engineering-location.json'), 'utf8'));
|
|
actualCacheEngineering = JSON.parse(fs.readFileSync(location.result, 'utf8'));
|
|
assert.equal(actualCacheEngineering.success, true); assert.equal(actualCacheEngineering.version, evidence.version);
|
|
assert.equal(actualCacheEngineering.realModelRequests, 0); assert.equal(actualCacheEngineering.cases.length, 9);
|
|
assert.deepEqual(actualCacheEngineering.results.map(value => value.check.results[0].tests.passed), [8, 9, 10]);
|
|
}
|
|
assert.deepEqual(snapshot(), inputs, 'Source package inputs changed during validation.');
|
|
const result = { success: true, version: evidence.version, testedAt: new Date().toISOString(), sourceRoot, directory, nodeVersion: process.version, python, offlineInstall: true,
|
|
inputs, inputFiles: inputs.length, logs, buildMatchedFiles: Object.keys(expected).length, buildFiles: Object.entries(expected).map(([path, sha256]) => ({ path: path.replaceAll('\\', '/'), sha256 })), tests: stats, actualRea, actualEngine, actualBrowser, actualControlled, actualDesktop, actualCompanion, actualProtocols, actualProtocolEngineering, actualVision, actualVisionEngineering, actualProjectCache, actualCacheEngineering, credentialsHandling: 'Only build/runtime environment paths are inherited; no configured model credentials are used. The controlled-engine stage supplies a declared owned local response fixture.' };
|
|
save(result); console.log(JSON.stringify({ success: true, version: result.version, sourceRoot, directory, tests: stats, nodeVersion: process.version, python, inputFiles: inputs.length, buildMatchedFiles: result.buildMatchedFiles, rea: !!actualRea?.success, engine: !!actualEngine?.success, browser: !!actualBrowser?.success, controlled: !!actualControlled?.success }));
|
|
} catch (error) {
|
|
save({ success: false, version: evidence.version, testedAt: new Date().toISOString(), stage, error: error.message, sourceRoot, directory, nodeVersion: process.version, python, inputs, logs });
|
|
throw error;
|
|
}
|