test(vision): accept screenshot engineering and wire 1.18 release validation

This commit is contained in:
Agent Workbench Dev
2026-11-01 00:52:31 +08:00
parent 96a7ed19c7
commit 01c427e749
18 changed files with 229 additions and 64 deletions
+32 -11
View File
@@ -19,7 +19,7 @@ assert.ok(fs.existsSync(npmCli), 'Use Node with its bundled npm.');
const files = (root, relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))
.flatMap(entry => entry.isDirectory() ? files(root, path.join(relative, entry.name)) : [path.join(relative, entry.name)]);
const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
const buildFolders = ['dist-client', 'dist-server', ...(['1.15.0', '1.16.0', '1.17.0'].includes(evidence.version) ? ['dist-desktop'] : [])];
const buildFolders = ['dist-client', 'dist-server', ...(['1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version) ? ['dist-desktop'] : [])];
const expected = Object.fromEntries(buildFolders.flatMap(folder => files(path.join(workspace, folder)).map(relative => [path.join(folder, relative), digest(path.join(workspace, folder, relative))])));
const allowedEnv = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON|AGENT_WORKBENCH_DEEPSEEK_ROOT|AGENT_WORKBENCH_PURGE_ROOT)$/i;
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => allowedEnv.test(key)));
@@ -30,7 +30,7 @@ const directory = path.join(sourceRoot, '.qa', 'package-validation-' + randomUUI
fs.mkdirSync(directory, { recursive: true });
const inputFolders = ['src', 'tests', 'scripts', ...['benchmarks', 'public', 'examples', 'research'].filter(folder => fs.existsSync(path.join(sourceRoot, folder)))];
const inputNames = [...inputFolders.flatMap(folder => files(path.join(sourceRoot, folder)).map(relative => path.join(folder, relative))),
'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', ...(['1.16.0', '1.17.0'].includes(evidence.version) ? ['mobile.html'] : []), ...['.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].filter(name => fs.existsSync(path.join(sourceRoot, name)))].sort();
'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', ...(['1.16.0', '1.17.0', '1.18.0'].includes(evidence.version) ? ['mobile.html'] : []), ...['.gitattributes', '.gitignore', 'Start-Desktop.cmd', 'Start-Workbench.cmd', 'Stop-Workbench.cmd'].filter(name => fs.existsSync(path.join(sourceRoot, name)))].sort();
const snapshot = () => inputNames.map(relative => ({ path: relative.replaceAll('\\', '/'), bytes: fs.statSync(path.join(sourceRoot, relative)).size, sha256: digest(path.join(sourceRoot, relative)) }));
const inputs = snapshot(); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2));
let stage = 'environment preflight', python = null;
@@ -83,7 +83,7 @@ try {
assert.deepEqual(actualFiles.slice().sort(), Object.keys(expected).sort());
for (const [relative, sha256] of Object.entries(expected)) assert.equal(digest(path.join(sourceRoot, relative)), sha256, relative);
let actualRea = null;
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) {
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) {
stage = 'contained REA';
const reaOutput = await run(['scripts/verify-rea.mjs', '--app-root', sourceRoot, '--contained'], 'rea.log');
actualRea = JSON.parse(reaOutput.trim().split('\n').at(-1));
@@ -91,7 +91,7 @@ try {
assert.ok(actualRea.results.every(item => item.rootExited === true));
}
let actualEngine = null;
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version) && process.argv.includes('--native-engines')) {
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version) && process.argv.includes('--native-engines')) {
stage = 'native DeepSeek SDK';
const output = await run(['scripts/verify-engines.mjs'], 'engines.log');
actualEngine = JSON.parse(output.trim().split('\n').at(-1));
@@ -99,7 +99,7 @@ try {
assert.equal(actualEngine.modelRequests, 0); assert.equal(actualEngine.rootExited, true); assert.equal(actualEngine.nativeMethods.length, 6);
}
let actualBrowser = null;
if (['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) {
if (['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) {
stage = 'contained browser runtime';
const output = await run(['scripts/verify-browser.mjs', '--app-root', sourceRoot, '--contained'], 'browser.log');
actualBrowser = JSON.parse(output.trim().split('\n').at(-1));
@@ -109,7 +109,7 @@ try {
assert.equal(actualBrowser.modelRequests, 0);
}
let actualControlled = null;
if (['1.14.0', '1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) {
if (['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) {
stage = 'compiled controlled-engine delivery';
const output = await run(['scripts/verify-controlled-release.mjs', '--app-root', sourceRoot], 'controlled-engine.log');
actualControlled = JSON.parse(output.trim().split('\n').at(-1));
@@ -117,7 +117,7 @@ try {
assert.equal(actualControlled.realModelRequests, 0); assert.equal(actualControlled.cases, 4); assert.ok(actualControlled.fixtureModelRequests > 0);
}
let actualCompanion = null;
if (['1.16.0', '1.17.0'].includes(evidence.version)) {
if (['1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) {
stage = 'contained companion gateway and UI';
const output = await run(['scripts/verify-companion-ui.mjs', '--app-root', sourceRoot], 'companion-ui.log');
const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
@@ -128,7 +128,7 @@ try {
assert.equal(actualCompanion.passed.length, 7); assert.deepEqual(actualCompanion.errors, []); assert.equal(actualCompanion.realModelRequests, 0);
}
let actualProtocols = null;
if (evidence.version === '1.17.0') {
if (['1.17.0', '1.18.0'].includes(evidence.version)) {
stage = 'contained model protocols and mobile streams';
const output = await run(['scripts/verify-model-protocols-ui.mjs', '--app-root', sourceRoot], 'model-protocols-ui.log');
const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
@@ -138,12 +138,23 @@ try {
assert.equal(fs.realpathSync.native(actualProtocols.appRoot), sourceRoot); assert.equal(actualProtocols.containedRuntime, true);
assert.equal(actualProtocols.passed.length, 5); assert.deepEqual(actualProtocols.pageErrors, []); assert.deepEqual(actualProtocols.requestFailures, []);
}
let actualVision = null;
if (evidence.version === '1.18.0') {
stage = 'contained screenshot inputs and exact Task budget resume';
const output = await run(['scripts/verify-vision-ui.mjs', '--app-root', sourceRoot], 'vision-ui.log');
const location = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1]?.trim();
assert.ok(location && path.resolve(location).startsWith(path.join(sourceRoot, '.qa') + path.sep));
actualVision = JSON.parse(fs.readFileSync(path.join(location, 'result.json'), 'utf8'));
assert.equal(actualVision.success, true); assert.equal(actualVision.version, evidence.version);
assert.equal(fs.realpathSync.native(actualVision.appRoot), sourceRoot); assert.equal(actualVision.containedRuntime, true);
assert.equal(actualVision.passed.length, 5); assert.deepEqual(actualVision.pageErrors, []); assert.equal(actualVision.realModelRequests, 0);
}
stage = 'tests';
const output = await run(['--test', '--test-concurrency=2', '--test-reporter=tap', ...files(path.join(sourceRoot, 'tests')).filter(file => !file.includes(path.sep) && file.endsWith('.test.mjs')).map(file => path.join('tests', file))], 'tests.tap');
const stats = Object.fromEntries([...output.matchAll(/^# (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gm)].map(match => [match[1], Number(match[2])]));
assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0);
let actualDesktop = null;
if (['1.15.0', '1.16.0', '1.17.0'].includes(evidence.version)) {
if (['1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(evidence.version)) {
stage = 'actual contained Electron window'; await run(['scripts/verify-desktop-window.mjs'], 'desktop-window.log');
const window = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/desktop-window-latest.json'), 'utf8'));
assert.equal(window.success, true); assert.equal(path.resolve(window.appRoot), sourceRoot); assert.equal(window.results.length, 7);
@@ -157,7 +168,7 @@ try {
actualDesktop = { window, ui, projects, scope: 'Independently rebuilt source and its shipped Electron binary; original window/engine/engineering assertions. Own credentials/HTTP fixtures; Windows native picker and complete host crash/stop have separate actual evidence.' };
}
let actualProtocolEngineering = null;
if (evidence.version === '1.17.0') {
if (['1.17.0', '1.18.0'].includes(evidence.version)) {
stage = 'Responses SSE actual Node React Python engineering';
await run(['scripts/verify-controlled-projects.mjs', '--responses'], 'responses-engineering.log');
const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/responses-engineering-location.json'), 'utf8'));
@@ -166,9 +177,19 @@ try {
assert.equal(actualProtocolEngineering.protocol, 'responses-stream'); assert.equal(actualProtocolEngineering.realModelRequests, 0);
assert.deepEqual(actualProtocolEngineering.results.map(value => value.parentCheck.results[0].tests.passed), [8, 9, 10]);
}
let actualVisionEngineering = null;
if (evidence.version === '1.18.0') {
stage = 'screenshot Task Node React Python engineering';
await run(['scripts/verify-vision-projects.mjs'], 'vision-engineering.log');
const location = JSON.parse(fs.readFileSync(path.join(sourceRoot, '.qa/vision-engineering-location.json'), 'utf8'));
actualVisionEngineering = JSON.parse(fs.readFileSync(location.result, 'utf8'));
assert.equal(actualVisionEngineering.success, true); assert.equal(actualVisionEngineering.version, evidence.version);
assert.equal(actualVisionEngineering.realModelRequests, 0);
assert.deepEqual(actualVisionEngineering.results.map(value => value.check.results[0].tests.passed), [8, 9, 10]);
}
assert.deepEqual(snapshot(), inputs, 'Source package inputs changed during validation.');
const result = { success: true, version: evidence.version, testedAt: new Date().toISOString(), sourceRoot, directory, nodeVersion: process.version, python, offlineInstall: true,
inputs, inputFiles: inputs.length, logs, buildMatchedFiles: Object.keys(expected).length, buildFiles: Object.entries(expected).map(([path, sha256]) => ({ path: path.replaceAll('\\', '/'), sha256 })), tests: stats, actualRea, actualEngine, actualBrowser, actualControlled, actualDesktop, actualCompanion, actualProtocols, actualProtocolEngineering, credentialsHandling: 'Only build/runtime environment paths are inherited; no configured model credentials are used. The controlled-engine stage supplies a declared owned local response fixture.' };
inputs, inputFiles: inputs.length, logs, buildMatchedFiles: Object.keys(expected).length, buildFiles: Object.entries(expected).map(([path, sha256]) => ({ path: path.replaceAll('\\', '/'), sha256 })), tests: stats, actualRea, actualEngine, actualBrowser, actualControlled, actualDesktop, actualCompanion, actualProtocols, actualProtocolEngineering, actualVision, actualVisionEngineering, credentialsHandling: 'Only build/runtime environment paths are inherited; no configured model credentials are used. The controlled-engine stage supplies a declared owned local response fixture.' };
save(result); console.log(JSON.stringify({ success: true, version: result.version, sourceRoot, directory, tests: stats, nodeVersion: process.version, python, inputFiles: inputs.length, buildMatchedFiles: result.buildMatchedFiles, rea: !!actualRea?.success, engine: !!actualEngine?.success, browser: !!actualBrowser?.success, controlled: !!actualControlled?.success }));
} catch (error) {
save({ success: false, version: evidence.version, testedAt: new Date().toISOString(), stage, error: error.message, sourceRoot, directory, nodeVersion: process.version, python, inputs, logs });