test(desktop): verify engineering delivery and native recovery
This commit is contained in:
@@ -8,12 +8,14 @@ import { workerHttp } from '../tests/fixtures/workers-http.mjs';
|
||||
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
||||
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
import { desktopProjectQa } from './desktop-project-qa.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-controlled-ui.mjs — real Edge desktop/390px, actual fixed Harness, owned in-memory model fixture, engine selection, reports, approvals, stop and explicit unknown recovery. No configured-model requests.'); process.exit(0);
|
||||
console.log('node scripts/verify-controlled-ui.mjs [--desktop] — real Edge or Electron desktop/390px, actual fixed Harness, owned in-memory model fixture, engine selection, reports, approvals, stop and explicit unknown recovery. No configured-model requests.'); process.exit(0);
|
||||
}
|
||||
assert.equal(process.argv.length, 2);
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, directory = path.resolve('.qa', 'controlled-ui-' + randomUUID());
|
||||
assert.ok(process.argv.slice(2).every(value => value === '--desktop')); assert.ok(process.argv.length <= 3);
|
||||
const desktopMode = process.argv.includes('--desktop');
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, directory = path.resolve('.qa', (desktopMode ? 'desktop-controlled-ui-' : 'controlled-ui-') + randomUUID());
|
||||
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT); assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex'), disposers = [], t = { after: value => disposers.push(value) };
|
||||
@@ -23,17 +25,19 @@ host.runtime.controlledEngines = new EngineRunner(f.store, host.provider, { dire
|
||||
function inputs() {
|
||||
const files = ['package.json', 'package-lock.json'];
|
||||
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) { const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file); } };
|
||||
for (const root of ['src', 'scripts', 'dist-client', 'dist-server']) walk(root);
|
||||
for (const root of ['src', 'scripts', 'dist-client', 'dist-server', ...(desktopMode ? ['dist-desktop'] : [])]) walk(root);
|
||||
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
|
||||
}
|
||||
const frozen = inputs(), browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const frozen = inputs(), desktop = desktopMode ? await desktopProjectQa({ store: f.store, provider: host.provider, runtime: host.runtime, sample: directory, kind: 'controlled-ui' }) : null;
|
||||
const browser = desktopMode ? null : await chromium.launch({ channel: 'msedge', headless: true }), page = desktop?.page ?? await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const resize = size => desktop ? desktop.resize(size.width, size.height) : page.setViewportSize(size);
|
||||
page.on('pageerror', error => pageErrors.push(error.message)); page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const panel = () => page.getByLabel('隔离协作控制台', { exact: true }), runPanel = () => page.getByLabel('受控引擎工程运行', { exact: true });
|
||||
const mark = text => { assertions.push(text); console.log('PASS ' + text); };
|
||||
async function screenshot(name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push({ file, sha256: hash(fs.readFileSync(file)) }); }
|
||||
async function closeInspector() {
|
||||
const close = page.getByRole('button', { name: '关闭检视区', exact: true });
|
||||
if (page.viewportSize().width > 940) { await expect(close).toHaveCount(0); return; }
|
||||
if (await page.evaluate(() => innerWidth) > 940) { await expect(close).toHaveCount(0); return; }
|
||||
await expect(close).toHaveCount(1);
|
||||
if (await close.isVisible()) { await close.click(); await expect(page.locator('.inspector')).toBeHidden(); }
|
||||
}
|
||||
@@ -50,8 +54,9 @@ async function applyOne() {
|
||||
}
|
||||
let result, releaseHeld;
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); await openWorkers();
|
||||
if (desktop) { await desktop.select(f.session); await resize({ width: 1440, height: 1000 }); }
|
||||
else { await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id }); await page.goto(host.url); await page.waitForLoadState('networkidle'); }
|
||||
await openWorkers();
|
||||
await panel().getByLabel('执行方式', { exact: true }).selectOption('deepseek');
|
||||
const rootField = panel().getByLabel('Harness 安装目录', { exact: true }); await rootField.fill(path.join(f.root, 'missing-installation'));
|
||||
await panel().getByRole('button', { name: '检测受控安装', exact: true }).click(); await expect(panel()).toContainText('安装尚不可用');
|
||||
@@ -70,7 +75,7 @@ try {
|
||||
const first = [...f.store.core.engine_run.values()].find(job => job.taskId === child.taskId); assert.equal(first.status, 'completed', first.error); assert.equal(first.native.cleanup.treeConfirmed, true);
|
||||
await expect(panel().locator('.worker-lane')).toHaveCount(1); await expect(panel()).toContainText('DeepSeek Harness'); await screenshot('desktop-controlled-parent');
|
||||
mark('actual Harness runs the owned bridge and returns reviewable proposals after confirmed process-tree closure');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await panel().locator('.worker-console-heading').scrollIntoViewIfNeeded();
|
||||
await resize({ width: 390, height: 844 }); await panel().locator('.worker-console-heading').scrollIntoViewIfNeeded();
|
||||
const refresh = panel().getByRole('button', { name: '刷新', exact: true });
|
||||
const bounds = await refresh.boundingBox(); assert.ok(bounds.width >= 76 && bounds.height >= 42, JSON.stringify(bounds)); assert.equal(await refresh.evaluate(node => getComputedStyle(node).whiteSpace), 'nowrap');
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390)); await screenshot('mobile-controlled-parent');
|
||||
@@ -80,8 +85,16 @@ try {
|
||||
await runPanel().getByRole('button', { name: '查看原输入', exact: true }).click(); const inputPreview = page.getByRole('dialog', { name: '引擎原执行输入', exact: true });
|
||||
await expect(inputPreview).toContainText(first.inputSha256); await expect(inputPreview.getByLabel('引擎运行原记录', { exact: true })).toContainText(child.taskId); await screenshot('mobile-controlled-input');
|
||||
await inputPreview.getByRole('button', { name: '下一页', exact: true }).click(); await expect(inputPreview.getByRole('button', { name: '回到首页', exact: true })).toBeEnabled(); await page.keyboard.press('Escape');
|
||||
const downloading = page.waitForEvent('download'); await runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click();
|
||||
const download = await downloading, reportFile = path.join(directory, download.suggestedFilename()); await download.saveAs(reportFile); assert.equal(hash(fs.readFileSync(reportFile)), first.reportSha256);
|
||||
let reportFile;
|
||||
if (desktop) {
|
||||
reportFile = path.join(directory, 'agent-workbench-engine-run-' + first.id + '.json');
|
||||
const download = await desktop.download(reportFile, () => runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click());
|
||||
assert.equal(download.suggestedFilename, path.basename(reportFile));
|
||||
} else {
|
||||
const downloading = page.waitForEvent('download'); await runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click();
|
||||
const download = await downloading; reportFile = path.join(directory, download.suggestedFilename()); await download.saveAs(reportFile);
|
||||
}
|
||||
assert.equal(hash(fs.readFileSync(reportFile)), first.reportSha256);
|
||||
mark('mobile child run view shows actual counts and closure, pages original inputs and downloads the exact original report bytes');
|
||||
let heldRoute; const latePattern = '**/api/sessions/' + child.sessionId + '/engine-runs/' + first.id + '/report**';
|
||||
await page.route(latePattern, async route => { heldRoute = route; await new Promise(resolve => { releaseHeld = resolve; }); expectedRejections.push('owned previous-run 500 response'); await route.fulfill({ status: 500, contentType: 'application/json', body: '{"error":"Owned late engine record error"}' }); });
|
||||
@@ -90,7 +103,7 @@ try {
|
||||
mark('closing a run view ignores an old delayed record failure and keeps the selected task unchanged');
|
||||
await applyOne(); await parentFromChild(); await start(); await expect.poll(() => f.store.session(child.sessionId).status, { timeout: 30000 }).toBe('awaiting_approval');
|
||||
await panel().getByRole('button', { name: '查看并审批检查', exact: true }).click(); await expect(page.locator('.approval-card')).toContainText('npm'); await expect(page.locator('.inspector')).toBeHidden(); await screenshot('mobile-controlled-approval'); await page.locator('.approval-card').getByRole('button', { name: '确认执行', exact: true }).click();
|
||||
await expect.poll(() => group.status, { timeout: 30000 }).toBe('ready_to_collect'); await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
await expect.poll(() => group.status, { timeout: 30000 }).toBe('ready_to_collect'); await resize({ width: 1440, height: 1000 });
|
||||
await openEngine(); await expect(runPanel().locator('[data-engine-run]')).toHaveCount(2); await screenshot('desktop-controlled-completed'); await page.keyboard.press('Escape');
|
||||
assert.equal(f.store.core.task.get(child.taskId).usage.requests, host.requests.length); assert.equal(f.store.core.task.get(child.taskId).usage.totalTokens, host.requests.length * 168);
|
||||
mark('explicit child file review and check approval create a new native Run, retain previous evidence and account actual model requests once');
|
||||
@@ -113,7 +126,7 @@ try {
|
||||
stoppedJob.status = 'unknown'; stoppedJob.requiresVerification = true; stoppedJob.verified = false;
|
||||
f.store.save(f.store.session(stoppedWorker.sessionId)); f.store.save(f.session);
|
||||
const originalReport = f.store.artifactStore.read(f.store.core.artifact.get(stoppedJob.reportArtifactId)), originalOutcome = structuredClone(stoppedJob.originalOutcome), previousUsage = structuredClone(stopped.usage);
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await panel().getByRole('button', { name: '刷新', exact: true }).click(); await expect(panel()).toContainText('结果待核对');
|
||||
await resize({ width: 390, height: 844 }); await panel().getByRole('button', { name: '刷新', exact: true }).click(); await expect(panel()).toContainText('结果待核对');
|
||||
await panel().getByRole('button', { name: '核对原输入与现场', exact: true }).click(); await expect.poll(() => !!stopped.evidence, { timeout: 15000 }).toBe(true);
|
||||
await expect(panel().getByRole('button', { name: '确认核对并继续', exact: true })).toBeDisabled();
|
||||
await panel().getByRole('button', { name: '查看核对记录', exact: true }).click(); await expect(page.getByRole('dialog', { name: '原输入与现场核对', exact: true })).toContainText(stoppedJob.inputSha256); await screenshot('mobile-controlled-recovery'); await page.keyboard.press('Escape');
|
||||
@@ -122,13 +135,13 @@ try {
|
||||
mark('390px recovery requires reading the owned observation before explicit review and preserves original outcome/report and cumulative budget');
|
||||
assert.deepEqual(inputs(), frozen); assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
result = { passed: true, targetVersion: '1.14.0', testedVersion: version, assertions, screenshots, pageErrors, requestFailures, expectedRejections, installation,
|
||||
engineRuns: [...f.store.core.engine_run.values()], groups: [group.id, stopped.id], inputs: frozen, realModelRequests: 0,
|
||||
scope: 'Actual native Harness and Edge with an owned predefined in-memory model fixture. Unknown recovery is a declared fixture transition; actual host-crash behavior is independently tested.' };
|
||||
engineRuns: [...f.store.core.engine_run.values()], groups: [group.id, stopped.id], inputs: frozen, realModelRequests: 0, ...(desktop ? { desktopEvents: desktop.events } : {}),
|
||||
scope: 'Actual native Harness and ' + (desktop ? 'Electron with fixed main/preload and own in-process Node host' : 'Edge') + ' with an owned predefined in-memory model fixture. Unknown recovery is a declared fixture transition; actual host-crash behavior is independently tested.' };
|
||||
} catch (error) { result = { passed: false, targetVersion: '1.14.0', testedVersion: version, error: error.stack, assertions, screenshots, pageErrors, requestFailures, realModelRequests: 0 }; await screenshot('controlled-ui-failure').catch(() => {}); throw error; }
|
||||
finally {
|
||||
releaseHeld?.(); await browser.close(); await host.runtime.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
releaseHeld?.(); await browser?.close(); await desktop?.close(); await host.runtime.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.cpSync(f.root, path.join(directory, 'original-fixture'), { recursive: true });
|
||||
for (const session of f.store.sessions.values()) fs.writeFileSync(path.join(directory, 'session-' + session.id + '.json'), JSON.stringify(await host.api('/sessions/' + session.id + '/export'), null, 2));
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/controlled-ui-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') }));
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/' + (desktopMode ? 'desktop-controlled-ui' : 'controlled-ui') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') }));
|
||||
for (const dispose of disposers.reverse()) await dispose(); console.log('EVIDENCE ' + directory);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user