test(desktop): verify engineering delivery and native recovery
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { _electron, expect } from 'playwright/test';
|
||||
import { createRequestHandler, createSessionPublisher } from '../src/server/app.mjs';
|
||||
import { hostAuthority } from '../src/core/desktop/host-authority.mjs';
|
||||
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
export async function desktopProjectQa({ store, provider, runtime, sample, kind }) {
|
||||
const appRoot = path.resolve(process.env.AGENT_WORKBENCH_DESKTOP_QA_APP_ROOT || process.cwd()), server = http.createServer(), clients = new Set(), token = randomUUID();
|
||||
runtime.appDirectory = appRoot; store.claimRuntime(); store.on('update', createSessionPublisher(clients));
|
||||
server.on('request', createRequestHandler({ appDirectory: appRoot, server, store, provider, runtime, token, clients, started: Date.now(), getPort: () => server.address().port, shutdown: () => {} }));
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||
const authority = hostAuthority({ appDirectory: appRoot, server, store, provider, runtime }); authority.initialize();
|
||||
let application, page, selected; const events = [], errors = [], screenshots = [];
|
||||
async function launch() {
|
||||
const executable = desktopExecutableIdentity(appRoot).path, packaged = executable.toLowerCase().endsWith('agentworkbench.exe');
|
||||
application = await _electron.launch({ executablePath: executable,
|
||||
args: [...(packaged ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]), '--app-root', appRoot, '--data-root', store.directory, '--node', process.execPath, '--port', String(server.address().port), '--hidden-window'],
|
||||
cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
||||
page = await application.firstWindow(); page.on('pageerror', error => errors.push(error.message));
|
||||
await page.locator('.connection.connected').waitFor({ timeout: 30000 });
|
||||
if (selected) await select(selected);
|
||||
}
|
||||
async function select(session) {
|
||||
selected = session;
|
||||
const project = page.getByRole('combobox', { name: '当前项目', exact: true });
|
||||
if (!await project.isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await project.selectOption(session.projectId);
|
||||
await expect.poll(() => page.evaluate(() => localStorage.getItem('awb-session')), { timeout: 20000 }).toBe(session.id);
|
||||
}
|
||||
async function closeInspector() { const close = page.getByRole('button', { name: '关闭检视区', exact: true }); if (await close.isVisible().catch(() => false)) await close.click(); }
|
||||
async function review(session, ids) {
|
||||
await select(session); await closeInspector();
|
||||
const prepared = [...store.core.change_set.values()].find(value => value.sessionId === session.id && value.status === 'prepared' && JSON.stringify(value.files.map(file => file.changeId).sort()) === JSON.stringify(ids.slice().sort()));
|
||||
if (prepared) {
|
||||
await page.getByRole('button', { name: '隔离协作', exact: true }).click();
|
||||
await page.getByRole('button', { name: '打开父工程变更集审查', exact: true }).click();
|
||||
await page.getByRole('button', { name: '审查变更集', exact: true }).click();
|
||||
} else {
|
||||
if (await page.getByRole('button', { name: '查看改动', exact: true }).isVisible()) await page.getByRole('button', { name: '查看改动', exact: true }).click();
|
||||
else await page.locator('.inspector-tabs').getByRole('button', { name: /^改动/ }).click();
|
||||
await page.getByRole('button', { name: '组合审查', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: '选择要一起审查的提案', exact: true }).getByRole('button', { name: '封存审查范围', exact: true }).click();
|
||||
}
|
||||
const dialog = page.getByRole('dialog', { name: '多文件变更集审查', exact: true });
|
||||
const confirm = dialog.getByRole('button', { name: '确认应用全部 ' + ids.length + ' 项', exact: true }); await confirm.waitFor();
|
||||
const matching = [...store.core.change_set.values()].filter(value => value.sessionId === session.id && value.status !== 'completed' && JSON.stringify(value.files.map(file => file.changeId).sort()) === JSON.stringify(ids.slice().sort()));
|
||||
assert.equal(matching.length, 1); const set = matching[0];
|
||||
await confirm.click(); await expect(dialog).toContainText('全部应用完成'); await page.keyboard.press('Escape');
|
||||
assert.equal(set.status, 'completed'); events.push({ action: 'actual-desktop-file-review', taskId: session.taskId, changeSetId: set.id, fingerprint: set.fingerprint, paths: set.files.map(value => value.path) }); return set;
|
||||
}
|
||||
async function approve(session, approval) {
|
||||
await select(session); await closeInspector();
|
||||
const button = page.getByRole('button', { name: '确认执行', exact: true }); await button.waitFor({ timeout: 20000 });
|
||||
assert.equal(session.approvals.filter(value => value.status === 'pending').length, 1); assert.equal(session.approvals.find(value => value.status === 'pending').id, approval.id);
|
||||
await button.click(); await expect.poll(() => approval.status).toBe('approved'); events.push({ action: 'actual-desktop-original-approval', approvalId: approval.id, taskId: session.taskId });
|
||||
}
|
||||
async function cycle(session, label) {
|
||||
await select(session); const identity = authority.identity(), eventSeq = session.eventSeq, taskId = session.taskId, runId = session.runId;
|
||||
const originalEvents = () => {
|
||||
const events = []; let cursor = 0;
|
||||
while (cursor < eventSeq) {
|
||||
const chunk = store.eventsAfter(cursor, { sessionId: session.id, limit: 500 });
|
||||
events.push(...chunk.events.filter(value => value.seq <= eventSeq));
|
||||
if (!chunk.hasMore || chunk.nextCursor >= eventSeq) break;
|
||||
assert.ok(chunk.nextCursor > cursor); cursor = chunk.nextCursor;
|
||||
}
|
||||
return events;
|
||||
};
|
||||
const retainedEvents = originalEvents(); assert.equal(retainedEvents.at(-1).seq, eventSeq);
|
||||
const completed = () => [...store.core.tool.values()].filter(value => value.sessionId === session.id && value.state === 'completed');
|
||||
const prior = hash(Buffer.from(JSON.stringify(completed()))), approvals = session.approvals.map(value => ({ id: value.id, status: value.status }));
|
||||
await application.close(); application = null;
|
||||
assert.equal(runtime.isActive(session.id), true); assert.equal(authority.identity().instanceId, identity.instanceId); assert.equal(process.pid, identity.pid);
|
||||
await launch(); assert.equal(session.taskId, taskId); assert.equal(session.runId, runId); assert.ok(session.eventSeq >= eventSeq); assert.deepEqual(originalEvents(), retainedEvents);
|
||||
const replay = await page.evaluate(async ({ sessionId, cursor }) => {
|
||||
const bootstrap = await (await fetch('/api/bootstrap')).json();
|
||||
const response = await fetch('/api/events/history?session=' + encodeURIComponent(sessionId) + '&after=' + cursor + '&limit=500', { headers: { 'x-workbench-token': bootstrap.token } });
|
||||
if (!response.ok) throw Error('Original desktop event replay failed (' + response.status + ')'); return response.json();
|
||||
}, { sessionId: session.id, cursor: eventSeq - 1 });
|
||||
assert.deepEqual(replay.events[0], retainedEvents.at(-1)); assert.ok(replay.events.every((value, index) => !index || value.seq > replay.events[index - 1].seq));
|
||||
assert.equal(hash(Buffer.from(JSON.stringify(completed()))), prior); assert.deepEqual(session.approvals.map(value => ({ id: value.id, status: value.status })), approvals);
|
||||
events.push({ action: label, hostPid: process.pid, taskId, runId, eventSeq, reopenedEventSeq: session.eventSeq, replayCursor: replay.nextCursor,
|
||||
retainedEvents: retainedEvents.length, retainedEventsSha256: hash(Buffer.from(JSON.stringify(retainedEvents))), instanceId: identity.instanceId, completedToolSha256: prior, approvals });
|
||||
}
|
||||
async function capture(session, label) {
|
||||
await select(session); await closeInspector(); await page.getByRole('button', { name: '隔离协作', exact: true }).click();
|
||||
for (const width of [1440, 390]) {
|
||||
const native = await application.browserWindow(page); await native.evaluate((value, width) => { value.showInactive(); value.setContentSize(width, 900); }, width);
|
||||
await expect.poll(() => page.evaluate(() => innerWidth)).toBe(width);
|
||||
const filename = path.join(sample, kind + '-' + label + '-' + width + '.png'); await page.screenshot({ path: filename }); fs.writeFileSync(filename + '.html', await page.content());
|
||||
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false); screenshots.push({ file: filename, width, sha256: hash(fs.readFileSync(filename)) });
|
||||
}
|
||||
await (await application.browserWindow(page)).evaluate(value => value.setContentSize(1440, 1000));
|
||||
await page.keyboard.press('Escape'); await expect(page.getByRole('dialog', { name: '隔离协作', exact: true })).toBeHidden();
|
||||
await closeInspector();
|
||||
}
|
||||
await launch();
|
||||
return { select, review, approve, cycle, capture, events, screenshots, errors, get page() { return page; },
|
||||
async resize(width, height) { await (await application.browserWindow(page)).evaluate((value, size) => { value.showInactive(); value.setContentSize(size.width, size.height); }, { width, height }); await expect.poll(() => page.evaluate(() => innerWidth)).toBe(width); },
|
||||
async download(filename, trigger) {
|
||||
const id = randomUUID();
|
||||
await application.evaluate(({ session }, { id, filename }) => {
|
||||
globalThis.workbenchQaDownloads ??= {};
|
||||
globalThis.workbenchQaDownloads[id] = { state: 'waiting' };
|
||||
session.defaultSession.once('will-download', (_event, item) => {
|
||||
const value = globalThis.workbenchQaDownloads[id]; value.suggestedFilename = item.getFilename(); value.state = 'started';
|
||||
item.setSavePath(filename); // Actual Electron DownloadItem; only the owned destination selection is a QA fixture.
|
||||
item.once('done', (_event, state) => { value.state = state; value.bytes = item.getReceivedBytes(); value.totalBytes = item.getTotalBytes(); });
|
||||
});
|
||||
}, { id, filename });
|
||||
await trigger(); await expect.poll(() => application.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id].state, id), { timeout: 30000 }).toBe('completed');
|
||||
const value = await application.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id], id);
|
||||
assert.equal(value.bytes, value.totalBytes); assert.equal(fs.statSync(filename).size, value.bytes);
|
||||
events.push({ action: 'actual-electron-native-download', ...value, file: filename, sha256: hash(fs.readFileSync(filename)), scope: 'Actual DownloadItem and transferred bytes; own output path chosen by QA without exercising the Windows save dialog.' });
|
||||
return value;
|
||||
},
|
||||
scope: 'Real Electron renderer and fixed native main/preload; own in-process Node fixture host, actual native Harness/HTTP/check/Git. The whole fixture host is never exited from its desktop UI.',
|
||||
async failure(error) { if (!page || page.isClosed()) return; const filename = path.join(sample, kind + '-desktop-failure'); fs.writeFileSync(filename + '.html', await page.content()); fs.writeFileSync(filename + '.json', JSON.stringify({ error: error.stack, dialogs: await page.getByRole('dialog').allTextContents(), errors, events }, null, 2)); await (await application.browserWindow(page)).evaluate(value => value.showInactive()); await page.screenshot({ path: filename + '.png' }); },
|
||||
async close() { await application?.close(); for (const response of clients) response.destroy(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); assert.deepEqual(errors, []); } };
|
||||
}
|
||||
@@ -12,15 +12,17 @@ import { originalEngineRun, verifyEngineModelInput } from '../src/shared/contrac
|
||||
import { resolvePurgeSource, assertPurgeModelInput, nativeSystemTexts, PURGE_DEFAULT_PROMPT } from '../src/core/engines/purge-strategy.mjs';
|
||||
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
import { desktopProjectQa } from './desktop-project-qa.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
assert.ok(args.every(value => ['--help', '--purge-comparison'].includes(value)), 'Unknown option; use --help.');
|
||||
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop'].includes(value)), 'Unknown option; use --help.');
|
||||
if (args.includes('--help')) {
|
||||
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. Comparison verifies eight independent Node projects: two Purge versions, two repetitions, enabled/disabled. Set AGENT_WORKBENCH_DEEPSEEK_ROOT; default also requires AGENT_WORKBENCH_TEST_PYTHON, comparison requires AGENT_WORKBENCH_PURGE_178_ROOT / AGENT_WORKBENCH_PURGE_181_ROOT. No configured-model requests.'); process.exit(0);
|
||||
}
|
||||
const comparisonMode = args.includes('--purge-comparison');
|
||||
const comparisonMode = args.includes('--purge-comparison'), desktopMode = args.includes('--desktop');
|
||||
assert.ok(!comparisonMode || !desktopMode, 'Desktop engineering and prompt comparison are separate acceptance runs.');
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
||||
const directory = path.resolve('.qa', (comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
|
||||
const directory = path.resolve('.qa', (desktopMode ? 'desktop-projects-' : comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT);
|
||||
assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
@@ -38,7 +40,7 @@ function inputs() {
|
||||
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
||||
const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file);
|
||||
} };
|
||||
for (const root of ['src', 'scripts', 'tests', 'benchmarks', 'dist-client', 'dist-server']) walk(root);
|
||||
for (const root of ['src', 'scripts', 'tests', 'benchmarks', 'dist-client', 'dist-server', ...(desktopMode ? ['dist-desktop'] : [])]) walk(root);
|
||||
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
|
||||
}
|
||||
const frozen = inputs(), mark = message => { cases.push(message); console.log('PASS ' + message); };
|
||||
@@ -69,7 +71,7 @@ function specification(kind, root) {
|
||||
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
|
||||
const startedAt = Date.now(), label = sampleName;
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
|
||||
fs.mkdirSync(sample); let runtime, group, result; const reviews = [], modelRequests = [];
|
||||
fs.mkdirSync(sample); let runtime, group, result, desktopQa, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
|
||||
try {
|
||||
const baseline = kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
|
||||
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
@@ -95,6 +97,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
|
||||
assertToolPairs(input.messages);
|
||||
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
|
||||
if (desktopQa && !cycledRequest) { cycledRequest = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-owned-model-request'); }
|
||||
const checks = [...f.store.core.check_job.values()].filter(job => job.taskId === worker.taskId), job = checks.at(-1);
|
||||
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === worker.taskId).at(-1);
|
||||
const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Fixed local acceptance repair; retain all tests and declarations.' }));
|
||||
@@ -116,6 +119,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
const provider = new Provider(); provider.setKey('self-owned-controlled-model-fixture'); runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
||||
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine-runs') });
|
||||
runtime.checkRunner().pythonExecutable = python; const manager = runtime.workerManager();
|
||||
if (desktopMode) desktopQa = await desktopProjectQa({ store: f.store, provider, runtime, sample, kind });
|
||||
group = await manager.plan(f.session, { goal: 'Deliver ' + kind + ' with the pinned controlled Harness, retained failures, explicit review, parent validation and local Git.',
|
||||
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256, ...(strategy ? { strategy } : {}) },
|
||||
budget: { maxTokens: 200000, maxSteps: 64, maxToolCalls: 128, maxDurationMs: 600000 },
|
||||
@@ -123,13 +127,14 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'ready', group.summary);
|
||||
const child = group.children[0], childRoot = f.store.project(child.projectId).root;
|
||||
assert.notEqual(childRoot, f.projectRoot); assert.deepEqual(ownerBytes(f), original); verify(childRoot); if (kind === 'react') install(childRoot, 'child');
|
||||
if (desktopQa) await desktopQa.select(f.store.session(child.sessionId));
|
||||
mark(label + ': pinned native engine and one owned Git worktree preserve parent staging and protected bytes');
|
||||
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
|
||||
const session = f.store.session(child.sessionId), ids = session.changes.filter(change => change.status === 'pending').map(change => change.id);
|
||||
if (ids.length) {
|
||||
assert.equal(group.status, 'awaiting_review', group.summary); const set = await runtime.changeSetManager().prepare(session, ids);
|
||||
assert.equal(group.status, 'awaiting_review', group.summary); const set = desktopQa ? await desktopQa.review(session, ids) : await runtime.changeSetManager().prepare(session, ids);
|
||||
reviews.push({ operation: 'review child batch', id: set.id, fingerprint: set.fingerprint, paths: set.files.map(file => file.path) });
|
||||
await (await runtime.changeSetManager().start(session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary);
|
||||
if (!desktopQa) await (await runtime.changeSetManager().start(session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary);
|
||||
}
|
||||
const started = manager.start(f.session, group.id, group.fingerprint); let done = false, failure;
|
||||
started.promise.then(() => { done = true; }, error => { failure = error; done = true; });
|
||||
@@ -138,7 +143,9 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
assert.equal(f.store.core.tool.get(approval.invocationId).tool, 'run_checks');
|
||||
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === child.taskId).at(-1);
|
||||
assert.deepEqual(plan.checks.map(check => check.name), spec.checks);
|
||||
reviews.push({ operation: 'approve original check', id: approval.id, taskId: child.taskId, planId: plan.id, contractSha256: plan.contractSha256 }); runtime.decide(session.id, approval.id, true);
|
||||
reviews.push({ operation: 'approve original check', id: approval.id, taskId: child.taskId, planId: plan.id, contractSha256: plan.contractSha256 });
|
||||
if (desktopQa) { if (!cycledApproval) { cycledApproval = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-original-approval'); } await desktopQa.approve(session, approval); }
|
||||
else runtime.decide(session.id, approval.id, true);
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 30));
|
||||
}
|
||||
@@ -167,10 +174,13 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
const collection = await manager.readCollection(f.session, group); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
|
||||
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'merge_prepared', group.summary);
|
||||
const merge = f.store.core.change_set.get(group.merge.changeSetId); reviews.push({ operation: 'review parent batch', id: merge.id, fingerprint: merge.fingerprint });
|
||||
await (await runtime.changeSetManager().start(f.session, merge.id, merge.fingerprint)).promise; assert.equal(merge.status, 'completed', merge.summary);
|
||||
if (desktopQa) assert.equal((await desktopQa.review(f.session, merge.files.map(file => file.changeId))).id, merge.id);
|
||||
else await (await runtime.changeSetManager().start(f.session, merge.id, merge.fingerprint)).promise;
|
||||
assert.equal(merge.status, 'completed', merge.summary);
|
||||
const checkPlan = await runtime.checkRunner().plan(f.session, spec.checks), check = await (await runtime.checkRunner().start(f.session, checkPlan.id)).promise;
|
||||
assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify(f.projectRoot);
|
||||
mark(label + ': separately reviewed parent merge passes all ' + spec.tests + ' functional tests and actual build/compile');
|
||||
if (desktopQa) await desktopQa.capture(f.session, 'parent-checks');
|
||||
const browserJobs = [];
|
||||
if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
|
||||
const steps = [ { action: 'assert_count', selector: '.task-list article', count: 3 },
|
||||
@@ -191,6 +201,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n');
|
||||
assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable);
|
||||
mark(label + ': local Git contains only reviewed functional changes and preserves original staged/unstaged owner bytes');
|
||||
if (desktopQa) { await desktopQa.capture(f.session, 'git-completed'); assert.equal(cycledRequest, true); assert.equal(cycledApproval, true); mark(label + ': real Electron reviews/approvals and window cycles retain Task/Run/eventSeq/results at 1440px and 390px'); }
|
||||
const requests = engineRuns.flatMap(job => job.requests), metrics = { durationMs: Date.now() - startedAt, baselineTree,
|
||||
contextLimit: group.config.contextLimit, budget: group.budget, model: group.model.model, tests: spec.tests,
|
||||
engineeringFailures: childChecks.filter(job => job.status === 'failed').length,
|
||||
@@ -202,10 +213,11 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
responseSequence: modelRequests.map(({ phase, responsePlan }) => ({ phase, responsePlan })) };
|
||||
result = { success: true, scope: 'actual-native-Harness-with-owned-HTTP-model-fixture', version, kind, sampleName, comparison, metrics, parentTaskId: group.taskId, groupId: group.id,
|
||||
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
|
||||
writable, reviews, modelRequests, protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
||||
writable, reviews, modelRequests, ...(desktopQa ? { desktop: { events: desktopQa.events, screenshots: desktopQa.screenshots, scope: desktopQa.scope, errors: desktopQa.errors } } : {}), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
||||
return result;
|
||||
} catch (error) { result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
await desktopQa?.close();
|
||||
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.cpSync(f.root, path.join(sample, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
for (const session of f.store.sessions.values()) fs.writeFileSync(path.join(sample, 'session-' + session.id + '.json'), JSON.stringify({ project: f.store.project(session.projectId), session, execution: f.store.executionSnapshot(session) }, null, 2));
|
||||
@@ -243,4 +255,4 @@ try {
|
||||
report = { success: true, targetVersion: '1.14.0', testedVersion: version, directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
|
||||
scope: 'Pinned actual Harness and workbench execution with an owned HTTP model fixture. Checks, approvals, failures, parent merges, Edge and Git are real. Model reasoning and Purge benefit remain separate.' };
|
||||
} catch (error) { report = { success: false, targetVersion: '1.14.0', testedVersion: version, directory, installation, cases, results, inputs: frozen, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
|
||||
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
|
||||
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
|
||||
|
||||
@@ -8,12 +8,14 @@ import { workerHttp } from '../tests/fixtures/workers-http.mjs';
|
||||
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
||||
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
import { desktopProjectQa } from './desktop-project-qa.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-controlled-ui.mjs — real Edge desktop/390px, actual fixed Harness, owned in-memory model fixture, engine selection, reports, approvals, stop and explicit unknown recovery. No configured-model requests.'); process.exit(0);
|
||||
console.log('node scripts/verify-controlled-ui.mjs [--desktop] — real Edge or Electron desktop/390px, actual fixed Harness, owned in-memory model fixture, engine selection, reports, approvals, stop and explicit unknown recovery. No configured-model requests.'); process.exit(0);
|
||||
}
|
||||
assert.equal(process.argv.length, 2);
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, directory = path.resolve('.qa', 'controlled-ui-' + randomUUID());
|
||||
assert.ok(process.argv.slice(2).every(value => value === '--desktop')); assert.ok(process.argv.length <= 3);
|
||||
const desktopMode = process.argv.includes('--desktop');
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, directory = path.resolve('.qa', (desktopMode ? 'desktop-controlled-ui-' : 'controlled-ui-') + randomUUID());
|
||||
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT); assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex'), disposers = [], t = { after: value => disposers.push(value) };
|
||||
@@ -23,17 +25,19 @@ host.runtime.controlledEngines = new EngineRunner(f.store, host.provider, { dire
|
||||
function inputs() {
|
||||
const files = ['package.json', 'package-lock.json'];
|
||||
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) { const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file); } };
|
||||
for (const root of ['src', 'scripts', 'dist-client', 'dist-server']) walk(root);
|
||||
for (const root of ['src', 'scripts', 'dist-client', 'dist-server', ...(desktopMode ? ['dist-desktop'] : [])]) walk(root);
|
||||
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
|
||||
}
|
||||
const frozen = inputs(), browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const frozen = inputs(), desktop = desktopMode ? await desktopProjectQa({ store: f.store, provider: host.provider, runtime: host.runtime, sample: directory, kind: 'controlled-ui' }) : null;
|
||||
const browser = desktopMode ? null : await chromium.launch({ channel: 'msedge', headless: true }), page = desktop?.page ?? await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const resize = size => desktop ? desktop.resize(size.width, size.height) : page.setViewportSize(size);
|
||||
page.on('pageerror', error => pageErrors.push(error.message)); page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const panel = () => page.getByLabel('隔离协作控制台', { exact: true }), runPanel = () => page.getByLabel('受控引擎工程运行', { exact: true });
|
||||
const mark = text => { assertions.push(text); console.log('PASS ' + text); };
|
||||
async function screenshot(name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push({ file, sha256: hash(fs.readFileSync(file)) }); }
|
||||
async function closeInspector() {
|
||||
const close = page.getByRole('button', { name: '关闭检视区', exact: true });
|
||||
if (page.viewportSize().width > 940) { await expect(close).toHaveCount(0); return; }
|
||||
if (await page.evaluate(() => innerWidth) > 940) { await expect(close).toHaveCount(0); return; }
|
||||
await expect(close).toHaveCount(1);
|
||||
if (await close.isVisible()) { await close.click(); await expect(page.locator('.inspector')).toBeHidden(); }
|
||||
}
|
||||
@@ -50,8 +54,9 @@ async function applyOne() {
|
||||
}
|
||||
let result, releaseHeld;
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); await openWorkers();
|
||||
if (desktop) { await desktop.select(f.session); await resize({ width: 1440, height: 1000 }); }
|
||||
else { await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id }); await page.goto(host.url); await page.waitForLoadState('networkidle'); }
|
||||
await openWorkers();
|
||||
await panel().getByLabel('执行方式', { exact: true }).selectOption('deepseek');
|
||||
const rootField = panel().getByLabel('Harness 安装目录', { exact: true }); await rootField.fill(path.join(f.root, 'missing-installation'));
|
||||
await panel().getByRole('button', { name: '检测受控安装', exact: true }).click(); await expect(panel()).toContainText('安装尚不可用');
|
||||
@@ -70,7 +75,7 @@ try {
|
||||
const first = [...f.store.core.engine_run.values()].find(job => job.taskId === child.taskId); assert.equal(first.status, 'completed', first.error); assert.equal(first.native.cleanup.treeConfirmed, true);
|
||||
await expect(panel().locator('.worker-lane')).toHaveCount(1); await expect(panel()).toContainText('DeepSeek Harness'); await screenshot('desktop-controlled-parent');
|
||||
mark('actual Harness runs the owned bridge and returns reviewable proposals after confirmed process-tree closure');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await panel().locator('.worker-console-heading').scrollIntoViewIfNeeded();
|
||||
await resize({ width: 390, height: 844 }); await panel().locator('.worker-console-heading').scrollIntoViewIfNeeded();
|
||||
const refresh = panel().getByRole('button', { name: '刷新', exact: true });
|
||||
const bounds = await refresh.boundingBox(); assert.ok(bounds.width >= 76 && bounds.height >= 42, JSON.stringify(bounds)); assert.equal(await refresh.evaluate(node => getComputedStyle(node).whiteSpace), 'nowrap');
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390)); await screenshot('mobile-controlled-parent');
|
||||
@@ -80,8 +85,16 @@ try {
|
||||
await runPanel().getByRole('button', { name: '查看原输入', exact: true }).click(); const inputPreview = page.getByRole('dialog', { name: '引擎原执行输入', exact: true });
|
||||
await expect(inputPreview).toContainText(first.inputSha256); await expect(inputPreview.getByLabel('引擎运行原记录', { exact: true })).toContainText(child.taskId); await screenshot('mobile-controlled-input');
|
||||
await inputPreview.getByRole('button', { name: '下一页', exact: true }).click(); await expect(inputPreview.getByRole('button', { name: '回到首页', exact: true })).toBeEnabled(); await page.keyboard.press('Escape');
|
||||
const downloading = page.waitForEvent('download'); await runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click();
|
||||
const download = await downloading, reportFile = path.join(directory, download.suggestedFilename()); await download.saveAs(reportFile); assert.equal(hash(fs.readFileSync(reportFile)), first.reportSha256);
|
||||
let reportFile;
|
||||
if (desktop) {
|
||||
reportFile = path.join(directory, 'agent-workbench-engine-run-' + first.id + '.json');
|
||||
const download = await desktop.download(reportFile, () => runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click());
|
||||
assert.equal(download.suggestedFilename, path.basename(reportFile));
|
||||
} else {
|
||||
const downloading = page.waitForEvent('download'); await runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click();
|
||||
const download = await downloading; reportFile = path.join(directory, download.suggestedFilename()); await download.saveAs(reportFile);
|
||||
}
|
||||
assert.equal(hash(fs.readFileSync(reportFile)), first.reportSha256);
|
||||
mark('mobile child run view shows actual counts and closure, pages original inputs and downloads the exact original report bytes');
|
||||
let heldRoute; const latePattern = '**/api/sessions/' + child.sessionId + '/engine-runs/' + first.id + '/report**';
|
||||
await page.route(latePattern, async route => { heldRoute = route; await new Promise(resolve => { releaseHeld = resolve; }); expectedRejections.push('owned previous-run 500 response'); await route.fulfill({ status: 500, contentType: 'application/json', body: '{"error":"Owned late engine record error"}' }); });
|
||||
@@ -90,7 +103,7 @@ try {
|
||||
mark('closing a run view ignores an old delayed record failure and keeps the selected task unchanged');
|
||||
await applyOne(); await parentFromChild(); await start(); await expect.poll(() => f.store.session(child.sessionId).status, { timeout: 30000 }).toBe('awaiting_approval');
|
||||
await panel().getByRole('button', { name: '查看并审批检查', exact: true }).click(); await expect(page.locator('.approval-card')).toContainText('npm'); await expect(page.locator('.inspector')).toBeHidden(); await screenshot('mobile-controlled-approval'); await page.locator('.approval-card').getByRole('button', { name: '确认执行', exact: true }).click();
|
||||
await expect.poll(() => group.status, { timeout: 30000 }).toBe('ready_to_collect'); await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
await expect.poll(() => group.status, { timeout: 30000 }).toBe('ready_to_collect'); await resize({ width: 1440, height: 1000 });
|
||||
await openEngine(); await expect(runPanel().locator('[data-engine-run]')).toHaveCount(2); await screenshot('desktop-controlled-completed'); await page.keyboard.press('Escape');
|
||||
assert.equal(f.store.core.task.get(child.taskId).usage.requests, host.requests.length); assert.equal(f.store.core.task.get(child.taskId).usage.totalTokens, host.requests.length * 168);
|
||||
mark('explicit child file review and check approval create a new native Run, retain previous evidence and account actual model requests once');
|
||||
@@ -113,7 +126,7 @@ try {
|
||||
stoppedJob.status = 'unknown'; stoppedJob.requiresVerification = true; stoppedJob.verified = false;
|
||||
f.store.save(f.store.session(stoppedWorker.sessionId)); f.store.save(f.session);
|
||||
const originalReport = f.store.artifactStore.read(f.store.core.artifact.get(stoppedJob.reportArtifactId)), originalOutcome = structuredClone(stoppedJob.originalOutcome), previousUsage = structuredClone(stopped.usage);
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await panel().getByRole('button', { name: '刷新', exact: true }).click(); await expect(panel()).toContainText('结果待核对');
|
||||
await resize({ width: 390, height: 844 }); await panel().getByRole('button', { name: '刷新', exact: true }).click(); await expect(panel()).toContainText('结果待核对');
|
||||
await panel().getByRole('button', { name: '核对原输入与现场', exact: true }).click(); await expect.poll(() => !!stopped.evidence, { timeout: 15000 }).toBe(true);
|
||||
await expect(panel().getByRole('button', { name: '确认核对并继续', exact: true })).toBeDisabled();
|
||||
await panel().getByRole('button', { name: '查看核对记录', exact: true }).click(); await expect(page.getByRole('dialog', { name: '原输入与现场核对', exact: true })).toContainText(stoppedJob.inputSha256); await screenshot('mobile-controlled-recovery'); await page.keyboard.press('Escape');
|
||||
@@ -122,13 +135,13 @@ try {
|
||||
mark('390px recovery requires reading the owned observation before explicit review and preserves original outcome/report and cumulative budget');
|
||||
assert.deepEqual(inputs(), frozen); assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
result = { passed: true, targetVersion: '1.14.0', testedVersion: version, assertions, screenshots, pageErrors, requestFailures, expectedRejections, installation,
|
||||
engineRuns: [...f.store.core.engine_run.values()], groups: [group.id, stopped.id], inputs: frozen, realModelRequests: 0,
|
||||
scope: 'Actual native Harness and Edge with an owned predefined in-memory model fixture. Unknown recovery is a declared fixture transition; actual host-crash behavior is independently tested.' };
|
||||
engineRuns: [...f.store.core.engine_run.values()], groups: [group.id, stopped.id], inputs: frozen, realModelRequests: 0, ...(desktop ? { desktopEvents: desktop.events } : {}),
|
||||
scope: 'Actual native Harness and ' + (desktop ? 'Electron with fixed main/preload and own in-process Node host' : 'Edge') + ' with an owned predefined in-memory model fixture. Unknown recovery is a declared fixture transition; actual host-crash behavior is independently tested.' };
|
||||
} catch (error) { result = { passed: false, targetVersion: '1.14.0', testedVersion: version, error: error.stack, assertions, screenshots, pageErrors, requestFailures, realModelRequests: 0 }; await screenshot('controlled-ui-failure').catch(() => {}); throw error; }
|
||||
finally {
|
||||
releaseHeld?.(); await browser.close(); await host.runtime.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
releaseHeld?.(); await browser?.close(); await desktop?.close(); await host.runtime.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.cpSync(f.root, path.join(directory, 'original-fixture'), { recursive: true });
|
||||
for (const session of f.store.sessions.values()) fs.writeFileSync(path.join(directory, 'session-' + session.id + '.json'), JSON.stringify(await host.api('/sessions/' + session.id + '/export'), null, 2));
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/controlled-ui-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') }));
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/' + (desktopMode ? 'desktop-controlled-ui' : 'controlled-ui') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') }));
|
||||
for (const dispose of disposers.reverse()) await dispose(); console.log('EVIDENCE ' + directory);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user