159 lines
14 KiB
JavaScript
159 lines
14 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import net from 'node:net';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { chromium, _electron } from 'playwright';
|
|
import { fixture, compatibleFixture, launchWorkbench, until, call, completion } from '../tests/helpers.mjs';
|
|
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
|
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
|
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
|
|
|
// Existing UI-verifier convention: a real browser/Electron, isolated data and a
|
|
// localhost model protocol fixture. No configured provider or real key is used.
|
|
const desktop = process.argv.includes('--desktop');
|
|
const appRoot = path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT || '.');
|
|
const directory = path.resolve('.qa/requirements-progress', (desktop ? 'shadow-electron-' : 'shadow-ui-') + randomUUID());
|
|
fs.mkdirSync(directory, { recursive: true });
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t);
|
|
let scenario = 'pages', requests = 0;
|
|
const fixtureRequests = [];
|
|
const model = await compatibleFixture(t, () => {
|
|
const request = ++requests;
|
|
fixtureRequests.push({ scenario, request, receivedAt: new Date().toISOString() });
|
|
if (scenario === 'network') return { status: 502, body: { error: { message: 'Owned shadow gateway failure, original response retained' } } };
|
|
if (scenario === 'approval') return { body: request === 1 ? completion([
|
|
call('read-before-approval', 'read_file', { path: 'README.md', limit: 10 }),
|
|
call('owned-command', 'run_command', { command: 'node --version', reason: '所属只读环境检查,需原审批' }),
|
|
]) : completion([], '原审批已拒绝,说明与原工具结果保留。') };
|
|
return { body: completion([call('page-' + request, 'read_file', { path: 'README.md', offset: request === 1 ? 0 : 10, limit: 10 })]) };
|
|
});
|
|
f.session.title = '单轮观察分页原任务'; f.store.save(f.session);
|
|
const networkSession = f.store.createSession(f.project.id); networkSession.title = '单轮观察原网络失败'; f.store.save(networkSession);
|
|
f.store.saveConfig({ mode: 'compatible', model: 'owned-shadow-ui', baseUrl: model.baseUrl, maxSteps: 3 }); f.store.close();
|
|
const passed = [], screenshots = [], errors = [], snapshots = [], sourceArtifacts = new Map(); let browser, application, controller, host, page, window, result;
|
|
const mark = name => { passed.push(name); console.log('PASS ' + name); };
|
|
const panel = () => page.getByLabel('单轮观察', { exact: true });
|
|
async function visiblePanel() {
|
|
if (!await page.getByLabel('选择运行记录', { exact: true }).isVisible()) await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
|
await panel().waitFor();
|
|
if (!await panel().evaluate(element => element.open)) await panel().locator('summary').click();
|
|
}
|
|
async function select(session) {
|
|
if (!await page.getByLabel('当前项目').isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
|
await page.getByLabel('当前项目').selectOption(f.project.id);
|
|
await page.locator('.session-list button').filter({ hasText: session.title }).click();
|
|
}
|
|
async function resize(width, height) {
|
|
if (window) { await window.evaluate((win, size) => win.setContentSize(size.width, size.height), { width, height }); await until(() => page.evaluate(w => innerWidth === w, width)); }
|
|
else await page.setViewportSize({ width, height });
|
|
}
|
|
async function shot(name) {
|
|
await panel().scrollIntoViewIfNeeded(); const file = path.join(directory, name + '.png');
|
|
await page.screenshot({ path: file, scale: 'css' }); screenshots.push(file);
|
|
}
|
|
async function state(session = f.session) { return host.api('/sessions/' + session.id); }
|
|
async function originalArtifact(artifact, session) {
|
|
let content = '', offset = 0, page;
|
|
do { page = await host.api('/artifacts/' + artifact.id + '?session=' + session.id + '&offset=' + offset + '&limit=16000'); content += page.content; offset += page.content.length; }
|
|
while (page.truncated && page.content.length);
|
|
const bytes = Buffer.from(content); assert.equal(createHash('sha256').update(bytes).digest('hex'), artifact.sha256);
|
|
assert.equal(bytes.length, artifact.bytes); return content;
|
|
}
|
|
async function saved(name, session = f.session) {
|
|
const snapshot = await host.api('/sessions/' + session.id + '/export');
|
|
for (const artifact of snapshot.execution.artifacts.filter(row => ['turn-progress-shadow', 'tool-result', 'model-http-response', 'model-stream-response', 'delivery-report', 'delivery-report-markdown'].includes(row.kind))) {
|
|
if (sourceArtifacts.has(artifact.id)) continue;
|
|
const content = await originalArtifact(artifact, session), relative = 'artifacts/' + artifact.id + '.txt';
|
|
fs.mkdirSync(path.join(directory, 'artifacts'), { recursive: true }); fs.writeFileSync(path.join(directory, relative), content);
|
|
sourceArtifacts.set(artifact.id, { id: artifact.id, kind: artifact.kind, sha256: artifact.sha256, bytes: artifact.bytes, path: relative });
|
|
}
|
|
fs.writeFileSync(path.join(directory, name + '.json'), JSON.stringify(snapshot, null, 2)); snapshots.push(name + '.json'); return snapshot;
|
|
}
|
|
try {
|
|
if (desktop) {
|
|
const socket = net.createServer(); await new Promise(resolve => socket.listen(0, '127.0.0.1', resolve));
|
|
const port = socket.address().port; await new Promise(resolve => socket.close(resolve)); assert.notEqual(port, 4318);
|
|
controller = new HostController({ appRoot, dataRoot: f.data, port, nodePath: process.execPath, timeoutMs: 60000 });
|
|
await controller.start(); host = { url: controller.url, bootstrap: await controller.api('/bootstrap'),
|
|
api: (route, input) => controller.api(route, input === undefined ? {} : { method: 'POST', input }) };
|
|
const executable = desktopExecutableIdentity(appRoot).path;
|
|
application = await _electron.launch({ executablePath: executable,
|
|
args: [...(executable.toLowerCase().endsWith('agentworkbench.exe') ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]), '--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', String(port), '--hidden-window'],
|
|
cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
|
page = await application.firstWindow(); window = await application.browserWindow(page);
|
|
await window.evaluate(win => { win.showInactive(); win.setContentSize(1440, 1080); }); await until(() => page.evaluate(() => innerWidth === 1440));
|
|
assert.equal(await page.evaluate(() => typeof globalThis.process), 'undefined');
|
|
} else {
|
|
host = await launchWorkbench(t, f.data, {}, { appRoot }); assert.notEqual(new URL(host.url).port, '4318');
|
|
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ viewport: { width: 1440, height: 1080 } }); await page.goto(host.url);
|
|
}
|
|
assert.equal(host.bootstrap.config.hasKey, false); page.on('pageerror', error => errors.push(error.message));
|
|
await page.locator('.connection.connected').waitFor(); await select(f.session);
|
|
await host.api('/sessions/' + f.session.id + '/messages', { content: '只研究README.md原说明,不修改文件;本机固定分页观察夹具。' });
|
|
await until(async () => (await state()).status === 'paused', 25000);
|
|
const first = await state(), firstRun = first.execution.run;
|
|
assert.equal(requests, 3); assert.equal(firstRun.stopReason, 'step_limit'); await visiblePanel();
|
|
await until(async () => (await panel().textContent()).includes('重复读取候选'));
|
|
assert.match(await panel().textContent(), /第 1 轮 · 新增外显证据/); assert.match(await panel().textContent(), /第 3 轮 · 重复读取候选/);
|
|
assert.equal(firstRun.progressShadow.turns[1].delta.read_characters, 10); assert.equal(firstRun.progressShadow.turns[2].delta.read_characters, 10);
|
|
await shot('desktop-pages'); await saved('pages-original'); mark('actual runtime pages grow then repeat under the original three-round limit');
|
|
await panel().locator('li').nth(1).getByRole('button', { name: '查看原观察收据', exact: true }).click();
|
|
const dialog = page.getByRole('dialog', { name: '执行结果工件', exact: true }); await dialog.waitFor();
|
|
assert.match(await dialog.textContent(), /evidence_growth/); assert.match(await dialog.textContent(), /result_artifact_id/);
|
|
await dialog.getByRole('button', { name: '关闭窗口', exact: true }).click(); mark('original observation receipt opens with the actual tool-result provenance');
|
|
await page.reload(); await page.locator('.connection.connected').waitFor(); await visiblePanel();
|
|
assert.equal(await page.getByLabel('选择运行记录').inputValue(), firstRun.id); assert.match(await panel().textContent(), /重复读取候选/);
|
|
mark('refresh restores the same Task, Run and persisted shadow observations');
|
|
await resize(390, 844); await visiblePanel();
|
|
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), true);
|
|
assert.ok((await panel().getByRole('button', { name: '查看原观察收据', exact: true }).first().boundingBox()).height >= 44);
|
|
await shot('narrow-pages'); mark('390px keeps observation facts and original receipt actions reachable without page overflow');
|
|
await resize(1440, 1080); scenario = 'approval'; requests = 0;
|
|
await host.api('/sessions/' + f.session.id + '/resume', { content: '继续核对所属环境;命令只经原审批执行。' });
|
|
await until(async () => (await state()).status === 'awaiting_approval', 25000); await visiblePanel();
|
|
await until(async () => (await panel().textContent()).includes('等待所属操作')); await shot('waiting-approval');
|
|
const waiting = await state(); assert.equal(waiting.execution.run.progressShadow.turns.at(-1).classification, 'waiting');
|
|
await saved('approval-waiting'); mark('an actual pending approval stays waiting and does not consume the earlier same-turn read twice');
|
|
await page.getByRole('button', { name: '拒绝执行', exact: true }).click();
|
|
await until(async () => (await state()).status === 'completed', 25000); const second = await state();
|
|
assert.equal(requests, 2); assert.equal(second.taskId, first.taskId); assert.notEqual(second.execution.run.id, firstRun.id);
|
|
assert.equal(second.execution.run.progressShadow.turns.find(row => row.phase === 'settled').delta.repeated_reads, 0);
|
|
await until(async () => (await panel().textContent()).includes('新失败 1')); await shot('denied-approval');
|
|
assert.match(await panel().textContent(), /失败类别:权限或审批拒绝/);
|
|
await saved('approval-denied'); mark('refusing the original approval preserves the permission failure and the original tool pairs');
|
|
await page.getByLabel('选择运行记录').selectOption(firstRun.id);
|
|
await until(async () => (await panel().textContent()).includes('第 3 轮 · 重复读取候选'));
|
|
await shot('historical-run'); await page.getByLabel('选择运行记录').selectOption(second.execution.run.id);
|
|
await until(async () => !(await panel().textContent()).includes('第 3 轮')); mark('selecting historical and current Runs keeps their independent observations');
|
|
await host.api('/sessions/' + f.session.id + '/delivery-reports', {});
|
|
const reportExport = await saved('report-original');
|
|
const reportArtifact = reportExport.execution.artifacts.filter(row => row.kind === 'delivery-report').at(-1);
|
|
const report = JSON.parse(await originalArtifact(reportArtifact, f.session));
|
|
assert.equal(report.turnObservations.length, 2); assert.ok(report.turnObservations.every(row => row.evidence_verified));
|
|
fs.writeFileSync(path.join(directory, 'report.json'), JSON.stringify(report, null, 2)); mark('delivery report uses both Runs and independently verifies the original observation receipts');
|
|
scenario = 'network'; requests = 0; await select(networkSession);
|
|
await host.api('/sessions/' + networkSession.id + '/messages', { content: '读取原工程,保存失败来源;本机502夹具。' });
|
|
await until(async () => (await state(networkSession)).status === 'failed', 25000); await visiblePanel();
|
|
await until(async () => (await panel().textContent()).includes('新增失败证据'));
|
|
assert.match(await panel().textContent(), /失败类别:上游服务/);
|
|
const failure = await state(networkSession); assert.equal(failure.lastFailure.httpStatus, 502); assert.equal(failure.execution.run.stopReason, null);
|
|
await shot('network-original-failure'); await saved('network-original', networkSession);
|
|
mark('original HTTP 502 remains a model failure and is not relabeled as no progress'); assert.deepEqual(errors, []);
|
|
result = { success: true, version: host.bootstrap.version, appRoot, passed, errors, screenshots, snapshots, actualElectron: desktop, installationValidated: false, realModelRequests: 0, officialServiceTouched: false };
|
|
} catch (error) {
|
|
try { await saved('failure-session'); } catch (captureError) {
|
|
fs.writeFileSync(path.join(directory, 'failure-session-error.txt'), captureError.stack);
|
|
}
|
|
if (page) { try { await page.screenshot({ path: path.join(directory, 'failure.png') }); fs.writeFileSync(path.join(directory, 'failure-dom.txt'), await page.locator('body').innerText()); } catch {} }
|
|
result = { success: false, passed, errors, screenshots, snapshots, error: error.stack, actualElectron: desktop, realModelRequests: 0, officialServiceTouched: false };
|
|
} finally {
|
|
fs.writeFileSync(path.join(directory, 'fixture-requests.json'), JSON.stringify(fixtureRequests, null, 2));
|
|
if (application) await application.close(); if (browser) await browser.close();
|
|
if (controller && await controller.health()) await controller.stop();
|
|
for (const dispose of disposers.reverse()) await dispose();
|
|
fs.writeFileSync(path.join(directory, 'source-artifacts.json'), JSON.stringify([...sourceArtifacts.values()], null, 2));
|
|
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
|
}
|
|
console.log(JSON.stringify({ directory, ...result })); if (!result.success) process.exitCode = 1;
|