Files
Agent-Workbench/scripts/verify-context-ui.mjs
T
2026-10-31 11:50:51 +08:00

197 lines
18 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { releaseEvidencePrefix } from './qa-evidence.mjs';
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID, createHash } from 'node:crypto';
import { spawn } from 'node:child_process';
import { chromium, expect } from 'playwright/test';
import { Store } from '../src/server/store.mjs';
import { RequestPlanner } from '../src/core/context/request-plan.mjs';
import { toolDefinitions } from '../src/server/provider.mjs';
import { call } from '../tests/helpers.mjs';
const root = path.resolve('.qa', 'context-ui-' + randomUUID()), data = path.join(root, 'data');
fs.mkdirSync(root, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
const store = new Store(data), cases = {};
for (const name of ['long', 'other', 'paused', 'paused-repeat', 'working']) {
const projectRoot = path.join(root, 'context-' + name); fs.mkdirSync(projectRoot);
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned context UI fixture\n');
const project = store.addProject(projectRoot), session = store.createSession(project.id, '上下文验收 ' + name);
const goal = '这是独立界面验收样本。完整任务要求与实际原记录保留,读取工件不得重新执行工具。';
store.message(session, 'user', goal); session.providerMessages.push({ role: 'user', content: goal });
const run = store.beginRun(session, goal, false, store.config), planner = new RequestPlanner(store);
if (name === 'paused' || name === 'paused-repeat') {
session.status = 'paused'; store.message(session, 'system', name === 'paused' ? '完整任务要求超过请求上下文预算,请核对配置后继续。' : '模型重复读取已覆盖工件区间,原任务已暂停,请核对后继续。');
if (name === 'paused-repeat') store.core.task.get(run.taskId).readProgress = { schemaVersion: 1, reads: [], repeated: 3, changingIndexes: [], warning: '重复读取已覆盖区间', pause: true };
store.finishRun(session, run, name === 'paused' ? 'context_limit' : 'no_progress'); cases[name] = { project, session, projectRoot, runs: [run.id] }; continue;
}
if (name === 'long') for (let index = 0; index < 18; index++) {
const tool = call('ui-original-' + index, 'read_file', { path: 'README.md', fixture: 'Preserve exact historical JSON.' });
const event = { id: randomUUID(), toolCallId: tool.id, tool: 'read_file', kind: 'tool', phase: 'queued', label: '读取文件',
createdAt: new Date().toISOString(), startedAt: new Date().toISOString(), input: { path: 'README.md' }, runId: run.id, taskId: run.taskId };
session.events.push(event); store.registerInvocation(session, tool, run, event);
store.beginInvocation(session, tool, run, event); store.invocationState(session, event, 'running');
const result = { status: 'completed', path: 'README.md', content: ('完整原记录🙂 ' + index + '\n').repeat(1200) };
event.phase = 'completed'; event.output = '已保存完整原记录。'; event.finishedAt = new Date().toISOString();
store.finishInvocation(session, event, result);
session.providerMessages.push({ role: 'assistant', content: 'Saved fixture read.', tool_calls: [tool] },
{ role: 'tool', tool_call_id: tool.id, content: JSON.stringify(result) });
}
if (name === 'working') {
const content = 'export const actual = "当前任务真实代码🙂";\n';
fs.writeFileSync(path.join(projectRoot, 'actual.mjs'), content);
const values = [{ tool: 'read_file', args: { path: 'actual.mjs' }, result: { path: 'actual.mjs', content,
sha256: createHash('sha256').update(content).digest('hex'), start_line: 1, end_line: 2, total_lines: 2, truncated: false } },
{ tool: 'query_code', args: { query: 'large' }, result: { results: 'Large owned mixed group '.repeat(16000) } }];
const calls = values.map(value => call(randomUUID(), value.tool, value.args)), messages = [];
for (const [index, invocation] of calls.entries()) {
const event = store.event(session, { kind: 'tool', tool: invocation.function.name, toolCallId: invocation.id,
runId: run.id, taskId: run.taskId, phase: 'queued', startedAt: new Date().toISOString(), label: '工作依据样本' });
store.registerInvocation(session, invocation, run, event); store.beginInvocation(session, invocation, run, event);
store.finishInvocation(session, event, values[index].result);
messages.push({ role: 'tool', tool_call_id: invocation.id, content: JSON.stringify(values[index].result) });
}
session.providerMessages.push({ role: 'assistant', content: '', tool_calls: calls }, ...messages); store.save(session);
}
const step = store.beginStep(session, run), first = planner.prepare(session, run, store.config, {
system: goal + '\n' + JSON.stringify({ project_id: project.id, session_id: session.id, task_id: run.taskId }), tools: toolDefinitions, stepId: step.id });
const response = { message: { role: 'assistant', content: '独立界面样本已保存;没有请求真实模型。' }, usage: { prompt_tokens: 2400, completion_tokens: 300 } };
planner.finish(session, first.record, { response, attempts: 1 }); session.providerMessages.push(response.message);
store.message(session, 'assistant', response.message.content); store.finishStep(session, step); session.status = 'completed'; store.finishRun(session, run);
const runs = [run.id], plans = [first.record.id];
if (name === 'long') {
const resumed = store.beginRun(session, '', true, store.config), nextStep = store.beginStep(session, resumed);
const next = planner.prepare(session, resumed, store.config, { system: goal, tools: toolDefinitions, stepId: nextStep.id });
planner.finish(session, next.record, { response, attempts: 1 }); store.finishStep(session, nextStep);
session.status = 'completed'; store.finishRun(session, resumed); runs.push(resumed.id); plans.push(next.record.id);
}
cases[name] = { project, session, projectRoot, runs, plans };
}
store.close();
const env = { ...process.env, AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' };
delete env.AGENT_WORKBENCH_API_KEY;
const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '', errors = '', url = '';
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const item = JSON.parse(line); if (item.ready) url = item.url; } catch {} } });
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
await until(() => url || child.exitCode !== null); assert.ok(url, errors);
const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
const proofPrefix = bootstrap.version === '1.6.0' ? 'workspaces-context' : releaseEvidencePrefix(bootstrap.version);
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
const passed = [], pageErrors = [], requestFailures = [], screenshots = [];
page.on('pageerror', error => pageErrors.push(error.message));
page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') requestFailures.push(request.failure()?.errorText); });
const mark = value => { passed.push(value); console.log('PASS ' + value); };
const panel = () => page.getByLabel('请求上下文', { exact: true }), history = () => page.getByRole('dialog', { name: '完整历史记录', exact: true });
async function screenshot(name) {
const file = '.qa/screenshots/' + proofPrefix + '-' + name + '.png'; await page.screenshot({ path: file, fullPage: true }); screenshots.push(file);
}
async function choose(name) {
const select = page.getByLabel('当前项目', { exact: true });
if (await select.inputValue() !== cases[name].project.id) {
if (!await select.isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
await expect(select).toBeVisible(); await select.selectOption(cases[name].project.id);
}
await expect(page.locator('.session-item.selected')).toContainText('上下文验收 ' + name);
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
await expect(page.getByLabel('任务记录', { exact: true })).toBeVisible();
}
async function lateFailure(pattern, trigger, label) {
let release, entered = false, finished = false;
const gate = new Promise(resolve => { release = resolve; });
await page.route(pattern, async route => { entered = true; await gate; await route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: label }) }); finished = true; });
await trigger(); await until(() => entered);
if (await history().isVisible()) await page.keyboard.press('Escape');
await choose('other'); release(); await until(() => finished);
await page.waitForTimeout(350); await expect(page.locator('.toast')).toHaveCount(0);
await expect(panel()).toContainText('2,400 tokens');
await page.unroute(pattern); mark(label + ' stays in its original scope after switching projects');
}
try {
await page.goto(url); await page.waitForLoadState('networkidle'); await choose('long');
await expect(panel()).toContainText('2,400 tokens'); await expect(panel()).toContainText('16,000 tokens');
await expect(panel().getByRole('button', { name: '检索完整历史', exact: true })).toBeVisible();
await panel().scrollIntoViewIfNeeded(); await screenshot('desktop-plan'); mark('desktop request estimate, actual usage, complete goal and history identity are visible');
await panel().getByRole('button', { name: '检索完整历史', exact: true }).click();
await expect(history().locator('.context-history-group')).toHaveCount(12);
await history().getByRole('button', { name: '下一页', exact: true }).click();
await expect(history().locator('.artifact-pagination')).toContainText('13–18 / 18');
await history().getByRole('button', { name: '上一页', exact: true }).click();
await expect(history().locator('.artifact-pagination')).toContainText('1–12 / 18'); await screenshot('desktop-history');
await history().getByRole('button', { name: '读取工具结果', exact: true }).first().click();
const artifact = page.getByRole('dialog', { name: '执行结果工件', exact: true });
await expect(artifact.locator('pre')).toContainText('完整原记录');
await page.keyboard.press('Escape'); mark('paged complete history opens actual tool result artifacts without executing tools');
await page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[0]);
await expect(panel()).toContainText('2,400 tokens');
const selected = await api('/sessions/' + cases.long.session.id + '/context-plans?run=' + cases.long.runs[0]);
await expect(panel().locator('.context-sha')).toHaveAttribute('title', selected[0].requestSha256);
await page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[1]); mark('historical and resumed Runs retain their own request records');
await page.setViewportSize({ width: 390, height: 844 }); await choose('long');
await panel().getByRole('button', { name: '检索完整历史', exact: true }).click();
await expect(history().locator('.context-history-group')).toHaveCount(12); await screenshot('mobile-history');
const bounds = await history().boundingBox(); assert.ok(bounds.x >= 0 && bounds.x + bounds.width <= 390 && bounds.y + bounds.height <= 844);
await history().getByRole('button', { name: '读取原记录', exact: true }).first().click(); await expect(artifact.locator('pre')).toContainText('Preserve exact historical JSON');
await screenshot('mobile-original'); await page.keyboard.press('Escape');
await expect(panel()).toBeVisible();
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click(); await expect(artifact.locator('pre')).toContainText('history_index');
await screenshot('mobile-request-body'); await page.keyboard.press('Escape');
await expect(panel()).toBeVisible();
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth) <= 390); mark('390px history, original JSON and sent-body dialogs fit and retain close controls');
await page.setViewportSize({ width: 1440, height: 1000 }); await choose('long');
const currentPlanId = cases.long.plans[1];
await lateFailure('**/context-plans/' + currentPlanId + '/history?*', () => panel().getByRole('button', { name: '检索完整历史', exact: true }).click(), 'late old history failure');
await choose('long');
await lateFailure('**/context-plans?run=' + cases.long.runs[0], () => page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[0]), 'late old Run failure');
await choose('long');
const currentPlan = await api('/sessions/' + cases.long.session.id + '/context-plans/' + currentPlanId);
await lateFailure('**/artifacts/' + currentPlan.requestArtifactId + '?*', () => panel().getByRole('button', { name: '查看发送内容', exact: true }).click(), 'late old request-body failure');
await choose('paused'); await expect(page.locator('.run-stop')).toContainText('完整任务要求超过请求上下文预算');
await expect(page.getByRole('button', { name: '继续任务', exact: true })).toBeVisible(); mark('context-limit pause explains the intact requirement and explicit continuation');
await page.setViewportSize({ width: 390, height: 844 });
await expect(page.getByLabel('当前项目', { exact: true })).toBeHidden();
await choose('paused-repeat'); await expect(page.locator('.run-stop')).toContainText('重复读取未推进任务');
await page.locator('.run-stop').scrollIntoViewIfNeeded();
assert.equal(await page.evaluate(() => window.innerWidth), 390);
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
await screenshot('mobile-read-pause');
await page.keyboard.press('Escape'); await expect(page.getByRole('button', { name: '继续任务', exact: true })).toBeVisible();
const pausedTaskId = (await api('/sessions/' + cases['paused-repeat'].session.id)).execution.task.id;
await page.getByRole('button', { name: '继续任务', exact: true }).click();
await until(async () => (await api('/sessions/' + cases['paused-repeat'].session.id)).execution.runs.length === 2);
const resumed = await api('/sessions/' + cases['paused-repeat'].session.id);
assert.equal(resumed.execution.task.id, pausedTaskId); assert.equal(resumed.execution.run.resumeOf, cases['paused-repeat'].runs[0]);
mark('390px repeated-read pause explains the cause and explicit continuation retains the same Task');
await page.setViewportSize({ width: 1440, height: 1000 }); await choose('working');
await expect(panel()).toContainText('当前任务依据'); await expect(panel()).toContainText('1 项代码与执行结果');
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
await expect(artifact.locator('pre')).toContainText('result_artifact_id'); await screenshot('desktop-working-evidence');
await page.keyboard.press('Escape'); mark('desktop working evidence shows actual code, original result provenance and the current Task');
await page.setViewportSize({ width: 390, height: 844 }); await choose('working');
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
const workingBounds = await artifact.boundingBox(); assert.ok(workingBounds.x >= 0 && workingBounds.x + workingBounds.width <= 390);
await screenshot('mobile-working-evidence'); await page.keyboard.press('Escape'); await expect(panel()).toBeVisible();
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click();
await expect(artifact.locator('pre')).toContainText('working_set'); await page.keyboard.press('Escape');
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
mark('390px working evidence and sent request stay readable and closing the artifact retains task navigation');
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), passed, pageErrors, requestFailures, screenshots, root, data,
mode: 'saved synthetic context evidence; no real model or project commands executed by this UI verifier', browser: browser.version() };
fs.writeFileSync('.qa/' + proofPrefix + '-ui-validation.json', JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify(result, null, 2));
} catch (error) {
await screenshot('failure').catch(() => {});
fs.writeFileSync('.qa/' + proofPrefix + '-ui-validation.json', JSON.stringify({ success: false, version: bootstrap.version, passed, pageErrors, requestFailures, error: error.message, screenshots }, null, 2) + '\n');
throw error;
} finally {
await browser.close();
const response = await fetch(url + '/api/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, body: '{}' }).catch(() => null);
if (response) assert.ok(response.ok);
await until(() => child.exitCode !== null || child.signalCode !== null).catch(() => { if (child.exitCode === null && child.signalCode === null) child.kill(); });
}