197 lines
18 KiB
JavaScript
197 lines
18 KiB
JavaScript
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import assert from 'node:assert/strict';
|
||
import { randomUUID, createHash } from 'node:crypto';
|
||
import { spawn } from 'node:child_process';
|
||
import { chromium, expect } from 'playwright/test';
|
||
import { Store } from '../src/server/store.mjs';
|
||
import { RequestPlanner } from '../src/core/context/request-plan.mjs';
|
||
import { toolDefinitions } from '../src/server/provider.mjs';
|
||
import { call } from '../tests/helpers.mjs';
|
||
|
||
const root = path.resolve('.qa', 'context-ui-' + randomUUID()), data = path.join(root, 'data');
|
||
fs.mkdirSync(root, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||
const store = new Store(data), cases = {};
|
||
for (const name of ['long', 'other', 'paused', 'paused-repeat', 'working']) {
|
||
const projectRoot = path.join(root, 'context-' + name); fs.mkdirSync(projectRoot);
|
||
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned context UI fixture\n');
|
||
const project = store.addProject(projectRoot), session = store.createSession(project.id, '上下文验收 ' + name);
|
||
const goal = '这是独立界面验收样本。完整任务要求与实际原记录保留,读取工件不得重新执行工具。';
|
||
store.message(session, 'user', goal); session.providerMessages.push({ role: 'user', content: goal });
|
||
const run = store.beginRun(session, goal, false, store.config), planner = new RequestPlanner(store);
|
||
if (name === 'paused' || name === 'paused-repeat') {
|
||
session.status = 'paused'; store.message(session, 'system', name === 'paused' ? '完整任务要求超过请求上下文预算,请核对配置后继续。' : '模型重复读取已覆盖工件区间,原任务已暂停,请核对后继续。');
|
||
if (name === 'paused-repeat') store.core.task.get(run.taskId).readProgress = { schemaVersion: 1, reads: [], repeated: 3, changingIndexes: [], warning: '重复读取已覆盖区间', pause: true };
|
||
store.finishRun(session, run, name === 'paused' ? 'context_limit' : 'no_progress'); cases[name] = { project, session, projectRoot, runs: [run.id] }; continue;
|
||
}
|
||
if (name === 'long') for (let index = 0; index < 18; index++) {
|
||
const tool = call('ui-original-' + index, 'read_file', { path: 'README.md', fixture: 'Preserve exact historical JSON.' });
|
||
const event = { id: randomUUID(), toolCallId: tool.id, tool: 'read_file', kind: 'tool', phase: 'queued', label: '读取文件',
|
||
createdAt: new Date().toISOString(), startedAt: new Date().toISOString(), input: { path: 'README.md' }, runId: run.id, taskId: run.taskId };
|
||
session.events.push(event); store.registerInvocation(session, tool, run, event);
|
||
store.beginInvocation(session, tool, run, event); store.invocationState(session, event, 'running');
|
||
const result = { status: 'completed', path: 'README.md', content: ('完整原记录🙂 ' + index + '\n').repeat(1200) };
|
||
event.phase = 'completed'; event.output = '已保存完整原记录。'; event.finishedAt = new Date().toISOString();
|
||
store.finishInvocation(session, event, result);
|
||
session.providerMessages.push({ role: 'assistant', content: 'Saved fixture read.', tool_calls: [tool] },
|
||
{ role: 'tool', tool_call_id: tool.id, content: JSON.stringify(result) });
|
||
}
|
||
if (name === 'working') {
|
||
const content = 'export const actual = "当前任务真实代码🙂";\n';
|
||
fs.writeFileSync(path.join(projectRoot, 'actual.mjs'), content);
|
||
const values = [{ tool: 'read_file', args: { path: 'actual.mjs' }, result: { path: 'actual.mjs', content,
|
||
sha256: createHash('sha256').update(content).digest('hex'), start_line: 1, end_line: 2, total_lines: 2, truncated: false } },
|
||
{ tool: 'query_code', args: { query: 'large' }, result: { results: 'Large owned mixed group '.repeat(16000) } }];
|
||
const calls = values.map(value => call(randomUUID(), value.tool, value.args)), messages = [];
|
||
for (const [index, invocation] of calls.entries()) {
|
||
const event = store.event(session, { kind: 'tool', tool: invocation.function.name, toolCallId: invocation.id,
|
||
runId: run.id, taskId: run.taskId, phase: 'queued', startedAt: new Date().toISOString(), label: '工作依据样本' });
|
||
store.registerInvocation(session, invocation, run, event); store.beginInvocation(session, invocation, run, event);
|
||
store.finishInvocation(session, event, values[index].result);
|
||
messages.push({ role: 'tool', tool_call_id: invocation.id, content: JSON.stringify(values[index].result) });
|
||
}
|
||
session.providerMessages.push({ role: 'assistant', content: '', tool_calls: calls }, ...messages); store.save(session);
|
||
}
|
||
const step = store.beginStep(session, run), first = planner.prepare(session, run, store.config, {
|
||
system: goal + '\n' + JSON.stringify({ project_id: project.id, session_id: session.id, task_id: run.taskId }), tools: toolDefinitions, stepId: step.id });
|
||
const response = { message: { role: 'assistant', content: '独立界面样本已保存;没有请求真实模型。' }, usage: { prompt_tokens: 2400, completion_tokens: 300 } };
|
||
planner.finish(session, first.record, { response, attempts: 1 }); session.providerMessages.push(response.message);
|
||
store.message(session, 'assistant', response.message.content); store.finishStep(session, step); session.status = 'completed'; store.finishRun(session, run);
|
||
const runs = [run.id], plans = [first.record.id];
|
||
if (name === 'long') {
|
||
const resumed = store.beginRun(session, '', true, store.config), nextStep = store.beginStep(session, resumed);
|
||
const next = planner.prepare(session, resumed, store.config, { system: goal, tools: toolDefinitions, stepId: nextStep.id });
|
||
planner.finish(session, next.record, { response, attempts: 1 }); store.finishStep(session, nextStep);
|
||
session.status = 'completed'; store.finishRun(session, resumed); runs.push(resumed.id); plans.push(next.record.id);
|
||
}
|
||
cases[name] = { project, session, projectRoot, runs, plans };
|
||
}
|
||
store.close();
|
||
const env = { ...process.env, AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' };
|
||
delete env.AGENT_WORKBENCH_API_KEY;
|
||
const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||
let output = '', errors = '', url = '';
|
||
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const item = JSON.parse(line); if (item.ready) url = item.url; } catch {} } });
|
||
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
||
const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
|
||
await until(() => url || child.exitCode !== null); assert.ok(url, errors);
|
||
const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
|
||
assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||
const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
|
||
const proofPrefix = bootstrap.version === '1.6.0' ? 'workspaces-context' : releaseEvidencePrefix(bootstrap.version);
|
||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||
const passed = [], pageErrors = [], requestFailures = [], screenshots = [];
|
||
page.on('pageerror', error => pageErrors.push(error.message));
|
||
page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') requestFailures.push(request.failure()?.errorText); });
|
||
const mark = value => { passed.push(value); console.log('PASS ' + value); };
|
||
const panel = () => page.getByLabel('请求上下文', { exact: true }), history = () => page.getByRole('dialog', { name: '完整历史记录', exact: true });
|
||
async function screenshot(name) {
|
||
const file = '.qa/screenshots/' + proofPrefix + '-' + name + '.png'; await page.screenshot({ path: file, fullPage: true }); screenshots.push(file);
|
||
}
|
||
async function choose(name) {
|
||
const select = page.getByLabel('当前项目', { exact: true });
|
||
if (await select.inputValue() !== cases[name].project.id) {
|
||
if (!await select.isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||
await expect(select).toBeVisible(); await select.selectOption(cases[name].project.id);
|
||
}
|
||
await expect(page.locator('.session-item.selected')).toContainText('上下文验收 ' + name);
|
||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||
await expect(page.getByLabel('任务记录', { exact: true })).toBeVisible();
|
||
}
|
||
async function lateFailure(pattern, trigger, label) {
|
||
let release, entered = false, finished = false;
|
||
const gate = new Promise(resolve => { release = resolve; });
|
||
await page.route(pattern, async route => { entered = true; await gate; await route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: label }) }); finished = true; });
|
||
await trigger(); await until(() => entered);
|
||
if (await history().isVisible()) await page.keyboard.press('Escape');
|
||
await choose('other'); release(); await until(() => finished);
|
||
await page.waitForTimeout(350); await expect(page.locator('.toast')).toHaveCount(0);
|
||
await expect(panel()).toContainText('2,400 tokens');
|
||
await page.unroute(pattern); mark(label + ' stays in its original scope after switching projects');
|
||
}
|
||
try {
|
||
await page.goto(url); await page.waitForLoadState('networkidle'); await choose('long');
|
||
await expect(panel()).toContainText('2,400 tokens'); await expect(panel()).toContainText('16,000 tokens');
|
||
await expect(panel().getByRole('button', { name: '检索完整历史', exact: true })).toBeVisible();
|
||
await panel().scrollIntoViewIfNeeded(); await screenshot('desktop-plan'); mark('desktop request estimate, actual usage, complete goal and history identity are visible');
|
||
await panel().getByRole('button', { name: '检索完整历史', exact: true }).click();
|
||
await expect(history().locator('.context-history-group')).toHaveCount(12);
|
||
await history().getByRole('button', { name: '下一页', exact: true }).click();
|
||
await expect(history().locator('.artifact-pagination')).toContainText('13–18 / 18');
|
||
await history().getByRole('button', { name: '上一页', exact: true }).click();
|
||
await expect(history().locator('.artifact-pagination')).toContainText('1–12 / 18'); await screenshot('desktop-history');
|
||
await history().getByRole('button', { name: '读取工具结果', exact: true }).first().click();
|
||
const artifact = page.getByRole('dialog', { name: '执行结果工件', exact: true });
|
||
await expect(artifact.locator('pre')).toContainText('完整原记录');
|
||
await page.keyboard.press('Escape'); mark('paged complete history opens actual tool result artifacts without executing tools');
|
||
await page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[0]);
|
||
await expect(panel()).toContainText('2,400 tokens');
|
||
const selected = await api('/sessions/' + cases.long.session.id + '/context-plans?run=' + cases.long.runs[0]);
|
||
await expect(panel().locator('.context-sha')).toHaveAttribute('title', selected[0].requestSha256);
|
||
await page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[1]); mark('historical and resumed Runs retain their own request records');
|
||
await page.setViewportSize({ width: 390, height: 844 }); await choose('long');
|
||
await panel().getByRole('button', { name: '检索完整历史', exact: true }).click();
|
||
await expect(history().locator('.context-history-group')).toHaveCount(12); await screenshot('mobile-history');
|
||
const bounds = await history().boundingBox(); assert.ok(bounds.x >= 0 && bounds.x + bounds.width <= 390 && bounds.y + bounds.height <= 844);
|
||
await history().getByRole('button', { name: '读取原记录', exact: true }).first().click(); await expect(artifact.locator('pre')).toContainText('Preserve exact historical JSON');
|
||
await screenshot('mobile-original'); await page.keyboard.press('Escape');
|
||
await expect(panel()).toBeVisible();
|
||
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click(); await expect(artifact.locator('pre')).toContainText('history_index');
|
||
await screenshot('mobile-request-body'); await page.keyboard.press('Escape');
|
||
await expect(panel()).toBeVisible();
|
||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth) <= 390); mark('390px history, original JSON and sent-body dialogs fit and retain close controls');
|
||
await page.setViewportSize({ width: 1440, height: 1000 }); await choose('long');
|
||
const currentPlanId = cases.long.plans[1];
|
||
await lateFailure('**/context-plans/' + currentPlanId + '/history?*', () => panel().getByRole('button', { name: '检索完整历史', exact: true }).click(), 'late old history failure');
|
||
await choose('long');
|
||
await lateFailure('**/context-plans?run=' + cases.long.runs[0], () => page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[0]), 'late old Run failure');
|
||
await choose('long');
|
||
const currentPlan = await api('/sessions/' + cases.long.session.id + '/context-plans/' + currentPlanId);
|
||
await lateFailure('**/artifacts/' + currentPlan.requestArtifactId + '?*', () => panel().getByRole('button', { name: '查看发送内容', exact: true }).click(), 'late old request-body failure');
|
||
await choose('paused'); await expect(page.locator('.run-stop')).toContainText('完整任务要求超过请求上下文预算');
|
||
await expect(page.getByRole('button', { name: '继续任务', exact: true })).toBeVisible(); mark('context-limit pause explains the intact requirement and explicit continuation');
|
||
await page.setViewportSize({ width: 390, height: 844 });
|
||
await expect(page.getByLabel('当前项目', { exact: true })).toBeHidden();
|
||
await choose('paused-repeat'); await expect(page.locator('.run-stop')).toContainText('重复读取未推进任务');
|
||
await page.locator('.run-stop').scrollIntoViewIfNeeded();
|
||
assert.equal(await page.evaluate(() => window.innerWidth), 390);
|
||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
|
||
await screenshot('mobile-read-pause');
|
||
await page.keyboard.press('Escape'); await expect(page.getByRole('button', { name: '继续任务', exact: true })).toBeVisible();
|
||
const pausedTaskId = (await api('/sessions/' + cases['paused-repeat'].session.id)).execution.task.id;
|
||
await page.getByRole('button', { name: '继续任务', exact: true }).click();
|
||
await until(async () => (await api('/sessions/' + cases['paused-repeat'].session.id)).execution.runs.length === 2);
|
||
const resumed = await api('/sessions/' + cases['paused-repeat'].session.id);
|
||
assert.equal(resumed.execution.task.id, pausedTaskId); assert.equal(resumed.execution.run.resumeOf, cases['paused-repeat'].runs[0]);
|
||
mark('390px repeated-read pause explains the cause and explicit continuation retains the same Task');
|
||
await page.setViewportSize({ width: 1440, height: 1000 }); await choose('working');
|
||
await expect(panel()).toContainText('当前任务依据'); await expect(panel()).toContainText('1 项代码与执行结果');
|
||
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
|
||
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
|
||
await expect(artifact.locator('pre')).toContainText('result_artifact_id'); await screenshot('desktop-working-evidence');
|
||
await page.keyboard.press('Escape'); mark('desktop working evidence shows actual code, original result provenance and the current Task');
|
||
await page.setViewportSize({ width: 390, height: 844 }); await choose('working');
|
||
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
|
||
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
|
||
const workingBounds = await artifact.boundingBox(); assert.ok(workingBounds.x >= 0 && workingBounds.x + workingBounds.width <= 390);
|
||
await screenshot('mobile-working-evidence'); await page.keyboard.press('Escape'); await expect(panel()).toBeVisible();
|
||
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click();
|
||
await expect(artifact.locator('pre')).toContainText('working_set'); await page.keyboard.press('Escape');
|
||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
|
||
mark('390px working evidence and sent request stay readable and closing the artifact retains task navigation');
|
||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), passed, pageErrors, requestFailures, screenshots, root, data,
|
||
mode: 'saved synthetic context evidence; no real model or project commands executed by this UI verifier', browser: browser.version() };
|
||
fs.writeFileSync('.qa/' + proofPrefix + '-ui-validation.json', JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify(result, null, 2));
|
||
} catch (error) {
|
||
await screenshot('failure').catch(() => {});
|
||
fs.writeFileSync('.qa/' + proofPrefix + '-ui-validation.json', JSON.stringify({ success: false, version: bootstrap.version, passed, pageErrors, requestFailures, error: error.message, screenshots }, null, 2) + '\n');
|
||
throw error;
|
||
} finally {
|
||
await browser.close();
|
||
const response = await fetch(url + '/api/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, body: '{}' }).catch(() => null);
|
||
if (response) assert.ok(response.ok);
|
||
await until(() => child.exitCode !== null || child.signalCode !== null).catch(() => { if (child.exitCode === null && child.signalCode === null) child.kill(); });
|
||
}
|