Files
Agent-Workbench/scripts/verify-delivery-reliability-ui.mjs

134 lines
11 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { chromium, expect } from 'playwright/test';
import { fixture, compatibleFixture, launchWorkbench, call, completion, until, assertToolPairs } from '../tests/helpers.mjs';
if (process.argv.includes('--help')) {
console.log('node scripts/verify-delivery-reliability-ui.mjs OUTPUT [APP_ROOT] — declared local HTTP model fixture, actual review/continue/check UI and saved budget cases; no real model. OUTPUT must be new.');
process.exit(0);
}
assert.ok(process.argv[2], 'Choose a new output directory.');
const output = path.resolve(process.argv[2]);
assert.ok(!fs.existsSync(output), 'Keep prior evidence; choose a new directory.');
fs.mkdirSync(output, { recursive: true });
const appRoot = path.resolve(process.argv[3] ?? '.');
const packageIdentity = JSON.parse(fs.readFileSync(path.join(appRoot,
fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json')));
const cleanups = [], t = { after: fn => cleanups.push(fn) };
const f = fixture(t), cases = {};
fs.mkdirSync(path.join(f.projectRoot, 'test'));
fs.writeFileSync(path.join(f.projectRoot, 'package.json'), JSON.stringify({ type: 'module', scripts: { test: 'node --test test/*.test.mjs' } }));
fs.writeFileSync(path.join(f.projectRoot, 'test/values.test.mjs'), "import test from 'node:test';import assert from 'node:assert/strict';import {left} from '../src/left.mjs';import {right} from '../src/right.mjs';test('left',()=>assert.equal(left,1));test('right',()=>assert.equal(right,2));\n");
for (const [name, limitingReason] of [['remaining', 'remaining_token_budget'], ['window', 'context_limit'], ['legacy', null]]) {
const root = path.join(f.root, name); fs.mkdirSync(root); fs.writeFileSync(path.join(root, 'README.md'), '# Declared saved-budget UI fixture\n');
const project = f.store.addProject(root), session = f.store.createSession(project.id, '预算诊断 ' + name);
const goal = '保留原任务和已完成工具;按原拒绝分项核对预算。';
const run = f.store.beginRun(session, goal, false, f.store.config);
run.budgetPause = { schema: 1, reason: 'engine_input_budget', ...(limitingReason ? { limitingReason } : {}),
requiredInputTokens: name === 'window' ? 28000 : 27424, inputBudget: 27552, contextLimit: 32000,
outputReservation: 2048, safetyTokens: 2400, runRemainingTokens: 28339, taskRemainingTokens: 28339,
remainingTokens: 28339, tokenReservation: { schemaVersion: 1, reservedTokens: 29600, routes: [] } };
session.status = 'paused'; f.store.finishRun(session, run, 'engine_input_budget'); cases[name] = { project, session };
}
f.store.close();
let requests = 0, browser, page;
const checks = [], errors = [], screenshots = [];
const mark = name => { checks.push(name); console.log('PASS ' + name); };
const screenshot = async name => { const file = path.join(output, name + '.png'); await page.screenshot({ path: file, fullPage: true }); screenshots.push(file); };
try {
const model = await compatibleFixture(t, (_route, input) => {
if (!input?.messages) return { data: [{ id: 'declared-ui-fixture' }] };
requests++; assertToolPairs(input.messages);
const values = input.messages.filter(message => message.role === 'tool').map(message => JSON.parse(message.content));
if (requests === 1) return completion([
call('ui-left', 'propose_file', { path: 'src/left.mjs', content: 'export const left = 1;\n', reason: 'Declared fixture left' }),
call('ui-right', 'propose_file', { path: 'src/right.mjs', content: 'export const right = 2;\n', reason: 'Declared fixture right' }),
]);
if (requests === 2) return completion([
call('ui-seal', 'prepare_change_set', { change_ids: values.map(value => value.change_id).filter(Boolean) }),
call('ui-same-turn-inspect', 'inspect_session', {}),
]);
if (requests === 3) return completion([call('ui-check-plan', 'plan_checks', { names: ['test'] })]);
if (requests === 4) return completion([call('ui-check-run', 'run_checks', { plan_id: values.findLast(value => value.checks && value.fingerprint).id })]);
return completion([], '两份模块已通过实际 Node 测试。');
});
const host = await launchWorkbench(t, f.data, {}, { appRoot });
assert.equal(host.bootstrap.version, packageIdentity.version, 'Compiled host and package must agree.');
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'declared-ui-fixture', apiKey: 'SYNTHETIC-QA-ONLY', maxSteps: 6 });
browser = await chromium.launch({ channel: 'msedge', headless: true });
page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
page.on('pageerror', error => errors.push(error.message));
page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') errors.push(request.url() + ': ' + request.failure()?.errorText); });
await page.goto(host.url); await page.waitForLoadState('networkidle');
async function choose(projectId) {
const selector = page.getByLabel('当前项目', { exact: true });
const expand = page.getByRole('button', { name: '展开项目栏', exact: true });
if ((await page.viewportSize()).width <= 620) await expect(expand).toBeVisible();
if (!await selector.isVisible()) await expand.click();
await expect(selector).toBeVisible();
await selector.selectOption(projectId);
if ((await page.viewportSize()).width <= 940 && await page.locator('.project-sidebar').isVisible()) await page.getByRole('button', { name: '收起项目栏', exact: true }).click();
}
await choose(f.project.id);
await page.getByLabel('任务内容', { exact: true }).fill('完成双模块工程,并在审查应用后继续实际测试。');
await page.getByRole('button', { name: '发送任务', exact: true }).click();
const route = '/sessions/' + f.session.id;
let reviewed;
await until(async () => { reviewed = await host.api(route); return reviewed.status === 'needs_review'; }, 30000);
assert.equal(requests, 2); assert.equal(reviewed.execution.run.stopReason, 'change_set_review');
assert.equal(reviewed.events.find(event => event.toolCallId === 'ui-same-turn-inspect').phase, 'completed');
assert.equal(fs.existsSync(path.join(f.projectRoot, 'src')), false);
mark('Actual compiled HTTP model loop settles both tools and hands off after exactly two fixture requests without writing files');
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
await expect(page.getByLabel('任务记录', { exact: true })).toContainText('完整文件批次已封存,等待审查与应用');
await screenshot('desktop-review-handoff');
await page.reload(); await page.waitForLoadState('networkidle');
assert.equal((await host.api(route)).execution.task.id, reviewed.execution.task.id); assert.equal(requests, 2);
mark('Review handoff persists across refresh without another model request');
await page.setViewportSize({ width: 390, height: 844 }); await choose(f.project.id);
await page.getByRole('button', { name: '查看改动', exact: true }).click();
await page.getByRole('button', { name: '审查变更集', exact: true }).click();
const dialog = page.getByRole('dialog', { name: '多文件变更集审查', exact: true });
await expect(dialog).toContainText('src/left.mjs'); await expect(dialog).toContainText('src/right.mjs');
await screenshot('mobile-batch-review');
await dialog.getByRole('button', { name: '确认应用全部 2 项', exact: true }).click();
await until(async () => (await host.api(route)).execution.changeSets.at(-1).status === 'completed', 30000);
await page.keyboard.press('Escape'); await expect(page.getByRole('button', { name: '继续原任务', exact: true })).toBeVisible();
await page.getByRole('button', { name: '关闭检视区', exact: true }).click();
assert.equal(requests, 2); mark('390px reviewed whole-batch application waits for explicit continuation');
await page.getByRole('button', { name: '继续原任务', exact: true }).click();
await until(async () => (await host.api(route)).approvals.some(approval => approval.status === 'pending'), 30000);
await screenshot('mobile-check-approval');
await page.getByRole('button', { name: '确认执行', exact: true }).click();
let done;
await until(async () => { done = await host.api(route); return done.status === 'completed'; }, 30000);
assert.equal(done.execution.task.id, reviewed.execution.task.id); assert.equal(done.execution.checks[0].results[0].tests.passed, 2);
assert.equal(requests, 5);
assert.equal(done.events.filter(event => ['propose_file', 'prepare_change_set'].includes(event.tool)).length, 3);
mark('Explicit same-Task continuation runs two actual Node assertions and never replays proposals or sealing');
for (const width of [1440, 390]) {
await page.setViewportSize({ width, height: width === 390 ? 844 : 1000 });
for (const [name, title] of [['remaining', '原生引擎剩余预算不足'], ['window', '原生引擎上下文窗口不足'], ['legacy', '原生引擎输入或剩余预算不足']]) {
await choose(cases[name].project.id);
await expect(page.locator('.session-item.selected')).toContainText('预算诊断 ' + name);
const panel = page.getByLabel('任务预算诊断', { exact: true });
await expect(panel.getByRole('heading', { name: title, exact: true })).toBeVisible();
if (name === 'remaining') { await expect(panel).toContainText('输入可容纳于当前上下文窗口'); await expect(panel).toContainText('29,600'); await expect(panel).toContainText('28,339'); }
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
await screenshot(width + '-budget-' + name);
mark(width + 'px ' + name + ' diagnosis renders actual limit, numbers and historical fallback without overflow');
}
}
assert.deepEqual(errors, []);
const result = { success: true, version: (await host.api('/bootstrap')).version, appRoot, checks, errors, screenshots,
fixtureRequests: requests, realModelRequests: 0, taskId: done.execution.task.id, runs: done.execution.runs.map(run => ({ id: run.id, status: run.status, stopReason: run.stopReason })),
scope: 'Declared local HTTP model fixture with actual engineering tools/checks, persisted review state and seeded diagnostic cases. No real model or user project.', browser: browser.version() };
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify({ success: true, checks: checks.length, output }));
} catch (error) {
if (page) await screenshot('failure').catch(() => {});
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify({ success: false, checks, errors, screenshots, error: error.stack, requests }, null, 2) + '\n', { flag: 'wx' });
throw error;
} finally { if (browser) await browser.close(); for (const cleanup of cleanups.reverse()) await cleanup(); }