134 lines
11 KiB
JavaScript
134 lines
11 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { chromium, expect } from 'playwright/test';
|
|
import { fixture, compatibleFixture, launchWorkbench, call, completion, until, assertToolPairs } from '../tests/helpers.mjs';
|
|
|
|
if (process.argv.includes('--help')) {
|
|
console.log('node scripts/verify-delivery-reliability-ui.mjs OUTPUT [APP_ROOT] — declared local HTTP model fixture, actual review/continue/check UI and saved budget cases; no real model. OUTPUT must be new.');
|
|
process.exit(0);
|
|
}
|
|
assert.ok(process.argv[2], 'Choose a new output directory.');
|
|
const output = path.resolve(process.argv[2]);
|
|
assert.ok(!fs.existsSync(output), 'Keep prior evidence; choose a new directory.');
|
|
fs.mkdirSync(output, { recursive: true });
|
|
const appRoot = path.resolve(process.argv[3] ?? '.');
|
|
const packageIdentity = JSON.parse(fs.readFileSync(path.join(appRoot,
|
|
fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json')));
|
|
const cleanups = [], t = { after: fn => cleanups.push(fn) };
|
|
const f = fixture(t), cases = {};
|
|
fs.mkdirSync(path.join(f.projectRoot, 'test'));
|
|
fs.writeFileSync(path.join(f.projectRoot, 'package.json'), JSON.stringify({ type: 'module', scripts: { test: 'node --test test/*.test.mjs' } }));
|
|
fs.writeFileSync(path.join(f.projectRoot, 'test/values.test.mjs'), "import test from 'node:test';import assert from 'node:assert/strict';import {left} from '../src/left.mjs';import {right} from '../src/right.mjs';test('left',()=>assert.equal(left,1));test('right',()=>assert.equal(right,2));\n");
|
|
for (const [name, limitingReason] of [['remaining', 'remaining_token_budget'], ['window', 'context_limit'], ['legacy', null]]) {
|
|
const root = path.join(f.root, name); fs.mkdirSync(root); fs.writeFileSync(path.join(root, 'README.md'), '# Declared saved-budget UI fixture\n');
|
|
const project = f.store.addProject(root), session = f.store.createSession(project.id, '预算诊断 ' + name);
|
|
const goal = '保留原任务和已完成工具;按原拒绝分项核对预算。';
|
|
const run = f.store.beginRun(session, goal, false, f.store.config);
|
|
run.budgetPause = { schema: 1, reason: 'engine_input_budget', ...(limitingReason ? { limitingReason } : {}),
|
|
requiredInputTokens: name === 'window' ? 28000 : 27424, inputBudget: 27552, contextLimit: 32000,
|
|
outputReservation: 2048, safetyTokens: 2400, runRemainingTokens: 28339, taskRemainingTokens: 28339,
|
|
remainingTokens: 28339, tokenReservation: { schemaVersion: 1, reservedTokens: 29600, routes: [] } };
|
|
session.status = 'paused'; f.store.finishRun(session, run, 'engine_input_budget'); cases[name] = { project, session };
|
|
}
|
|
f.store.close();
|
|
let requests = 0, browser, page;
|
|
const checks = [], errors = [], screenshots = [];
|
|
const mark = name => { checks.push(name); console.log('PASS ' + name); };
|
|
const screenshot = async name => { const file = path.join(output, name + '.png'); await page.screenshot({ path: file, fullPage: true }); screenshots.push(file); };
|
|
try {
|
|
const model = await compatibleFixture(t, (_route, input) => {
|
|
if (!input?.messages) return { data: [{ id: 'declared-ui-fixture' }] };
|
|
requests++; assertToolPairs(input.messages);
|
|
const values = input.messages.filter(message => message.role === 'tool').map(message => JSON.parse(message.content));
|
|
if (requests === 1) return completion([
|
|
call('ui-left', 'propose_file', { path: 'src/left.mjs', content: 'export const left = 1;\n', reason: 'Declared fixture left' }),
|
|
call('ui-right', 'propose_file', { path: 'src/right.mjs', content: 'export const right = 2;\n', reason: 'Declared fixture right' }),
|
|
]);
|
|
if (requests === 2) return completion([
|
|
call('ui-seal', 'prepare_change_set', { change_ids: values.map(value => value.change_id).filter(Boolean) }),
|
|
call('ui-same-turn-inspect', 'inspect_session', {}),
|
|
]);
|
|
if (requests === 3) return completion([call('ui-check-plan', 'plan_checks', { names: ['test'] })]);
|
|
if (requests === 4) return completion([call('ui-check-run', 'run_checks', { plan_id: values.findLast(value => value.checks && value.fingerprint).id })]);
|
|
return completion([], '两份模块已通过实际 Node 测试。');
|
|
});
|
|
const host = await launchWorkbench(t, f.data, {}, { appRoot });
|
|
assert.equal(host.bootstrap.version, packageIdentity.version, 'Compiled host and package must agree.');
|
|
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'declared-ui-fixture', apiKey: 'SYNTHETIC-QA-ONLY', maxSteps: 6 });
|
|
browser = await chromium.launch({ channel: 'msedge', headless: true });
|
|
page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
|
page.on('pageerror', error => errors.push(error.message));
|
|
page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') errors.push(request.url() + ': ' + request.failure()?.errorText); });
|
|
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
|
async function choose(projectId) {
|
|
const selector = page.getByLabel('当前项目', { exact: true });
|
|
const expand = page.getByRole('button', { name: '展开项目栏', exact: true });
|
|
if ((await page.viewportSize()).width <= 620) await expect(expand).toBeVisible();
|
|
if (!await selector.isVisible()) await expand.click();
|
|
await expect(selector).toBeVisible();
|
|
await selector.selectOption(projectId);
|
|
if ((await page.viewportSize()).width <= 940 && await page.locator('.project-sidebar').isVisible()) await page.getByRole('button', { name: '收起项目栏', exact: true }).click();
|
|
}
|
|
await choose(f.project.id);
|
|
await page.getByLabel('任务内容', { exact: true }).fill('完成双模块工程,并在审查应用后继续实际测试。');
|
|
await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
|
const route = '/sessions/' + f.session.id;
|
|
let reviewed;
|
|
await until(async () => { reviewed = await host.api(route); return reviewed.status === 'needs_review'; }, 30000);
|
|
assert.equal(requests, 2); assert.equal(reviewed.execution.run.stopReason, 'change_set_review');
|
|
assert.equal(reviewed.events.find(event => event.toolCallId === 'ui-same-turn-inspect').phase, 'completed');
|
|
assert.equal(fs.existsSync(path.join(f.projectRoot, 'src')), false);
|
|
mark('Actual compiled HTTP model loop settles both tools and hands off after exactly two fixture requests without writing files');
|
|
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
|
await expect(page.getByLabel('任务记录', { exact: true })).toContainText('完整文件批次已封存,等待审查与应用');
|
|
await screenshot('desktop-review-handoff');
|
|
await page.reload(); await page.waitForLoadState('networkidle');
|
|
assert.equal((await host.api(route)).execution.task.id, reviewed.execution.task.id); assert.equal(requests, 2);
|
|
mark('Review handoff persists across refresh without another model request');
|
|
await page.setViewportSize({ width: 390, height: 844 }); await choose(f.project.id);
|
|
await page.getByRole('button', { name: '查看改动', exact: true }).click();
|
|
await page.getByRole('button', { name: '审查变更集', exact: true }).click();
|
|
const dialog = page.getByRole('dialog', { name: '多文件变更集审查', exact: true });
|
|
await expect(dialog).toContainText('src/left.mjs'); await expect(dialog).toContainText('src/right.mjs');
|
|
await screenshot('mobile-batch-review');
|
|
await dialog.getByRole('button', { name: '确认应用全部 2 项', exact: true }).click();
|
|
await until(async () => (await host.api(route)).execution.changeSets.at(-1).status === 'completed', 30000);
|
|
await page.keyboard.press('Escape'); await expect(page.getByRole('button', { name: '继续原任务', exact: true })).toBeVisible();
|
|
await page.getByRole('button', { name: '关闭检视区', exact: true }).click();
|
|
assert.equal(requests, 2); mark('390px reviewed whole-batch application waits for explicit continuation');
|
|
await page.getByRole('button', { name: '继续原任务', exact: true }).click();
|
|
await until(async () => (await host.api(route)).approvals.some(approval => approval.status === 'pending'), 30000);
|
|
await screenshot('mobile-check-approval');
|
|
await page.getByRole('button', { name: '确认执行', exact: true }).click();
|
|
let done;
|
|
await until(async () => { done = await host.api(route); return done.status === 'completed'; }, 30000);
|
|
assert.equal(done.execution.task.id, reviewed.execution.task.id); assert.equal(done.execution.checks[0].results[0].tests.passed, 2);
|
|
assert.equal(requests, 5);
|
|
assert.equal(done.events.filter(event => ['propose_file', 'prepare_change_set'].includes(event.tool)).length, 3);
|
|
mark('Explicit same-Task continuation runs two actual Node assertions and never replays proposals or sealing');
|
|
for (const width of [1440, 390]) {
|
|
await page.setViewportSize({ width, height: width === 390 ? 844 : 1000 });
|
|
for (const [name, title] of [['remaining', '原生引擎剩余预算不足'], ['window', '原生引擎上下文窗口不足'], ['legacy', '原生引擎输入或剩余预算不足']]) {
|
|
await choose(cases[name].project.id);
|
|
await expect(page.locator('.session-item.selected')).toContainText('预算诊断 ' + name);
|
|
const panel = page.getByLabel('任务预算诊断', { exact: true });
|
|
await expect(panel.getByRole('heading', { name: title, exact: true })).toBeVisible();
|
|
if (name === 'remaining') { await expect(panel).toContainText('输入可容纳于当前上下文窗口'); await expect(panel).toContainText('29,600'); await expect(panel).toContainText('28,339'); }
|
|
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
|
|
await screenshot(width + '-budget-' + name);
|
|
mark(width + 'px ' + name + ' diagnosis renders actual limit, numbers and historical fallback without overflow');
|
|
}
|
|
}
|
|
assert.deepEqual(errors, []);
|
|
const result = { success: true, version: (await host.api('/bootstrap')).version, appRoot, checks, errors, screenshots,
|
|
fixtureRequests: requests, realModelRequests: 0, taskId: done.execution.task.id, runs: done.execution.runs.map(run => ({ id: run.id, status: run.status, stopReason: run.stopReason })),
|
|
scope: 'Declared local HTTP model fixture with actual engineering tools/checks, persisted review state and seeded diagnostic cases. No real model or user project.', browser: browser.version() };
|
|
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
|
|
console.log(JSON.stringify({ success: true, checks: checks.length, output }));
|
|
} catch (error) {
|
|
if (page) await screenshot('failure').catch(() => {});
|
|
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify({ success: false, checks, errors, screenshots, error: error.stack, requests }, null, 2) + '\n', { flag: 'wx' });
|
|
throw error;
|
|
} finally { if (browser) await browser.close(); for (const cleanup of cleanups.reverse()) await cleanup(); }
|