feat: integrate approved MCP tools and recovery for 1.12
This commit is contained in:
@@ -3,7 +3,7 @@ const prefixes = Object.freeze({
|
||||
'1.4.0': 'm2', '1.5.0': 'context', '1.5.1': 'context-fix', '1.6.0': 'workspaces',
|
||||
'1.6.1': 'working-context', '1.6.2': 'working-retention', '1.7.0': 'python-engine',
|
||||
'1.8.0': 'rea-engine', '1.9.0': 'external-engines', '1.10.0': 'browser-engine',
|
||||
'1.10.1': 'browser-hotfix', '1.11.0': 'task-plans',
|
||||
'1.10.1': 'browser-hotfix', '1.11.0': 'task-plans', '1.12.0': 'model-center',
|
||||
});
|
||||
export function releaseEvidencePrefix(version) {
|
||||
const prefix = prefixes[version];
|
||||
|
||||
@@ -64,7 +64,7 @@ export async function upgradeRunning({ url, pid, currentRoot, newRoot, handlerRo
|
||||
if (!fs.existsSync(originalScript) || !fs.existsSync(upgradeScript) || !fs.existsSync(path.join(next, 'dist-client', 'index.html'))) throw new Error('Upgrade bundles are incomplete.');
|
||||
if (!Number.isInteger(pid) || pid < 1 || pid === process.pid) throw new Error('Invalid target process.');
|
||||
const before = await readUpgradeJson(origin + '/api/bootstrap', 5000, 'Read original Workbench');
|
||||
if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.');
|
||||
if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.');
|
||||
if (before.sessions.some(session => ['running', 'awaiting_approval', 'stopping'].includes(session.status))) throw new Error('Wait for existing tasks and approvals before upgrading.');
|
||||
let protocol, socket, enabledHere = false, paused = false;
|
||||
const breakpoints = [];
|
||||
|
||||
+29
-14
@@ -2,25 +2,26 @@ import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
||||
assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version), 'Review evidence paths when introducing the next release.');
|
||||
assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(version), 'Review evidence paths when introducing the next release.');
|
||||
const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = [];
|
||||
const cases = [
|
||||
...(version === '1.11.0' ? [['verify-task-plans-ui.mjs', 'docs/task-plans-task-ui-validation.json']] : []),
|
||||
...(['1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []),
|
||||
...(['1.11.0', '1.12.0'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []),
|
||||
...(['1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []),
|
||||
['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'],
|
||||
['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'],
|
||||
['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`],
|
||||
['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`],
|
||||
['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`],
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []),
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []),
|
||||
...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []),
|
||||
...(['1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []),
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []),
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []),
|
||||
...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []),
|
||||
...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []),
|
||||
...(version === '1.12.0' ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []),
|
||||
];
|
||||
fs.mkdirSync('.qa', { recursive: true });
|
||||
const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']);
|
||||
@@ -45,23 +46,37 @@ async function startHost(script) {
|
||||
} finally { if (child.exitCode === null && child.signalCode === null) child.kill(); }
|
||||
} };
|
||||
}
|
||||
for (const [script, proof] of cases) {
|
||||
for (const [script, fixedProof] of cases) {
|
||||
console.log('UI regression: ' + script); const started = Date.now(), log = `.qa/${prefix}-${script.replace('.mjs', '')}.log`;
|
||||
const stream = fs.createWriteStream(log);
|
||||
let host;
|
||||
try {
|
||||
if (externalHost.has(script)) host = await startHost(script);
|
||||
await new Promise((resolve, reject) => {
|
||||
const output = await new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, [path.join('scripts', script)], { cwd: process.cwd(), env: { ...gitEnvironment(), ...(host ? { AGENT_WORKBENCH_URL: host.url } : {}) }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let tail = ''; const capture = bytes => { stream.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-10000); };
|
||||
child.stdout.on('data', capture); child.stderr.on('data', capture);
|
||||
const timer = setTimeout(() => child.kill(), 240000);
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
if (process.platform === 'win32' && child.pid) {
|
||||
try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000, stdio: 'ignore' }); } catch { child.kill(); }
|
||||
} else child.kill();
|
||||
}, 240000);
|
||||
child.once('error', error => { clearTimeout(timer); reject(error); });
|
||||
child.once('close', code => { clearTimeout(timer); code === 0 ? resolve() : reject(new Error(script + ' failed (' + code + '): ' + tail)); });
|
||||
child.once('close', code => { clearTimeout(timer); code === 0 && !timedOut ? resolve(tail) : reject(new Error(script + (timedOut ? ' timed out' : ' failed (' + code + ')') + ': ' + tail)); });
|
||||
});
|
||||
stream.end(); const evidence = JSON.parse(fs.readFileSync(proof, 'utf8'));
|
||||
stream.end();
|
||||
const directory = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1].trim();
|
||||
const proof = fixedProof || (directory && path.join(directory, 'result.json'));
|
||||
assert.ok(proof, 'Verifier must identify its current evidence.');
|
||||
if (!fixedProof) {
|
||||
const relative = path.relative(path.resolve('.qa'), path.resolve(proof));
|
||||
assert.ok(relative && !relative.startsWith('..') && !path.isAbsolute(relative), 'Evidence must belong to this workspace QA directory.');
|
||||
}
|
||||
const evidence = JSON.parse(fs.readFileSync(proof, 'utf8'));
|
||||
assert.notEqual(evidence.success, false); if (evidence.version) assert.equal(evidence.version, version);
|
||||
const passed = evidence.passed ?? evidence.cases;
|
||||
const passed = evidence.passed ?? evidence.cases ?? evidence.assertions;
|
||||
assert.ok(Array.isArray(passed) && passed.length > 0);
|
||||
const errors = evidence.pageErrors ?? [], failures = evidence.requestFailures ?? [];
|
||||
assert.equal(errors.length, 0); assert.equal(failures.length, 0);
|
||||
|
||||
@@ -34,7 +34,7 @@ child.stdout.on('data', value => { output += value; for (const line of output.sp
|
||||
child.stderr.on('data', value => { errors += value; });
|
||||
const wait = async predicate => { const deadline = Date.now() + 15000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('UI fixture timed out: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
|
||||
await wait(() => url); const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.ok(['1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
assert.ok(['1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const proofPrefix = releaseEvidencePrefix(bootstrap.version) + '-batch';
|
||||
const screenshotPrefix = bootstrap.version === '1.4.0' ? '1.4-batch' : proofPrefix;
|
||||
const api = async (route, input) => { const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
||||
|
||||
@@ -77,7 +77,7 @@ child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
||||
const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
|
||||
await until(() => url || child.exitCode !== null); assert.ok(url, errors);
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
|
||||
assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
|
||||
const proofPrefix = bootstrap.version === '1.6.0' ? 'workspaces-context' : releaseEvidencePrefix(bootstrap.version);
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, compatibleFixture, completion, call } from '../tests/helpers.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
import { Runtime } from '../src/server/runtime.mjs';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
|
||||
const directory = path.resolve('.qa', 'mcp-recovery-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t);
|
||||
const runtime = new Runtime(f.store, new Provider()), registry = runtime.mcpRegistry();
|
||||
const entry = path.join(f.root, 'owned-recovery-mcp.mjs'); fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
|
||||
const saved = registry.register(f.project.id, { name: '中断恢复工具', executable: process.execPath, entry, timeoutMs: 45000 });
|
||||
await registry.start(f.session, { id: saved.id, configurationRevision: 1, confirm: true }).promise;
|
||||
const value = registry.find(f.project.id, saved.id);
|
||||
const enabled = registry.configure(f.project.id, { id: saved.id, configurationRevision: value.configurationRevision, enabled: true, allowedTools: ['hold'] });
|
||||
const model = await compatibleFixture(t, () => completion([call('ui-recovery-crash', 'call_registered_mcp', {
|
||||
registration_id: enabled.id, configuration_revision: enabled.configurationRevision, contract_sha256: enabled.contract.sha256,
|
||||
tool: 'hold', arguments: { text: 'owned-ui-crash-effect' },
|
||||
})]));
|
||||
f.store.saveConfig({ mode: 'compatible', model: 'owned-recovery-ui', baseUrl: model.baseUrl, contextLimit: 24000 }); f.store.close();
|
||||
const crash = await promisify(execFile)(process.execPath, [path.resolve('tests/fixtures/mcp-host-crash.mjs'), f.data, f.session.id],
|
||||
{ env: gitEnvironment(), windowsHide: true, timeout: 18000 })
|
||||
.then(() => assert.fail('Expected owned host exit.'), error => { assert.equal(error.code, 85, error.stderr); return JSON.parse(error.stdout.trim().split('\n').at(-1)); });
|
||||
const host = await launchWorkbench(t, f.data), browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const pageErrors = [], requestFailures = [], assertions = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const mark = value => { assertions.push(value); console.log('PASS ' + value); };
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
||||
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }), history = dialog.getByLabel('MCP运行记录', { exact: true });
|
||||
const row = history.locator('.mcp-history-row.state-unknown'); await expect(row).toHaveCount(1);
|
||||
await row.getByRole('button', { name: '查看原始输入', exact: true }).click();
|
||||
const evidence = history.getByLabel('MCP原报告'); await expect(evidence).toContainText('没有完整原报告');
|
||||
await evidence.locator('summary').click(); await expect(evidence).toContainText('owned-ui-crash-effect');
|
||||
await evidence.scrollIntoViewIfNeeded(); await page.screenshot({ path: path.join(directory, 'desktop-mcp-interrupted-input.png') });
|
||||
mark('actual host crash with no result report exposes its original verified input and unknown status');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await evidence.scrollIntoViewIfNeeded();
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
await page.screenshot({ path: path.join(directory, 'mobile-mcp-interrupted-input.png') });
|
||||
await row.getByRole('button', { name: '人工核对结果', exact: true }).click();
|
||||
const review = page.getByRole('dialog', { name: '核对未知的外部结果', exact: true });
|
||||
await expect(review).toContainText('没有完整原报告'); await review.locator('summary').click(); await expect(review).toContainText('owned-ui-crash-effect');
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
assert.ok(await review.evaluate(node => node.scrollWidth <= node.clientWidth));
|
||||
await page.screenshot({ path: path.join(directory, 'mobile-mcp-review.png') });
|
||||
await review.getByRole('button', { name: '我已核对原输入和进程', exact: true }).click(); await expect(review).toBeHidden();
|
||||
const job = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === crash.jobId);
|
||||
assert.equal(job.status, 'unknown'); assert.equal(job.requiresVerification, false); assert.equal(job.reportArtifactId, undefined);
|
||||
const session = await host.api('/sessions/' + f.session.id); assert.deepEqual(session.execution.progress.requiresVerification, []);
|
||||
assert.equal(model.requests.length, 1);
|
||||
mark('390px original-input review clears the job and invocation gates while retaining unknown, no report and no automatic model replay');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, assertions, viewportWidths: [1440, 390],
|
||||
pageErrors, requestFailures, localFixtureModelRequests: model.requests.length, realModelRequests: 0, jobId: job.id }, null, 2));
|
||||
console.log('EVIDENCE ' + directory);
|
||||
} catch (error) {
|
||||
await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {});
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.message, assertions, pageErrors, requestFailures }, null, 2));
|
||||
console.error('EVIDENCE ' + directory); throw error;
|
||||
} finally { await browser.close(); await host.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
@@ -0,0 +1,111 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, compatibleFixture, completion, call, until } from '../tests/helpers.mjs';
|
||||
|
||||
const directory = path.resolve('.qa', 'mcp-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t); f.store.close();
|
||||
const entry = path.join(f.root, 'registered-mcp.mjs'); fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
|
||||
let registration;
|
||||
const model = await compatibleFixture(t, () => completion([call('ui-unknown-call', 'call_registered_mcp', {
|
||||
registration_id: registration.id, configuration_revision: registration.configurationRevision, contract_sha256: registration.contract.sha256,
|
||||
tool: 'hold', arguments: { text: 'effect-can-be-unknown' },
|
||||
})]));
|
||||
const host = await launchWorkbench(t, f.data); const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const pageErrors = [], requestFailures = [], assertions = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const mark = value => { assertions.push(value); console.log('PASS ' + value); };
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
||||
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); await expect(dialog).toBeVisible();
|
||||
const form = dialog.locator('.mcp-register');
|
||||
await form.getByLabel('显示名称', { exact: true }).fill('只读与工程工具');
|
||||
await form.getByLabel('可执行文件绝对路径', { exact: true }).fill(process.execPath);
|
||||
await form.getByLabel('Node 脚本入口(可选)', { exact: true }).fill(entry);
|
||||
await form.getByRole('button', { name: '登记后保持停用', exact: true }).click();
|
||||
const row = dialog.locator('.mcp-registration').filter({ hasText: '只读与工程工具' });
|
||||
await expect(row).toContainText('已停用'); await expect(row).toContainText('尚未探测');
|
||||
mark('MCP registration stays disabled and shows the hashed local executable and Node entry');
|
||||
|
||||
await row.getByRole('button', { name: '实际探测', exact: true }).click();
|
||||
const confirm = page.getByRole('dialog', { name: '启动 MCP 实际探测' });
|
||||
await expect(confirm).toContainText('不等于 OS 沙箱');
|
||||
await confirm.getByRole('button', { name: '确认启动并探测', exact: true }).click();
|
||||
await expect(confirm).toBeHidden(); await expect(row).toContainText('握手已核验'); await expect(row).toContainText('发现 3 项工具');
|
||||
await expect(row.locator('.mcp-tool-option')).toHaveCount(3);
|
||||
const echo = row.locator('.mcp-tool-option').filter({ hasText: 'echo' });
|
||||
await echo.locator('summary').click(); await expect(echo.locator('pre')).toContainText('additionalProperties');
|
||||
await echo.locator('input[type="checkbox"]').check();
|
||||
await row.getByRole('button', { name: '保存并启用所选工具', exact: true }).click();
|
||||
await expect(row).toContainText('已启用'); await expect(dialog.locator('.mcp-history-row')).toContainText('完成');
|
||||
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
|
||||
assert.equal(registration.enabled, true); assert.deepEqual(registration.allowedTools, ['echo']);
|
||||
await dialog.locator('.mcp-heading').scrollIntoViewIfNeeded();
|
||||
await page.screenshot({ path: path.join(directory, 'desktop-mcp-registry.png') });
|
||||
mark('actual stdio tools/list schema is visible; only the reviewed echo tool is enabled');
|
||||
|
||||
fs.appendFileSync(entry, '\n// Owned changed entry for disable acceptance.\n');
|
||||
await dialog.getByRole('button', { name: '刷新', exact: true }).click(); await expect(row).toContainText('来源或契约异常');
|
||||
await row.getByRole('button', { name: '立即停用', exact: true }).click(); await expect(row).toContainText('已停用');
|
||||
assert.equal((await host.api('/projects/' + f.project.id + '/mcp')).registrations[0].enabled, false);
|
||||
fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
|
||||
await dialog.getByRole('button', { name: '刷新', exact: true }).click();
|
||||
mark('an enabled program whose source changes can be disabled immediately without executing the changed entry');
|
||||
|
||||
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
|
||||
await host.api('/projects/' + f.project.id + '/mcp/configure', { id: registration.id, configurationRevision: registration.configurationRevision, enabled: true, allowedTools: ['hold'] });
|
||||
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
|
||||
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'owned-mcp-ui', apiKey: 'ui-mcp-fixture-key', contextLimit: 24000 });
|
||||
await host.api('/sessions/' + f.session.id + '/messages', { content: 'Use the enabled external tool and preserve its real execution record.' });
|
||||
await until(async () => (await host.api('/sessions/' + f.session.id)).status === 'awaiting_approval');
|
||||
await page.keyboard.press('Escape');
|
||||
await page.getByRole('button', { name: '确认执行', exact: true }).click();
|
||||
await until(async () => (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).some(value => value.operation === 'call' && value.status === 'calling'));
|
||||
const job = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.operation === 'call' && value.status === 'calling');
|
||||
await until(() => fs.existsSync(path.join(job.directory, 'owned-call-dispatched.txt')));
|
||||
await page.getByRole('button', { name: '停止', exact: true }).click();
|
||||
await until(async () => (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).some(value => value.id === job.id && value.status === 'unknown'));
|
||||
const unknown = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === job.id);
|
||||
assert.equal(unknown.requiresVerification, true); assert.ok(unknown.reportArtifactId);
|
||||
mark('approved external call interrupted after dispatch remains unknown and does not replay');
|
||||
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||||
const history = page.getByLabel('MCP运行记录', { exact: true }); await expect(history).toContainText('结果待核对');
|
||||
await history.locator('.mcp-history-row.state-unknown').getByRole('button', { name: '查看原报告', exact: true }).click();
|
||||
await expect(history.getByLabel('MCP原报告')).toContainText('"status": "unknown"');
|
||||
await history.getByLabel('MCP原报告').locator('summary').click();
|
||||
await expect(history.getByLabel('MCP原报告')).toContainText('effect-can-be-unknown');
|
||||
await history.getByRole('button', { name: '人工核对结果', exact: true }).click();
|
||||
const review = page.getByRole('dialog', { name: '核对未知的外部结果' }); await expect(review).toContainText('不会把未知结果改成成功');
|
||||
await review.getByRole('button', { name: '我已查看原报告', exact: true }).click();
|
||||
await expect(review).toBeHidden();
|
||||
const acknowledged = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === job.id);
|
||||
assert.equal(acknowledged.status, 'unknown'); assert.equal(acknowledged.requiresVerification, false);
|
||||
assert.deepEqual((await host.api('/sessions/' + f.session.id)).execution.progress.requiresVerification, []);
|
||||
mark('TaskPanel exposes Task/Run source, original report, and human acknowledgement without marking unknown as success');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
|
||||
const mobileDialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); await expect(mobileDialog).toBeVisible();
|
||||
await expect(mobileDialog.locator('.mcp-registration')).toHaveCount(1);
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
await mobileDialog.locator('.mcp-heading').scrollIntoViewIfNeeded();
|
||||
await page.screenshot({ path: path.join(directory, 'mobile-mcp-registry.png') });
|
||||
mark('390px MCP registry, long executable paths, controls, and operation history have no horizontal overflow');
|
||||
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, assertions, viewportWidths: [1440, 390], pageErrors, requestFailures,
|
||||
localFixtureModelRequests: model.requests.length, realModelRequests: 0, unknownJobId: job.id,
|
||||
screenshots: ['desktop-mcp-registry.png', 'mobile-mcp-registry.png'] }, null, 2));
|
||||
console.log('EVIDENCE ' + directory);
|
||||
} catch (error) {
|
||||
await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {});
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.message, assertions, pageErrors, requestFailures }, null, 2));
|
||||
console.error('EVIDENCE ' + directory); throw error;
|
||||
} finally { await browser.close(); await host.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
@@ -92,7 +92,7 @@ try {
|
||||
const checkLog = await api('/artifacts/' + checked.results[0].logArtifactId + '?session=' + session.id); assert.match(checkLog.content, /tests 4/);
|
||||
assert.ok(fs.existsSync(path.join(appRoot, 'runtime/node_modules/npm/bin/npm-cli.js')));
|
||||
const actualRea = [];
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(manifest.version)) {
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(manifest.version)) {
|
||||
const installation = await api('/research/availability'); assert.equal(installation.status, 'available'); assert.equal(installation.version, '6.8.0');
|
||||
assert.equal(fs.realpathSync.native(installation.packageRoot), fs.realpathSync.native(path.join(appRoot, 'runtime/node_modules/rea-agents')));
|
||||
assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
|
||||
@@ -124,7 +124,7 @@ try {
|
||||
console.log('PASS actual contained REA: ASAR, PE imports and PE resources; zero target execution/model requests');
|
||||
}
|
||||
let actualEngine = null;
|
||||
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(manifest.version) && process.argv.includes('--native-engines')) {
|
||||
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(manifest.version) && process.argv.includes('--native-engines')) {
|
||||
const installation = await api('/engines/deepseek/availability'); assert.equal(installation.status, 'available', installation.message);
|
||||
assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
|
||||
const engineRoot = path.join(data, 'engine-project'); fs.mkdirSync(engineRoot); fs.writeFileSync(path.join(engineRoot, 'README.md'), '# Owned portable engine connection\n');
|
||||
@@ -144,7 +144,7 @@ try {
|
||||
assert.equal((await api('/research')).length, 4);
|
||||
const page = await fetch(url); assert.equal(page.status, 200);
|
||||
assert.match(await page.text(), /Agent Workbench/);
|
||||
const actualBrowser = ['1.10.0', '1.10.1', '1.11.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null;
|
||||
const actualBrowser = ['1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null;
|
||||
console.log((await stop()).trim()); started = false;
|
||||
await until(async () => { try { await fetch(url + '/api/health'); return false; } catch { return true; } });
|
||||
const result = { success: true, date: new Date().toISOString(), appRoot, version: manifest.version, nodeVersion: manifest.nodeVersion,
|
||||
|
||||
@@ -43,14 +43,19 @@ function run(args, logName) {
|
||||
const logPath = path.join(directory, logName);
|
||||
const outputFile = fs.createWriteStream(logPath);
|
||||
const child = spawn(process.execPath, args, { cwd: sourceRoot, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let tail = '';
|
||||
let tail = '', timedOut = false;
|
||||
const capture = bytes => { outputFile.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-12000); };
|
||||
child.stdout.on('data', capture); child.stderr.on('data', capture);
|
||||
const timer = setTimeout(() => child.kill(), 600000);
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
if (process.platform === 'win32' && child.pid) {
|
||||
try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000, stdio: 'ignore' }); } catch { child.kill(); }
|
||||
} else child.kill();
|
||||
}, 30 * 60 * 1000);
|
||||
child.once('error', error => { clearTimeout(timer); outputFile.end(); reject(error); });
|
||||
child.once('close', code => { clearTimeout(timer); outputFile.end(() => {
|
||||
logs.push({ stage, path: logPath, sha256: digest(logPath), bytes: fs.statSync(logPath).size });
|
||||
code === 0 ? resolve(tail) : reject(new Error(stage + ' failed (' + code + '): ' + tail));
|
||||
code === 0 && !timedOut ? resolve(tail) : reject(new Error(stage + (timedOut ? ' timed out after 30 minutes' : ' failed (' + code + ')') + ': ' + tail));
|
||||
}); });
|
||||
});
|
||||
}
|
||||
@@ -74,7 +79,7 @@ try {
|
||||
assert.deepEqual(actualFiles.slice().sort(), Object.keys(expected).sort());
|
||||
for (const [relative, sha256] of Object.entries(expected)) assert.equal(digest(path.join(sourceRoot, relative)), sha256, relative);
|
||||
let actualRea = null;
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(evidence.version)) {
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(evidence.version)) {
|
||||
stage = 'contained REA';
|
||||
const reaOutput = await run(['scripts/verify-rea.mjs', '--app-root', sourceRoot, '--contained'], 'rea.log');
|
||||
actualRea = JSON.parse(reaOutput.trim().split('\n').at(-1));
|
||||
@@ -82,7 +87,7 @@ try {
|
||||
assert.ok(actualRea.results.every(item => item.rootExited === true));
|
||||
}
|
||||
let actualEngine = null;
|
||||
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(evidence.version) && process.argv.includes('--native-engines')) {
|
||||
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(evidence.version) && process.argv.includes('--native-engines')) {
|
||||
stage = 'native DeepSeek SDK';
|
||||
const output = await run(['scripts/verify-engines.mjs'], 'engines.log');
|
||||
actualEngine = JSON.parse(output.trim().split('\n').at(-1));
|
||||
@@ -90,7 +95,7 @@ try {
|
||||
assert.equal(actualEngine.modelRequests, 0); assert.equal(actualEngine.rootExited, true); assert.equal(actualEngine.nativeMethods.length, 6);
|
||||
}
|
||||
let actualBrowser = null;
|
||||
if (['1.10.0', '1.10.1', '1.11.0'].includes(evidence.version)) {
|
||||
if (['1.10.0', '1.10.1', '1.11.0', '1.12.0'].includes(evidence.version)) {
|
||||
stage = 'contained browser runtime';
|
||||
const output = await run(['scripts/verify-browser.mjs', '--app-root', sourceRoot, '--contained'], 'browser.log');
|
||||
actualBrowser = JSON.parse(output.trim().split('\n').at(-1));
|
||||
|
||||
@@ -164,5 +164,6 @@ try {
|
||||
result = { success: false, version: host.bootstrap.version, testedAt: new Date().toISOString(), directory, passed, pageErrors, requestFailures, screenshots, error: error.stack }; process.exitCode = 1;
|
||||
} finally { releaseHeld?.(); deliberateNavigation = true; await browser.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync('docs/task-plans-task-ui-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
const proofPrefix = (await import('./qa-evidence.mjs')).releaseEvidencePrefix(result.version);
|
||||
fs.writeFileSync(`docs/${proofPrefix}-task-ui-validation.json`, JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, groups: passed.length, directory, error: result.error }));
|
||||
|
||||
@@ -20,14 +20,26 @@ const inputs = snapshot(), tests = fs.readdirSync('tests').filter(name => name.e
|
||||
assert.ok(tests.length); fs.writeFileSync(path.join(directory, 'inputs.json'), JSON.stringify(inputs, null, 2) + '\n');
|
||||
const allowed = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all|AGENT_WORKBENCH_TEST_PYTHON)$/i;
|
||||
const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => allowed.test(name)));
|
||||
const logPath = path.join(directory, 'tests.tap'), output = fs.createWriteStream(logPath), startedAt = new Date().toISOString(); let result, failure;
|
||||
const timeoutMs = 30 * 60 * 1000;
|
||||
const logPath = path.join(directory, 'tests.tap'), output = fs.createWriteStream(logPath), startedAt = new Date().toISOString(); let result, failure, timedOut = false;
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['--test', '--test-concurrency=2', '--test-reporter=tap', ...tests], { cwd: root, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
child.stdout.pipe(output, { end: false }); child.stderr.pipe(output, { end: false });
|
||||
const timer = setTimeout(() => child.kill(), 600000);
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
if (process.platform === 'win32' && child.pid) {
|
||||
const killer = spawn('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' });
|
||||
killer.once('error', () => child.kill());
|
||||
killer.once('close', code => { if (code !== 0) child.kill(); });
|
||||
} else child.kill('SIGTERM');
|
||||
}, timeoutMs);
|
||||
child.once('error', error => { clearTimeout(timer); reject(error); });
|
||||
child.once('close', code => { clearTimeout(timer); code === 0 ? resolve() : reject(new Error('Regression process exit: ' + code)); });
|
||||
child.once('close', code => {
|
||||
clearTimeout(timer);
|
||||
if (timedOut) reject(new Error(`Regression timed out after ${timeoutMs}ms; its process tree was terminated.`));
|
||||
else code === 0 ? resolve() : reject(new Error('Regression process exit: ' + code));
|
||||
});
|
||||
});
|
||||
} catch (error) { failure = error; }
|
||||
await new Promise(resolve => output.end(resolve));
|
||||
@@ -36,14 +48,12 @@ try {
|
||||
if (failure) throw Object.assign(failure, { tail: text.slice(-16000) });
|
||||
assert.ok(stats.tests > 0); assert.equal(stats.pass, stats.tests); for (const key of ['fail', 'cancelled', 'skipped', 'todo']) assert.equal(stats[key], 0);
|
||||
assert.deepEqual(snapshot(), inputs, 'Inputs changed during regression; this is not final-source evidence.');
|
||||
result = { success: true, version, startedAt, finishedAt: new Date().toISOString(), tests: stats, directory, nodeVersion: process.version, inputFiles: inputs.length, inputs,
|
||||
result = { success: true, version, startedAt, finishedAt: new Date().toISOString(), timeoutMs, tests: stats, directory, nodeVersion: process.version, inputFiles: inputs.length, inputs,
|
||||
log: { path: logPath, bytes: fs.statSync(logPath).size, sha256: hash(logPath) }, credentialsHandling: 'Only runtime/system paths and the independent QA Python path are inherited; no real model configuration or credentials.' };
|
||||
} catch (error) {
|
||||
result = { success: false, version, startedAt, finishedAt: new Date().toISOString(), directory, error: error.stack, tail: error.tail, inputs, log: { path: logPath, sha256: hash(logPath) } }; process.exitCode = 1;
|
||||
result = { success: false, version, startedAt, finishedAt: new Date().toISOString(), timeoutMs, timedOut, directory, error: error.stack, tail: error.tail, inputs, log: { path: logPath, sha256: hash(logPath) } }; process.exitCode = 1;
|
||||
}
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n'); fs.writeFileSync('.qa/tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
if (version === '1.8.0') fs.writeFileSync('docs/rea-engine-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
if (version === '1.9.0') fs.writeFileSync('docs/external-engines-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
if (version === '1.10.0') fs.writeFileSync('docs/browser-engine-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
if (version === '1.11.0') fs.writeFileSync('docs/task-plans-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
// Release evidence is archived explicitly by the release workflow. A development
|
||||
// regression must not replace a previously published report for the same version.
|
||||
console.log(JSON.stringify({ success: result.success, version, tests: result.tests, inputFiles: result.inputFiles, directory, error: result.error, tail: result.tail }, null, 2));
|
||||
|
||||
Reference in New Issue
Block a user