feat: verify task dependency plans against original execution evidence
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
// Keep every release's acceptance files separate from historical evidence.
|
||||
const prefixes = Object.freeze({
|
||||
'1.4.0': 'm2', '1.5.0': 'context', '1.5.1': 'context-fix', '1.6.0': 'workspaces',
|
||||
'1.6.1': 'working-context', '1.6.2': 'working-retention', '1.7.0': 'python-engine',
|
||||
'1.8.0': 'rea-engine', '1.9.0': 'external-engines', '1.10.0': 'browser-engine',
|
||||
'1.10.1': 'browser-hotfix', '1.11.0': 'task-plans',
|
||||
});
|
||||
export function releaseEvidencePrefix(version) {
|
||||
const prefix = prefixes[version];
|
||||
if (!prefix) throw new Error('Review evidence paths when introducing release ' + version + '.');
|
||||
return prefix;
|
||||
}
|
||||
@@ -64,7 +64,7 @@ export async function upgradeRunning({ url, pid, currentRoot, newRoot, handlerRo
|
||||
if (!fs.existsSync(originalScript) || !fs.existsSync(upgradeScript) || !fs.existsSync(path.join(next, 'dist-client', 'index.html'))) throw new Error('Upgrade bundles are incomplete.');
|
||||
if (!Number.isInteger(pid) || pid < 1 || pid === process.pid) throw new Error('Invalid target process.');
|
||||
const before = await readUpgradeJson(origin + '/api/bootstrap', 5000, 'Read original Workbench');
|
||||
if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.');
|
||||
if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.');
|
||||
if (before.sessions.some(session => ['running', 'awaiting_approval', 'stopping'].includes(session.status))) throw new Error('Wait for existing tasks and approvals before upgrading.');
|
||||
let protocol, socket, enabledHere = false, paused = false;
|
||||
const breakpoints = [];
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
@@ -6,19 +7,20 @@ import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
||||
assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0'].includes(version), 'Review evidence paths when introducing the next release.');
|
||||
const prefix = version === '1.6.0' ? 'workspaces' : version === '1.6.1' ? 'working-context' : version === '1.6.2' ? 'working-retention' : version === '1.7.0' ? 'python-engine' : version === '1.10.0' ? 'browser-engine' : version === '1.9.0' ? 'external-engines' : 'rea-engine', resultFile = `.qa/${prefix}-all-ui-validation.json`, results = [];
|
||||
assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version), 'Review evidence paths when introducing the next release.');
|
||||
const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = [];
|
||||
const cases = [
|
||||
...(version === '1.10.0' ? [['verify-browser-ui.mjs', 'docs/browser-engine-browser-ui-validation.json']] : []),
|
||||
...(version === '1.11.0' ? [['verify-task-plans-ui.mjs', 'docs/task-plans-task-ui-validation.json']] : []),
|
||||
...(['1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []),
|
||||
['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'],
|
||||
['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'],
|
||||
['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`],
|
||||
['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`],
|
||||
['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`],
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []),
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []),
|
||||
...(['1.8.0', '1.9.0', '1.10.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []),
|
||||
...(['1.9.0', '1.10.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.10.0' ? 'docs/browser-engine-engines-ui-validation.json' : 'docs/engines-ui-validation.json']] : []),
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []),
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []),
|
||||
...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []),
|
||||
...(['1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []),
|
||||
];
|
||||
fs.mkdirSync('.qa', { recursive: true });
|
||||
const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']);
|
||||
@@ -64,7 +66,7 @@ for (const [script, proof] of cases) {
|
||||
const errors = evidence.pageErrors ?? [], failures = evidence.requestFailures ?? [];
|
||||
assert.equal(errors.length, 0); assert.equal(failures.length, 0);
|
||||
const saved = script === 'verify-ui.mjs' ? `docs/${prefix}-basic-ui-validation.json`
|
||||
: script === 'verify-workspaces-ui.mjs' ? proof
|
||||
: ['verify-workspaces-ui.mjs', 'verify-task-plans-ui.mjs'].includes(script) ? proof
|
||||
: `docs/${prefix}-${script.replace('verify-', '').replace('.mjs', '')}-validation.json`;
|
||||
if (path.resolve(saved) !== path.resolve(proof)) fs.copyFileSync(proof, saved);
|
||||
results.push({ script, success: true, groups: passed.length, durationMs: Date.now() - started, evidence: saved, log });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
@@ -136,5 +137,5 @@ try {
|
||||
result = { success: false, version: host.bootstrap.version, testedAt: new Date().toISOString(), directory, passed, pageErrors, requestFailures, screenshots, error: error.stack }; process.exitCode = 1;
|
||||
} finally { deliberateNavigation = true; await browser.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync('docs/browser-engine-browser-ui-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(`docs/${releaseEvidencePrefix(host.bootstrap.version)}-browser-ui-validation.json`, JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, groups: passed.length, directory, error: result.error }));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
@@ -33,8 +34,8 @@ child.stdout.on('data', value => { output += value; for (const line of output.sp
|
||||
child.stderr.on('data', value => { errors += value; });
|
||||
const wait = async predicate => { const deadline = Date.now() + 15000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('UI fixture timed out: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
|
||||
await wait(() => url); const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.ok(['1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const proofPrefix = bootstrap.version === '1.4.0' ? 'm2-batch' : bootstrap.version === '1.5.0' ? 'context-batch' : bootstrap.version === '1.5.1' ? 'context-fix-batch' : bootstrap.version === '1.6.0' ? 'workspaces-batch' : bootstrap.version === '1.6.1' ? 'working-context-batch' : bootstrap.version === '1.6.2' ? 'working-retention-batch' : bootstrap.version === '1.7.0' ? 'python-engine-batch' : bootstrap.version === '1.10.0' ? 'browser-engine-batch' : bootstrap.version === '1.9.0' ? 'external-engines-batch' : 'rea-engine-batch';
|
||||
assert.ok(['1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const proofPrefix = releaseEvidencePrefix(bootstrap.version) + '-batch';
|
||||
const screenshotPrefix = bootstrap.version === '1.4.0' ? '1.4-batch' : proofPrefix;
|
||||
const api = async (route, input) => { const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }) }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
@@ -76,9 +77,9 @@ child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
||||
const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
|
||||
await until(() => url || child.exitCode !== null); assert.ok(url, errors);
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
|
||||
assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
|
||||
const proofPrefix = bootstrap.version === '1.5.0' ? 'context' : bootstrap.version === '1.5.1' ? 'context-fix' : bootstrap.version === '1.6.0' ? 'workspaces-context' : bootstrap.version === '1.6.1' ? 'working-context' : bootstrap.version === '1.6.2' ? 'working-retention' : bootstrap.version === '1.7.0' ? 'python-engine' : bootstrap.version === '1.10.0' ? 'browser-engine' : bootstrap.version === '1.9.0' ? 'external-engines' : 'rea-engine';
|
||||
const proofPrefix = bootstrap.version === '1.6.0' ? 'workspaces-context' : releaseEvidencePrefix(bootstrap.version);
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const passed = [], pageErrors = [], requestFailures = [], screenshots = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
|
||||
@@ -52,21 +52,28 @@ assert.equal(await new Promise(resolve => exited.once('close', resolve)), 73, ex
|
||||
const crashed = JSON.parse(exitOutput.trim()), recoveredStore = new Store(data), recoveryJob = structuredClone(recoveredStore.owned('delivery_job', crashed.jobId, recoverySession.id));
|
||||
assert.equal(recoveryJob.status, 'unknown'); recoveredStore.close();
|
||||
|
||||
let modelMode = 'initial';
|
||||
let modelMode = 'initial', modelStage = 0;
|
||||
const model = http.createServer(async (request, response) => {
|
||||
try {
|
||||
const chunks = []; for await (const chunk of request) chunks.push(chunk);
|
||||
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
||||
const last = input.messages.findLastIndex(item => item.role === 'user');
|
||||
const results = input.messages.slice(last + 1).filter(item => item.role === 'tool').map(item => JSON.parse(item.content));
|
||||
const plan = results.find(value => value.checks && value.fingerprint);
|
||||
// The request intentionally seals old complete tool groups when bounded.
|
||||
// Consume the original fields in its visible feedback, not history length.
|
||||
const feedback = input.messages.filter(item => item.role === 'system').flatMap(item => {
|
||||
for (const line of item.content.split('\n')) try { const value = JSON.parse(line); if (value.data?.kind === 'latest-tool-feedback') return value.data.entries.map(entry => entry.result); } catch {}
|
||||
return [];
|
||||
});
|
||||
const plan = [...results, ...feedback].findLast(value => value.checks && value.fingerprint);
|
||||
let calls = [];
|
||||
if (!results.length) calls = [['inspect_project', {}]];
|
||||
else if (results.length === 1) calls = [['plan_checks', {}]];
|
||||
else if (results.length === 2) calls = [['run_checks', { plan_id: plan.id }]];
|
||||
else if (results.length === 3 && modelMode === 'initial') calls = allowed.map(relative => ['propose_file', { path: relative, content: repairs[relative], base_sha256: initialHashes[relative], reason: 'Controlled four-module repair after actual failure logs' }]);
|
||||
else if (results.length === 3 && modelMode === 'final') calls = [['plan_delivery', { message: 'feat: prepare verified four-module delivery' }]];
|
||||
else if (results.length === 4 && modelMode === 'final') calls = [['delivery_report', {}]];
|
||||
if (modelStage === 0) calls = [['inspect_project', {}]];
|
||||
else if (modelStage === 1) calls = [['plan_checks', {}]];
|
||||
else if (modelStage === 2) { assert.ok(plan?.id, 'The actual check plan must remain visible in raw results or original-field feedback.'); calls = [['run_checks', { plan_id: plan.id }]]; }
|
||||
else if (modelStage === 3 && modelMode === 'initial') calls = allowed.map(relative => ['propose_file', { path: relative, content: repairs[relative], base_sha256: initialHashes[relative], reason: 'Controlled four-module repair after actual failure logs' }]);
|
||||
else if (modelStage === 3 && modelMode === 'final') calls = [['plan_delivery', { message: 'feat: prepare verified four-module delivery' }]];
|
||||
else if (modelStage === 4 && modelMode === 'final') calls = [['delivery_report', {}]];
|
||||
modelStage++;
|
||||
response.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: calls.length ? '' : modelMode === 'initial' ? '四个文件已准备提案,等待审查。' : '实际14项测试和构建通过,Git预览已准备,等待确认。',
|
||||
...(calls.length ? { tool_calls: calls.map(([name, args]) => ({ id: randomUUID(), type: 'function', function: { name, arguments: JSON.stringify(args) } })) } : {}) },
|
||||
@@ -119,7 +126,7 @@ try {
|
||||
await page.locator('.change-list').getByRole('button').filter({ hasText: relative }).click();
|
||||
await page.getByRole('button', { name: '应用这项改动', exact: true }).click();
|
||||
}
|
||||
await waitStatus('completed'); modelMode = 'final'; await page.getByRole('button', { name: '继续检查与交付', exact: true }).click();
|
||||
await waitStatus('completed'); modelMode = 'final'; modelStage = 0; await page.getByRole('button', { name: '继续检查与交付', exact: true }).click();
|
||||
await waitStatus('awaiting_approval'); await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitStatus('needs_review');
|
||||
const verified = await state(); assert.equal(verified.execution.task.id, reviewed.execution.task.id); assert.equal(verified.execution.checks.at(-1).status, 'passed');
|
||||
assert.equal(verified.execution.checks.at(-1).results[0].tests.passed, 14); assert.equal(verified.execution.deliveries.length, 0);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
@@ -137,5 +138,5 @@ try {
|
||||
result = { success: false, version: host.bootstrap.version, testedAt: new Date().toISOString(), root, passed, pageErrors, requestFailures, expectedRequestFailures, screenshots, error: error.stack }; process.exitCode = 1;
|
||||
} finally { deliberateNavigation = true; await browser.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(result.version === '1.10.0' ? 'docs/browser-engine-engines-ui-validation.json' : 'docs/engines-ui-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(result.version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${releaseEvidencePrefix(result.version)}-engines-ui-validation.json`, JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, groups: passed.length, root, pageErrors, requestFailures, error: result.error }));
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { APP_VERSION } from '../src/server/diagnostics.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-live-task-plans.mjs — resume the prior page-only model Task with two sealed technical steps; reuse its verified desktop result and let the original Provider correct/verify mobile'); process.exit(0); }
|
||||
const prior = JSON.parse(fs.readFileSync('docs/browser-engine-live-model-validation.json', 'utf8'));
|
||||
const engineering = JSON.parse(fs.readFileSync('docs/live-react-engineering-validation.json', 'utf8'));
|
||||
const url = 'http://127.0.0.1:4318', directory = path.resolve('.qa', 'live-task-plans-' + randomUUID()); fs.mkdirSync(directory);
|
||||
const sessionId = prior.sessionId, taskId = prior.taskId, root = engineering.workspaceRoot;
|
||||
const sha = value => createHash('sha256').update(Buffer.isBuffer(value) || typeof value === 'string' ? value : JSON.stringify(value)).digest('hex');
|
||||
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
||||
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', root, ...args], { env: gitEnvironment(), windowsHide: true, encoding: 'utf8' }).trim();
|
||||
function sourceFiles(relative = '') {
|
||||
return fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
|
||||
if (entry.name.startsWith('.') || ['node_modules', 'dist', 'build', 'out'].includes(entry.name)) return [];
|
||||
assert.equal(entry.isSymbolicLink(), false); const file = path.join(relative, entry.name);
|
||||
return entry.isDirectory() ? sourceFiles(file) : [{ path: file, sha256: sha(fs.readFileSync(path.join(root, file))) }];
|
||||
});
|
||||
}
|
||||
let token, current, baseline, result, beforeUsage, initialModelRequests = 0, modelStarted = false;
|
||||
const decisions = [], continuations = [], verifierProofs = [], startedAt = Date.now();
|
||||
async function api(route, input) {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { Connection: 'close', 'Content-Type': 'application/json', ...(token ? { 'X-Workbench-Token': token } : {}) },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
|
||||
const value = await response.json(); if (!response.ok) throw new Error(value.error ?? 'HTTP ' + response.status); return value;
|
||||
}
|
||||
const sessionRoute = '/sessions/' + sessionId;
|
||||
async function exportOriginal(job) {
|
||||
let text = '', offset = 0, page;
|
||||
do { page = await api(sessionRoute + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=16000'); text += page.content; offset += page.content.length; } while (page.truncated);
|
||||
assert.equal(sha(text), job.resultSha256); const original = JSON.parse(text); assert.equal(original.taskId, taskId);
|
||||
fs.writeFileSync(path.join(directory, 'original-' + job.id + '.json'), text);
|
||||
for (const image of job.screenshots) {
|
||||
const response = await fetch(url + '/api' + sessionRoute + '/browser-jobs/' + job.id + '/images/' + image.artifactId, { headers: { 'X-Workbench-Token': token }, signal: AbortSignal.timeout(15000) });
|
||||
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), image.sha256);
|
||||
fs.writeFileSync(path.join(directory, job.id + '-' + image.label + '.png'), bytes);
|
||||
}
|
||||
return original;
|
||||
}
|
||||
try {
|
||||
assert.equal(prior.success, false); assert.equal(prior.version, '1.10.1'); assert.equal(engineering.success, true);
|
||||
const before = await api('/bootstrap'); token = before.token; assert.equal(before.version, APP_VERSION); assert.equal(APP_VERSION, '1.11.0');
|
||||
assert.equal(before.config.mode, 'compatible'); assert.equal(before.config.hasKey, true);
|
||||
const project = before.projects.find(value => value.id === engineering.projectId); assert.ok(project); assert.equal(project.root, root);
|
||||
assert.ok(!before.sessions.some(value => value.projectId === project.id && ['running', 'awaiting_approval', 'stopping'].includes(value.status)));
|
||||
current = await api(sessionRoute); assert.equal(current.execution.task.id, taskId); assert.equal(current.projectId, project.id);
|
||||
assert.ok(['paused', 'failed', 'cancelled'].includes(current.status), 'Only an unfinished original Task may be resumed.');
|
||||
assert.equal(current.changes.length, 0); beforeUsage = structuredClone(current.execution.task.usage); initialModelRequests = beforeUsage.requests;
|
||||
const originalExport = await api(sessionRoute + '/export');
|
||||
const oldEngineering = await api('/sessions/' + engineering.sessionId + '/export'); delete oldEngineering.exportedAt;
|
||||
baseline = { source: sourceFiles(), head: git(['rev-parse', 'HEAD']), index: sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))),
|
||||
originalMessages: originalExport.session.providerMessages, originalJobs: originalExport.execution.browserJobs,
|
||||
engineering: sha(canonical(oldEngineering)), config: sha(canonical(before.config)) };
|
||||
fs.writeFileSync(path.join(directory, 'baseline.json'), JSON.stringify(baseline, null, 2) + '\n');
|
||||
const previousPlan = await api(sessionRoute + '/task-plan');
|
||||
const saved = await api(sessionRoute + '/task-plans', { taskId, runId: current.execution.run.id, expected_revision: previousPlan.revision,
|
||||
steps: [
|
||||
{ key: 'desktop', title: '桌面搜索、筛选与单任务切换', criterion: '1280px实际填写、筛选和点击,原断言通过,完成2项且待办工时5小时', kind: 'browser', viewport_width: 1280, actions: ['fill', 'select', 'click', 'assert_count'], depends_on: [] },
|
||||
{ key: 'mobile', title: '手机布局与控件实际操作', criterion: '390px实际点击并核对可见性及数量,原截图无横向溢出', kind: 'browser', viewport_width: 390, actions: ['click', 'assert_visible', 'assert_count'], depends_on: ['desktop'] },
|
||||
] });
|
||||
current = saved.session;
|
||||
const desktop = current.execution.browserJobs.findLast(job => job.status === 'passed' && current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 1280);
|
||||
assert.ok(desktop, 'The original desktop pass must be present; this verifier does not regenerate it.');
|
||||
const proof = await api(sessionRoute + '/task-proofs', { taskId, runId: current.execution.run.id, plan_id: saved.plan.plan_id,
|
||||
expected_revision: saved.plan.revision, step_key: 'desktop', evidence_id: desktop.id });
|
||||
assert.equal(proof.verification.status, 'completed'); verifierProofs.push({ step: 'desktop', evidenceId: desktop.id, proofId: proof.verification.proof_id, originalExecutionReplayed: false });
|
||||
await api(sessionRoute + '/resume', {}); modelStarted = true;
|
||||
const decided = new Set();
|
||||
while (Date.now() - startedAt < 210000) {
|
||||
current = await api(sessionRoute); assert.equal(current.execution.task.id, taskId); assert.equal(current.changes.length, 0);
|
||||
if (current.status === 'awaiting_approval') {
|
||||
const approval = current.approvals.find(value => value.status === 'pending' && !decided.has(value.id));
|
||||
if (approval) {
|
||||
assert.ok(approval.browserPlanId, 'Only the original page-only scope may execute.');
|
||||
const plan = current.execution.browserPlans.find(value => value.id === approval.browserPlanId);
|
||||
assert.equal(plan.taskId, taskId); assert.equal(plan.projectId, project.id); assert.equal(plan.build.directory, 'dist'); assert.equal(plan.path, '/');
|
||||
assert.ok([1280, 390].includes(plan.viewport.width)); assert.ok(plan.steps.length <= 20);
|
||||
decided.add(approval.id); decisions.push({ id: approval.id, planId: plan.id, fingerprint: plan.fingerprint, viewport: plan.viewport, steps: plan.steps.length });
|
||||
await api(sessionRoute + '/approvals/' + approval.id, { allow: true });
|
||||
}
|
||||
} else if (['completed', 'paused'].includes(current.status)) {
|
||||
const view = await api(sessionRoute + '/task-plan');
|
||||
if (current.status === 'completed' && view.all_verified) break;
|
||||
const reason = current.execution.run.stopReason;
|
||||
if (current.status !== 'paused' || continuations.length >= 2 || !['step_limit', 'task_plan_incomplete'].includes(reason)) throw new Error('Actual model task remains ' + current.status + '/' + reason);
|
||||
continuations.push({ previousRunId: current.execution.run.id, reason, at: new Date().toISOString() }); await api(sessionRoute + '/resume', {});
|
||||
} else if (['failed', 'cancelled'].includes(current.status)) throw new Error(current.failure?.message ?? current.status);
|
||||
await new Promise(resolve => setTimeout(resolve, 75));
|
||||
}
|
||||
assert.equal(current.status, 'completed', 'Actual model did not finish within the original 210-second acceptance bound.');
|
||||
const view = await api(sessionRoute + '/task-plan'); assert.equal(view.all_verified, true); assert.equal(view.verified_steps, 2);
|
||||
const records = [];
|
||||
for (const step of view.steps) {
|
||||
const job = current.execution.browserJobs.find(value => value.id === step.evidence.id); assert.ok(job); assert.equal(job.cleanup.exited, true);
|
||||
records.push({ job, original: await exportOriginal(job), viewport: current.execution.browserPlans.find(value => value.id === job.planId).viewport });
|
||||
}
|
||||
const desktopRecord = records.find(value => value.viewport.width === 1280), mobile = records.find(value => value.viewport.width === 390);
|
||||
assert.match(desktopRecord.original.pages.at(-1).text, /完成任务\s*2/); assert.match(desktopRecord.original.pages.at(-1).text, /待办工时\s*5\s*小时/);
|
||||
assert.ok(mobile.original.pages.at(-1).viewport.scrollWidth <= 390);
|
||||
assert.deepEqual(sourceFiles(), baseline.source); assert.equal(git(['rev-parse', 'HEAD']), baseline.head);
|
||||
assert.equal(sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))), baseline.index);
|
||||
const afterEngineering = await api('/sessions/' + engineering.sessionId + '/export'); delete afterEngineering.exportedAt;
|
||||
assert.equal(sha(canonical(afterEngineering)), baseline.engineering); assert.equal(sha(canonical((await api('/bootstrap')).config)), baseline.config);
|
||||
const exported = await api(sessionRoute + '/export');
|
||||
assert.deepEqual(exported.session.providerMessages.slice(0, baseline.originalMessages.length), baseline.originalMessages);
|
||||
assert.deepEqual(exported.execution.browserJobs.slice(0, baseline.originalJobs.length), baseline.originalJobs);
|
||||
fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n');
|
||||
result = { success: true, version: APP_VERSION, testedAt: new Date().toISOString(), directory, model: before.config.model, projectId: project.id, root, sessionId, taskId,
|
||||
modelStatus: current.status, sourcePreserved: true, indexPreserved: true, originalEngineeringTaskPreserved: true, originalMessagesAndJobsPreserved: true,
|
||||
beforeUsage, usage: current.execution.task.usage, newModelRequests: current.execution.task.usage.requests - initialModelRequests,
|
||||
elapsedSeconds: (Date.now() - startedAt) / 1000, taskPlan: view, verifierProofs, decisions, continuations,
|
||||
jobs: records.map(value => ({ id: value.job.id, status: value.job.status, viewport: value.viewport, assertions: value.job.assertions, resultSha256: value.job.resultSha256, cleanup: value.job.cleanup })),
|
||||
scope: 'The verifier seals two technical steps and verifies the original desktop evidence once. The actual configured model resumes the same unfinished Task and must produce/verify the mobile result. This assisted page-only acceptance does not measure autonomous planning, code repair or Purge benefit.',
|
||||
credentialsHandling: 'Original in-memory Provider is retained. No model key is read, copied, printed or saved.' };
|
||||
} catch (error) {
|
||||
if (token && modelStarted) { await api(sessionRoute + '/stop', {}).catch(() => {}); current = await api(sessionRoute).catch(() => current); }
|
||||
result = { success: false, version: APP_VERSION, testedAt: new Date().toISOString(), directory, sessionId, taskId, modelStatus: current?.status,
|
||||
beforeUsage, usage: current?.execution?.task?.usage, decisions, verifierProofs, continuations, error: error.stack }; process.exitCode = 1;
|
||||
if (token) { const exported = await api(sessionRoute + '/export').catch(() => null); if (exported) fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n'); }
|
||||
}
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync('docs/task-plans-live-model-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, sessionId, taskId, modelStatus: result.modelStatus, newModelRequests: result.newModelRequests,
|
||||
elapsedSeconds: result.elapsedSeconds, usage: result.usage, jobs: result.jobs, error: result.error, directory }));
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
@@ -12,7 +13,7 @@ const protectedFiles = createPythonDeliveryProject(f), source = captureWorkspace
|
||||
f.session.title = 'Python工程检查与交付'; f.store.core.task.get(f.session.taskId).goal = '修复三个Python模块,保护测试与声明,核对实际检查并交付本地Git报告。';
|
||||
f.store.save(f.session); f.store.close();
|
||||
const host = await launchWorkbench(t, f.data, process.env.AGENT_WORKBENCH_TEST_PYTHON ? { AGENT_WORKBENCH_PYTHON_PATH: process.env.AGENT_WORKBENCH_TEST_PYTHON } : {});
|
||||
const proofPrefix = host.bootstrap.version === '1.8.0' ? 'rea-engine' : host.bootstrap.version === '1.10.0' ? 'browser-engine' : host.bootstrap.version === '1.9.0' ? 'external-engines' : 'python-engine';
|
||||
const proofPrefix = releaseEvidencePrefix(host.bootstrap.version);
|
||||
assert.notEqual(new URL(host.url).port, '4318'); assert.equal(host.bootstrap.config.hasKey, false);
|
||||
fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
@@ -117,7 +118,7 @@ try {
|
||||
checks: actual.results.map(result => ({ name: result.name, status: result.status, exitCode: result.exitCode, tests: result.tests })),
|
||||
commit: job.commit, changes: job.paths, protectedFiles: Object.fromEntries(Object.entries(protectedBytes).map(([relative, bytes]) => [relative, sha(bytes)])),
|
||||
scope: 'Owned frozen React TypeScript project with offline-installed fixed dependencies, real npm/typecheck/build/browser and Workbench review/Git/report. API fixtures supply repairs; configured-model success is separate.' };
|
||||
fs.writeFileSync(host.bootstrap.version === '1.8.0' ? 'docs/rea-engine-react-delivery-validation.json' : host.bootstrap.version === '1.10.0' ? 'docs/browser-engine-react-delivery-validation.json' : host.bootstrap.version === '1.9.0' ? 'docs/external-engines-react-delivery-validation.json' : 'docs/react-engine-delivery-validation.json', JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify(result, null, 2));
|
||||
fs.writeFileSync(host.bootstrap.version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${releaseEvidencePrefix(host.bootstrap.version)}-react-delivery-validation.json`, JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
await snapshot('failure').catch(() => {});
|
||||
fs.writeFileSync('.qa/react-engine-delivery-failure.json', JSON.stringify({ error: error.message, passed, pageErrors, requestFailures, session }, null, 2)); throw error;
|
||||
|
||||
@@ -92,7 +92,7 @@ try {
|
||||
const checkLog = await api('/artifacts/' + checked.results[0].logArtifactId + '?session=' + session.id); assert.match(checkLog.content, /tests 4/);
|
||||
assert.ok(fs.existsSync(path.join(appRoot, 'runtime/node_modules/npm/bin/npm-cli.js')));
|
||||
const actualRea = [];
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1'].includes(manifest.version)) {
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(manifest.version)) {
|
||||
const installation = await api('/research/availability'); assert.equal(installation.status, 'available'); assert.equal(installation.version, '6.8.0');
|
||||
assert.equal(fs.realpathSync.native(installation.packageRoot), fs.realpathSync.native(path.join(appRoot, 'runtime/node_modules/rea-agents')));
|
||||
assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
|
||||
@@ -124,7 +124,7 @@ try {
|
||||
console.log('PASS actual contained REA: ASAR, PE imports and PE resources; zero target execution/model requests');
|
||||
}
|
||||
let actualEngine = null;
|
||||
if (['1.9.0', '1.10.0', '1.10.1'].includes(manifest.version) && process.argv.includes('--native-engines')) {
|
||||
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(manifest.version) && process.argv.includes('--native-engines')) {
|
||||
const installation = await api('/engines/deepseek/availability'); assert.equal(installation.status, 'available', installation.message);
|
||||
assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
|
||||
const engineRoot = path.join(data, 'engine-project'); fs.mkdirSync(engineRoot); fs.writeFileSync(path.join(engineRoot, 'README.md'), '# Owned portable engine connection\n');
|
||||
@@ -144,7 +144,7 @@ try {
|
||||
assert.equal((await api('/research')).length, 4);
|
||||
const page = await fetch(url); assert.equal(page.status, 200);
|
||||
assert.match(await page.text(), /Agent Workbench/);
|
||||
const actualBrowser = ['1.10.0', '1.10.1'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null;
|
||||
const actualBrowser = ['1.10.0', '1.10.1', '1.11.0'].includes(manifest.version) ? await verifyBrowserApi({ api, url, token: bootstrap.token, directory: data, appRoot, contained: true }) : null;
|
||||
console.log((await stop()).trim()); started = false;
|
||||
await until(async () => { try { await fetch(url + '/api/health'); return false; } catch { return true; } });
|
||||
const result = { success: true, date: new Date().toISOString(), appRoot, version: manifest.version, nodeVersion: manifest.nodeVersion,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
@@ -7,7 +8,7 @@ import { Store } from '../src/server/store.mjs';
|
||||
import { launchWorkbench } from '../tests/helpers.mjs';
|
||||
import { createResearchSamples, ownedAsar } from './research-fixtures.mjs';
|
||||
|
||||
const proofPrefix = JSON.parse(fs.readFileSync('package.json', 'utf8')).version === '1.10.0' ? 'browser-engine' : JSON.parse(fs.readFileSync('package.json', 'utf8')).version === '1.9.0' ? 'external-engines' : 'rea-engine';
|
||||
const proofPrefix = releaseEvidencePrefix(JSON.parse(fs.readFileSync('package.json', 'utf8')).version);
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-research-ui.mjs — independent owned projects, real REA static operations and a declared stdio stop fixture'); process.exit(0); }
|
||||
const root = path.resolve('.qa', 'research-ui-' + randomUUID()), data = path.join(root, 'data'), projectRoot = path.join(root, 'project');
|
||||
|
||||
@@ -42,7 +42,7 @@ try {
|
||||
assert.deepEqual(actualFiles.slice().sort(), Object.keys(expected).sort());
|
||||
for (const [relative, sha256] of Object.entries(expected)) assert.equal(digest(path.join(sourceRoot, relative)), sha256, relative);
|
||||
let actualRea = null;
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1'].includes(evidence.version)) {
|
||||
if (['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(evidence.version)) {
|
||||
stage = 'contained REA';
|
||||
const reaOutput = await run(['scripts/verify-rea.mjs', '--app-root', sourceRoot, '--contained'], 'rea.log');
|
||||
actualRea = JSON.parse(reaOutput.trim().split('\n').at(-1));
|
||||
@@ -50,7 +50,7 @@ try {
|
||||
assert.ok(actualRea.results.every(item => item.rootExited === true));
|
||||
}
|
||||
let actualEngine = null;
|
||||
if (['1.9.0', '1.10.0', '1.10.1'].includes(evidence.version) && process.argv.includes('--native-engines')) {
|
||||
if (['1.9.0', '1.10.0', '1.10.1', '1.11.0'].includes(evidence.version) && process.argv.includes('--native-engines')) {
|
||||
stage = 'native DeepSeek SDK';
|
||||
const output = await run(['scripts/verify-engines.mjs'], 'engines.log');
|
||||
actualEngine = JSON.parse(output.trim().split('\n').at(-1));
|
||||
@@ -58,7 +58,7 @@ try {
|
||||
assert.equal(actualEngine.modelRequests, 0); assert.equal(actualEngine.rootExited, true); assert.equal(actualEngine.nativeMethods.length, 6);
|
||||
}
|
||||
let actualBrowser = null;
|
||||
if (['1.10.0', '1.10.1'].includes(evidence.version)) {
|
||||
if (['1.10.0', '1.10.1', '1.11.0'].includes(evidence.version)) {
|
||||
stage = 'contained browser runtime';
|
||||
const output = await run(['scripts/verify-browser.mjs', '--app-root', sourceRoot, '--contained'], 'browser.log');
|
||||
actualBrowser = JSON.parse(output.trim().split('\n').at(-1));
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench } from '../tests/helpers.mjs';
|
||||
import { beginPlanTask, completeRead } from '../tests/fixtures/task-plan-fixture.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-task-plans-ui.mjs — independent task dependencies, original evidence, stale checks, sealed revisions, desktop/mobile and late responses'); process.exit(0); }
|
||||
const directory = path.resolve('.qa', 'task-plan-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t);
|
||||
fs.mkdirSync(path.join(f.projectRoot, 'test'));
|
||||
fs.writeFileSync(path.join(f.projectRoot, 'package.json'), JSON.stringify({ name: 'task-plan-ui-fixture', type: 'module', scripts: { test: 'node --test test/*.test.mjs', build: 'node --check src.mjs' } }));
|
||||
fs.writeFileSync(path.join(f.projectRoot, 'src.mjs'), 'export const sum = (a, b) => a + b;\n');
|
||||
fs.writeFileSync(path.join(f.projectRoot, 'test/sum.test.mjs'), "import test from 'node:test'; import assert from 'node:assert/strict'; import { sum } from '../src.mjs'; test('addition', () => assert.equal(sum(2, 3), 5)); test('negative', () => assert.equal(sum(-1, 2), 1));\n");
|
||||
beginPlanTask(f, '读取工程并核验测试、构建和移动页面'); completeRead(f);
|
||||
f.session.title = '有依赖的实际任务计划'; f.store.save(f.session); f.store.close();
|
||||
let host = await launchWorkbench(t, f.data), deliberateNavigation = false, releaseHeld = null;
|
||||
assert.notEqual(new URL(host.url).port, '4318'); assert.equal(host.bootstrap.config.hasKey, false);
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const passed = [], pageErrors = [], requestFailures = [], screenshots = [];
|
||||
const route = '/sessions/' + f.session.id, state = () => host.api(route), panel = () => page.getByLabel('执行计划', { exact: true });
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => { if (!deliberateNavigation) requestFailures.push(request.url() + ': ' + request.failure()?.errorText); });
|
||||
const mark = text => { passed.push(text); console.log('PASS ' + text); };
|
||||
const screenshot = async name => { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push(file); };
|
||||
async function selectTask() {
|
||||
if (!await page.getByLabel('当前项目').isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await page.getByLabel('当前项目').selectOption(f.project.id);
|
||||
await page.locator('.session-list button').filter({ hasText: f.session.title }).click();
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||||
await expect(panel().getByRole('button', { name: /建立计划|补充步骤/ })).toBeEnabled();
|
||||
}
|
||||
const steps = () => panel().locator('.plan-step');
|
||||
const editor = () => page.getByRole('dialog', { name: '编辑执行计划', exact: true });
|
||||
async function expandEvidence(index) {
|
||||
const details = steps().nth(index).locator('.plan-evidence');
|
||||
if (!await details.evaluate(node => node.open)) await details.locator('summary').click();
|
||||
}
|
||||
async function verify(index) { await expandEvidence(index); await steps().nth(index).getByRole('button', { name: '核验此步骤', exact: true }).click(); }
|
||||
async function newSession() {
|
||||
if (!await page.getByRole('button', { name: '新建任务', exact: true }).first().isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await page.getByRole('button', { name: '新建任务', exact: true }).first().click();
|
||||
await expect.poll(() => page.evaluate(() => localStorage.getItem('awb-session'))).not.toBe(f.session.id);
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); await selectTask();
|
||||
await panel().getByRole('button', { name: '建立计划', exact: true }).click();
|
||||
await editor().getByLabel('步骤1名称', { exact: true }).fill('读取原工程依据');
|
||||
await editor().getByLabel('步骤1通过条件', { exact: true }).fill('实际读取本工程,保留原调用和结果');
|
||||
await editor().getByLabel('识别工程', { exact: true }).uncheck(); await editor().getByLabel('读取文件', { exact: true }).check();
|
||||
await editor().getByRole('button', { name: '追加步骤', exact: true }).click();
|
||||
await editor().getByLabel('步骤2名称', { exact: true }).fill('测试和构建实际通过');
|
||||
await editor().getByLabel('步骤2通过条件', { exact: true }).fill('当前源码两项测试及构建均实际通过');
|
||||
await editor().getByLabel('步骤2类别', { exact: true }).selectOption('checks');
|
||||
await editor().getByLabel('读取原工程依据', { exact: true }).check();
|
||||
await screenshot('task-plan-desktop-editor');
|
||||
await editor().getByRole('button', { name: '保存执行计划', exact: true }).click();
|
||||
await expect(steps()).toHaveCount(2); await expect(steps().nth(1)).toContainText('先核验:读取原工程依据');
|
||||
await expandEvidence(1); await expect(steps().nth(1).getByRole('button', { name: '核验此步骤', exact: true })).toBeDisabled();
|
||||
const firstPlan = await host.api(route + '/task-plan'); assert.equal(firstPlan.revision, 1); assert.equal(firstPlan.all_verified, false);
|
||||
assert.equal((await state()).execution.progress.totalTools, 1);
|
||||
mark('structured editor seals two real steps and their dependency without running tools');
|
||||
|
||||
await verify(0); await expect(steps().nth(0).locator('.status')).toHaveText('已核验');
|
||||
assert.equal((await state()).execution.progress.totalTools, 1);
|
||||
await expandEvidence(0); await steps().nth(0).getByRole('button', { name: '查看原核验记录', exact: true }).click();
|
||||
const artifact = page.getByRole('dialog', { name: '执行结果工件', exact: true });
|
||||
await expect(artifact).toContainText('task-step-proof'); await expect(artifact).toContainText('SHA256');
|
||||
await page.keyboard.press('Escape'); await expect(artifact).toHaveCount(0); await expect(panel()).toBeVisible();
|
||||
mark('original owned read verifies the prerequisite and opens its hashed proof without replay');
|
||||
|
||||
const planned = await host.api(route + '/check-plans', {}), checkId = planned.execution.plans.at(-1).id;
|
||||
await host.api(route + '/check-jobs', { planId: checkId });
|
||||
await expect.poll(async () => (await state()).execution.checks.at(-1)?.status, { timeout: 15000 }).toBe('passed');
|
||||
await panel().getByRole('button', { name: '核对当前计划', exact: true }).click();
|
||||
await expect(steps().nth(1).getByRole('button', { name: '核验此步骤', exact: true })).toBeEnabled();
|
||||
await verify(1); await expect(steps().nth(1).locator('.status')).toHaveText('已核验');
|
||||
assert.equal((await host.api(route + '/task-plan')).all_verified, true);
|
||||
assert.equal((await state()).execution.checks.at(-1).results[0].tests.total, 2);
|
||||
await panel().evaluate(node => node.scrollIntoView({ block: 'start' }));
|
||||
await screenshot('task-plan-desktop-verified');
|
||||
mark('real two-test and build job supplies the current-source proof and completes the sealed plan');
|
||||
|
||||
fs.appendFileSync(path.join(f.projectRoot, 'src.mjs'), '\n// Outside edit after the verified check.\n');
|
||||
await panel().getByRole('button', { name: '核对当前计划', exact: true }).click();
|
||||
await expect(steps().nth(1).locator('.status')).toHaveText('需重新检查');
|
||||
await verify(1); await expect(steps().nth(1)).toContainText('最近一次核验未通过');
|
||||
const stale = await host.api(route + '/task-plan'); assert.equal(stale.all_verified, false);
|
||||
assert.equal((await state()).execution.taskProofs.at(-1).status, 'rejected');
|
||||
const reported = await host.api(route + '/delivery-reports', {});
|
||||
const reportArtifact = reported.execution.artifacts.findLast(value => value.kind === 'delivery-report');
|
||||
const reportView = await host.api('/artifacts/' + reportArtifact.id + '?session=' + f.session.id + '&limit=16000');
|
||||
assert.equal(JSON.parse(reportView.content).status, 'needs_plan_verification');
|
||||
mark('outside source edits invalidate the current proof, retain the rejection and block a passed delivery report');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await expect(panel()).toBeVisible();
|
||||
await panel().getByRole('button', { name: '补充步骤', exact: true }).click();
|
||||
await expect(editor().getByLabel('步骤1通过条件', { exact: true })).toBeDisabled();
|
||||
await editor().getByRole('button', { name: '追加步骤', exact: true }).click();
|
||||
await editor().getByLabel('步骤3名称', { exact: true }).fill('手机页面操作与断言');
|
||||
await editor().getByLabel('步骤3通过条件', { exact: true }).fill('在390像素页面实际填写、点击并核对结果,保留原截图');
|
||||
await editor().getByLabel('步骤3类别', { exact: true }).selectOption('browser');
|
||||
await editor().getByLabel('步骤3页面宽度', { exact: true }).selectOption('390');
|
||||
await editor().getByLabel('填写', { exact: true }).check(); await editor().getByLabel('点击', { exact: true }).check();
|
||||
await editor().getByLabel('测试和构建实际通过', { exact: true }).check();
|
||||
await screenshot('task-plan-mobile-editor');
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
await editor().getByRole('button', { name: '保存执行计划', exact: true }).click();
|
||||
await expect(steps()).toHaveCount(3); await expect(steps().nth(2)).toContainText('390px');
|
||||
await expandEvidence(2); await expect(steps().nth(2).getByRole('button', { name: '核验此步骤', exact: true })).toBeDisabled();
|
||||
const revised = await host.api(route + '/task-plan'); assert.equal(revised.revision, 2); assert.equal(revised.all_verified, false);
|
||||
assert.deepEqual(revised.steps.slice(0, 2).map(({ key, title, criterion }) => ({ key, title, criterion })), firstPlan.steps.map(({ key, title, criterion }) => ({ key, title, criterion })));
|
||||
await verify(0); await expect(steps().nth(0).locator('.status')).toHaveText('已核验');
|
||||
await panel().getByRole('button', { name: '查看原计划', exact: true }).click(); await expect(artifact).toContainText('task-plan-contract');
|
||||
await screenshot('task-plan-mobile-artifact'); await page.keyboard.press('Escape'); await expect(panel()).toBeVisible();
|
||||
await screenshot('task-plan-mobile-steps');
|
||||
await panel().evaluate(node => node.scrollIntoView({ block: 'start' })); await screenshot('task-plan-mobile-overview');
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
mark('390px immutable revision adds a mobile step, preserves old criteria and keeps evidence dialogs operable');
|
||||
|
||||
await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
let held; const gate = new Promise(resolve => { releaseHeld = resolve; });
|
||||
const matcher = '**/task-plan?taskId=*';
|
||||
await page.route(matcher, pending => { held = pending; return gate; });
|
||||
await panel().getByRole('button', { name: '核对当前计划', exact: true }).click(); await expect.poll(() => !!held).toBeTruthy();
|
||||
await newSession();
|
||||
await held.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: '原任务计划迟到错误' }) }); releaseHeld(); releaseHeld = null; await page.unroute(matcher);
|
||||
await expect(page.getByText('原任务计划迟到错误', { exact: false })).toHaveCount(0);
|
||||
mark('a delayed plan-read failure stays with the original task after selection changes');
|
||||
|
||||
await selectTask(); await panel().getByRole('button', { name: '补充步骤', exact: true }).click();
|
||||
await editor().getByRole('button', { name: '追加步骤', exact: true }).click();
|
||||
await editor().getByLabel('步骤4名称', { exact: true }).fill('追加原工程读取'); await editor().getByLabel('步骤4通过条件', { exact: true }).fill('保存到原任务的第四步骤');
|
||||
let heldSave, savedReply; const saveGate = new Promise(resolve => { releaseHeld = resolve; });
|
||||
const saveMatcher = '**/sessions/' + f.session.id + '/task-plans';
|
||||
await page.route(saveMatcher, async pending => { heldSave = pending; const response = await pending.fetch(); savedReply = await response.json(); return saveGate; });
|
||||
await editor().getByRole('button', { name: '保存执行计划', exact: true }).click(); await expect.poll(() => !!savedReply).toBeTruthy();
|
||||
// Controlled lifecycle fault: dispatch the real React tab action while the
|
||||
// save dialog blocks pointer input. This is not a claim that its backdrop
|
||||
// permits user clicks; it verifies response handling after component unmount.
|
||||
await page.locator('.inspector-tabs').getByRole('button', { name: '轨迹', exact: true }).dispatchEvent('click');
|
||||
await heldSave.fulfill({ status: 201, contentType: 'application/json', body: JSON.stringify(savedReply) }); releaseHeld(); releaseHeld = null; await page.unroute(saveMatcher);
|
||||
await expect(page.getByRole('dialog', { name: '编辑执行计划', exact: true })).toHaveCount(0);
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await expect(steps()).toHaveCount(4);
|
||||
assert.equal((await host.api(route + '/task-plan')).revision, 3);
|
||||
mark('controlled tab unmount during a delayed successful save retains the original task result without restoring the closed editor');
|
||||
|
||||
const before = await host.api(route + '/export'); deliberateNavigation = true; await host.close(); host = await launchWorkbench(t, f.data);
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); deliberateNavigation = false; await selectTask();
|
||||
await expect(steps()).toHaveCount(4); const after = await host.api(route + '/export');
|
||||
assert.deepEqual(after.execution.taskPlans, before.execution.taskPlans); assert.deepEqual(after.execution.taskProofs, before.execution.taskProofs);
|
||||
assert.deepEqual(after.session.providerMessages, before.session.providerMessages); assert.equal(after.execution.progress.totalTools, 1);
|
||||
assert.equal(after.execution.task.usage.requests, 0); assert.equal(after.execution.checks.length, 1);
|
||||
mark('real host restart retains all plan revisions, accepted/rejected proofs and the original operations without replay');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
result = { success: true, version: host.bootstrap.version, testedAt: new Date().toISOString(), directory, passed, pageErrors, requestFailures, screenshots,
|
||||
taskId: f.session.taskId, sessionId: f.session.id, planRevisions: after.execution.taskPlans.length, proofs: after.execution.taskProofs.length,
|
||||
scope: 'Independent real Edge desktop/390px Workbench UI, actual npm checks and owned read fixture. No real model credentials or production service access; semantic project acceptance is separate.' };
|
||||
} catch (error) {
|
||||
await screenshot('task-plan-ui-failure').catch(() => {}); fs.writeFileSync(path.join(directory, 'failure-dom.html'), await page.content().catch(() => ''));
|
||||
result = { success: false, version: host.bootstrap.version, testedAt: new Date().toISOString(), directory, passed, pageErrors, requestFailures, screenshots, error: error.stack }; process.exitCode = 1;
|
||||
} finally { releaseHeld?.(); deliberateNavigation = true; await browser.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync('docs/task-plans-task-ui-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, groups: passed.length, directory, error: result.error }));
|
||||
@@ -45,4 +45,5 @@ fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, nul
|
||||
if (version === '1.8.0') fs.writeFileSync('docs/rea-engine-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
if (version === '1.9.0') fs.writeFileSync('docs/external-engines-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
if (version === '1.10.0') fs.writeFileSync('docs/browser-engine-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
if (version === '1.11.0') fs.writeFileSync('docs/task-plans-tests-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version, tests: result.tests, inputFiles: result.inputFiles, directory, error: result.error, tail: result.tail }, null, 2));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
@@ -12,7 +13,7 @@ import { captureWorkspaceSource, assertWorkspaceSource } from '../tests/fixtures
|
||||
// Every host/project is synthetic and independently owned. Never uses 4318.
|
||||
const root = path.resolve('.qa', 'workspace-ui-' + randomUUID()), data = path.join(root, 'data');
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
||||
const proofPrefix = version === '1.6.0' ? 'workspaces' : version === '1.6.1' ? 'working-context-workspaces' : version === '1.6.2' ? 'working-retention-workspaces' : version === '1.7.0' ? 'python-engine-workspaces' : version === '1.10.0' ? 'browser-engine-workspaces' : version === '1.9.0' ? 'external-engines-workspaces' : 'rea-engine-workspaces';
|
||||
const proofPrefix = version === '1.6.0' ? 'workspaces' : releaseEvidencePrefix(version) + '-workspaces';
|
||||
fs.mkdirSync(root, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const store = new Store(data), cases = {}, passed = [], pageErrors = [], requestFailures = [], screenshots = [], createPosts = [];
|
||||
for (const name of ['normal', 'mobile', 'recovery', 'late-success', 'late-failure']) {
|
||||
|
||||
Reference in New Issue
Block a user