Files
Agent-Workbench/scripts/verify-live-task-plans.mjs

130 lines
13 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { gitEnvironment } from '../src/core/git-process.mjs';
import { APP_VERSION } from '../src/server/diagnostics.mjs';
if (process.argv.includes('--help')) { console.log('node scripts/verify-live-task-plans.mjs — resume the prior page-only model Task with two sealed technical steps; reuse its verified desktop result and let the original Provider correct/verify mobile'); process.exit(0); }
const prior = JSON.parse(fs.readFileSync('docs/browser-engine-live-model-validation.json', 'utf8'));
const engineering = JSON.parse(fs.readFileSync('docs/live-react-engineering-validation.json', 'utf8'));
const url = 'http://127.0.0.1:4318', directory = path.resolve('.qa', 'live-task-plans-' + randomUUID()); fs.mkdirSync(directory);
const sessionId = prior.sessionId, taskId = prior.taskId, root = engineering.workspaceRoot;
const sha = value => createHash('sha256').update(Buffer.isBuffer(value) || typeof value === 'string' ? value : JSON.stringify(value)).digest('hex');
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', root, ...args], { env: gitEnvironment(), windowsHide: true, encoding: 'utf8' }).trim();
function sourceFiles(relative = '') {
return fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
if (entry.name.startsWith('.') || ['node_modules', 'dist', 'build', 'out'].includes(entry.name)) return [];
assert.equal(entry.isSymbolicLink(), false); const file = path.join(relative, entry.name);
return entry.isDirectory() ? sourceFiles(file) : [{ path: file, sha256: sha(fs.readFileSync(path.join(root, file))) }];
});
}
let token, current, baseline, result, beforeUsage, initialModelRequests = 0, modelStarted = false;
const decisions = [], continuations = [], verifierProofs = [], startedAt = Date.now();
async function api(route, input) {
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { Connection: 'close', 'Content-Type': 'application/json', ...(token ? { 'X-Workbench-Token': token } : {}) },
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
const value = await response.json(); if (!response.ok) throw new Error(value.error ?? 'HTTP ' + response.status); return value;
}
const sessionRoute = '/sessions/' + sessionId;
async function exportOriginal(job) {
let text = '', offset = 0, page;
do { page = await api(sessionRoute + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=16000'); text += page.content; offset += page.content.length; } while (page.truncated);
assert.equal(sha(text), job.resultSha256); const original = JSON.parse(text); assert.equal(original.taskId, taskId);
fs.writeFileSync(path.join(directory, 'original-' + job.id + '.json'), text);
for (const image of job.screenshots) {
const response = await fetch(url + '/api' + sessionRoute + '/browser-jobs/' + job.id + '/images/' + image.artifactId, { headers: { 'X-Workbench-Token': token }, signal: AbortSignal.timeout(15000) });
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), image.sha256);
fs.writeFileSync(path.join(directory, job.id + '-' + image.label + '.png'), bytes);
}
return original;
}
try {
assert.equal(prior.success, false); assert.equal(prior.version, '1.10.1'); assert.equal(engineering.success, true);
const before = await api('/bootstrap'); token = before.token; assert.equal(before.version, APP_VERSION); assert.equal(APP_VERSION, '1.11.0');
assert.equal(before.config.mode, 'compatible'); assert.equal(before.config.hasKey, true);
const project = before.projects.find(value => value.id === engineering.projectId); assert.ok(project); assert.equal(project.root, root);
assert.ok(!before.sessions.some(value => value.projectId === project.id && ['running', 'awaiting_approval', 'stopping'].includes(value.status)));
current = await api(sessionRoute); assert.equal(current.execution.task.id, taskId); assert.equal(current.projectId, project.id);
assert.ok(['paused', 'failed', 'cancelled'].includes(current.status), 'Only an unfinished original Task may be resumed.');
assert.equal(current.changes.length, 0); beforeUsage = structuredClone(current.execution.task.usage); initialModelRequests = beforeUsage.requests;
const originalExport = await api(sessionRoute + '/export');
const oldEngineering = await api('/sessions/' + engineering.sessionId + '/export'); delete oldEngineering.exportedAt;
baseline = { source: sourceFiles(), head: git(['rev-parse', 'HEAD']), index: sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))),
originalMessages: originalExport.session.providerMessages, originalJobs: originalExport.execution.browserJobs,
engineering: sha(canonical(oldEngineering)), config: sha(canonical(before.config)) };
fs.writeFileSync(path.join(directory, 'baseline.json'), JSON.stringify(baseline, null, 2) + '\n');
const previousPlan = await api(sessionRoute + '/task-plan');
const saved = await api(sessionRoute + '/task-plans', { taskId, runId: current.execution.run.id, expected_revision: previousPlan.revision,
steps: [
{ key: 'desktop', title: '桌面搜索、筛选与单任务切换', criterion: '1280px实际填写、筛选和点击,原断言通过,完成2项且待办工时5小时', kind: 'browser', viewport_width: 1280, actions: ['fill', 'select', 'click', 'assert_count'], depends_on: [] },
{ key: 'mobile', title: '手机布局与控件实际操作', criterion: '390px实际点击并核对可见性及数量,原截图无横向溢出', kind: 'browser', viewport_width: 390, actions: ['click', 'assert_visible', 'assert_count'], depends_on: ['desktop'] },
] });
current = saved.session;
const desktop = current.execution.browserJobs.findLast(job => job.status === 'passed' && current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 1280);
assert.ok(desktop, 'The original desktop pass must be present; this verifier does not regenerate it.');
const proof = await api(sessionRoute + '/task-proofs', { taskId, runId: current.execution.run.id, plan_id: saved.plan.plan_id,
expected_revision: saved.plan.revision, step_key: 'desktop', evidence_id: desktop.id });
assert.equal(proof.verification.status, 'completed'); verifierProofs.push({ step: 'desktop', evidenceId: desktop.id, proofId: proof.verification.proof_id, originalExecutionReplayed: false });
await api(sessionRoute + '/resume', {}); modelStarted = true;
const decided = new Set();
while (Date.now() - startedAt < 210000) {
current = await api(sessionRoute); assert.equal(current.execution.task.id, taskId); assert.equal(current.changes.length, 0);
if (current.status === 'awaiting_approval') {
const approval = current.approvals.find(value => value.status === 'pending' && !decided.has(value.id));
if (approval) {
assert.ok(approval.browserPlanId, 'Only the original page-only scope may execute.');
const plan = current.execution.browserPlans.find(value => value.id === approval.browserPlanId);
assert.equal(plan.taskId, taskId); assert.equal(plan.projectId, project.id); assert.equal(plan.build.directory, 'dist'); assert.equal(plan.path, '/');
assert.ok([1280, 390].includes(plan.viewport.width)); assert.ok(plan.steps.length <= 20);
decided.add(approval.id); decisions.push({ id: approval.id, planId: plan.id, fingerprint: plan.fingerprint, viewport: plan.viewport, steps: plan.steps.length });
await api(sessionRoute + '/approvals/' + approval.id, { allow: true });
}
} else if (['completed', 'paused'].includes(current.status)) {
const view = await api(sessionRoute + '/task-plan');
if (current.status === 'completed' && view.all_verified) break;
const reason = current.execution.run.stopReason;
if (current.status !== 'paused' || continuations.length >= 2 || !['step_limit', 'task_plan_incomplete'].includes(reason)) throw new Error('Actual model task remains ' + current.status + '/' + reason);
continuations.push({ previousRunId: current.execution.run.id, reason, at: new Date().toISOString() }); await api(sessionRoute + '/resume', {});
} else if (['failed', 'cancelled'].includes(current.status)) throw new Error(current.failure?.message ?? current.status);
await new Promise(resolve => setTimeout(resolve, 75));
}
assert.equal(current.status, 'completed', 'Actual model did not finish within the original 210-second acceptance bound.');
const view = await api(sessionRoute + '/task-plan'); assert.equal(view.all_verified, true); assert.equal(view.verified_steps, 2);
const records = [];
for (const step of view.steps) {
const job = current.execution.browserJobs.find(value => value.id === step.evidence.id); assert.ok(job); assert.equal(job.cleanup.exited, true);
records.push({ job, original: await exportOriginal(job), viewport: current.execution.browserPlans.find(value => value.id === job.planId).viewport });
}
const desktopRecord = records.find(value => value.viewport.width === 1280), mobile = records.find(value => value.viewport.width === 390);
assert.match(desktopRecord.original.pages.at(-1).text, /完成任务\s*2/); assert.match(desktopRecord.original.pages.at(-1).text, /待办工时\s*5\s*小时/);
assert.ok(mobile.original.pages.at(-1).viewport.scrollWidth <= 390);
assert.deepEqual(sourceFiles(), baseline.source); assert.equal(git(['rev-parse', 'HEAD']), baseline.head);
assert.equal(sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))), baseline.index);
const afterEngineering = await api('/sessions/' + engineering.sessionId + '/export'); delete afterEngineering.exportedAt;
assert.equal(sha(canonical(afterEngineering)), baseline.engineering); assert.equal(sha(canonical((await api('/bootstrap')).config)), baseline.config);
const exported = await api(sessionRoute + '/export');
assert.deepEqual(exported.session.providerMessages.slice(0, baseline.originalMessages.length), baseline.originalMessages);
assert.deepEqual(exported.execution.browserJobs.slice(0, baseline.originalJobs.length), baseline.originalJobs);
fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n');
result = { success: true, version: APP_VERSION, testedAt: new Date().toISOString(), directory, model: before.config.model, projectId: project.id, root, sessionId, taskId,
modelStatus: current.status, sourcePreserved: true, indexPreserved: true, originalEngineeringTaskPreserved: true, originalMessagesAndJobsPreserved: true,
beforeUsage, usage: current.execution.task.usage, newModelRequests: current.execution.task.usage.requests - initialModelRequests,
elapsedSeconds: (Date.now() - startedAt) / 1000, taskPlan: view, verifierProofs, decisions, continuations,
jobs: records.map(value => ({ id: value.job.id, status: value.job.status, viewport: value.viewport, assertions: value.job.assertions, resultSha256: value.job.resultSha256, cleanup: value.job.cleanup })),
scope: 'The verifier seals two technical steps and verifies the original desktop evidence once. The actual configured model resumes the same unfinished Task and must produce/verify the mobile result. This assisted page-only acceptance does not measure autonomous planning, code repair or Purge benefit.',
credentialsHandling: 'Original in-memory Provider is retained. No model key is read, copied, printed or saved.' };
} catch (error) {
if (token && modelStarted) { await api(sessionRoute + '/stop', {}).catch(() => {}); current = await api(sessionRoute).catch(() => current); }
result = { success: false, version: APP_VERSION, testedAt: new Date().toISOString(), directory, sessionId, taskId, modelStatus: current?.status,
beforeUsage, usage: current?.execution?.task?.usage, decisions, verifierProofs, continuations, error: error.stack }; process.exitCode = 1;
if (token) { const exported = await api(sessionRoute + '/export').catch(() => null); if (exported) fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n'); }
}
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
fs.writeFileSync('docs/task-plans-live-model-validation.json', JSON.stringify(result, null, 2) + '\n');
console.log(JSON.stringify({ success: result.success, sessionId, taskId, modelStatus: result.modelStatus, newModelRequests: result.newModelRequests,
elapsedSeconds: result.elapsedSeconds, usage: result.usage, jobs: result.jobs, error: result.error, directory }));