test(desktop): verify engineering delivery and native recovery

This commit is contained in:
Agent Workbench Dev
2026-10-31 11:31:00 +08:00
parent f7a5969fa4
commit b77783ba67
79 changed files with 74078 additions and 32 deletions
+24 -12
View File
@@ -12,15 +12,17 @@ import { originalEngineRun, verifyEngineModelInput } from '../src/shared/contrac
import { resolvePurgeSource, assertPurgeModelInput, nativeSystemTexts, PURGE_DEFAULT_PROMPT } from '../src/core/engines/purge-strategy.mjs';
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
import { desktopProjectQa } from './desktop-project-qa.mjs';
const args = process.argv.slice(2);
assert.ok(args.every(value => ['--help', '--purge-comparison'].includes(value)), 'Unknown option; use --help.');
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop'].includes(value)), 'Unknown option; use --help.');
if (args.includes('--help')) {
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. Comparison verifies eight independent Node projects: two Purge versions, two repetitions, enabled/disabled. Set AGENT_WORKBENCH_DEEPSEEK_ROOT; default also requires AGENT_WORKBENCH_TEST_PYTHON, comparison requires AGENT_WORKBENCH_PURGE_178_ROOT / AGENT_WORKBENCH_PURGE_181_ROOT. No configured-model requests.'); process.exit(0);
}
const comparisonMode = args.includes('--purge-comparison');
const comparisonMode = args.includes('--purge-comparison'), desktopMode = args.includes('--desktop');
assert.ok(!comparisonMode || !desktopMode, 'Desktop engineering and prompt comparison are separate acceptance runs.');
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
const directory = path.resolve('.qa', (comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
const directory = path.resolve('.qa', (desktopMode ? 'desktop-projects-' : comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT);
assert.equal(installation.controlledStatus, 'available', installation.message);
@@ -38,7 +40,7 @@ function inputs() {
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file);
} };
for (const root of ['src', 'scripts', 'tests', 'benchmarks', 'dist-client', 'dist-server']) walk(root);
for (const root of ['src', 'scripts', 'tests', 'benchmarks', 'dist-client', 'dist-server', ...(desktopMode ? ['dist-desktop'] : [])]) walk(root);
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
}
const frozen = inputs(), mark = message => { cases.push(message); console.log('PASS ' + message); };
@@ -69,7 +71,7 @@ function specification(kind, root) {
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
const startedAt = Date.now(), label = sampleName;
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
fs.mkdirSync(sample); let runtime, group, result; const reviews = [], modelRequests = [];
fs.mkdirSync(sample); let runtime, group, result, desktopQa, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
try {
const baseline = kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
@@ -95,6 +97,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
assertToolPairs(input.messages);
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
if (desktopQa && !cycledRequest) { cycledRequest = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-owned-model-request'); }
const checks = [...f.store.core.check_job.values()].filter(job => job.taskId === worker.taskId), job = checks.at(-1);
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === worker.taskId).at(-1);
const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Fixed local acceptance repair; retain all tests and declarations.' }));
@@ -116,6 +119,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
const provider = new Provider(); provider.setKey('self-owned-controlled-model-fixture'); runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine-runs') });
runtime.checkRunner().pythonExecutable = python; const manager = runtime.workerManager();
if (desktopMode) desktopQa = await desktopProjectQa({ store: f.store, provider, runtime, sample, kind });
group = await manager.plan(f.session, { goal: 'Deliver ' + kind + ' with the pinned controlled Harness, retained failures, explicit review, parent validation and local Git.',
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256, ...(strategy ? { strategy } : {}) },
budget: { maxTokens: 200000, maxSteps: 64, maxToolCalls: 128, maxDurationMs: 600000 },
@@ -123,13 +127,14 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'ready', group.summary);
const child = group.children[0], childRoot = f.store.project(child.projectId).root;
assert.notEqual(childRoot, f.projectRoot); assert.deepEqual(ownerBytes(f), original); verify(childRoot); if (kind === 'react') install(childRoot, 'child');
if (desktopQa) await desktopQa.select(f.store.session(child.sessionId));
mark(label + ': pinned native engine and one owned Git worktree preserve parent staging and protected bytes');
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
const session = f.store.session(child.sessionId), ids = session.changes.filter(change => change.status === 'pending').map(change => change.id);
if (ids.length) {
assert.equal(group.status, 'awaiting_review', group.summary); const set = await runtime.changeSetManager().prepare(session, ids);
assert.equal(group.status, 'awaiting_review', group.summary); const set = desktopQa ? await desktopQa.review(session, ids) : await runtime.changeSetManager().prepare(session, ids);
reviews.push({ operation: 'review child batch', id: set.id, fingerprint: set.fingerprint, paths: set.files.map(file => file.path) });
await (await runtime.changeSetManager().start(session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary);
if (!desktopQa) await (await runtime.changeSetManager().start(session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary);
}
const started = manager.start(f.session, group.id, group.fingerprint); let done = false, failure;
started.promise.then(() => { done = true; }, error => { failure = error; done = true; });
@@ -138,7 +143,9 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
assert.equal(f.store.core.tool.get(approval.invocationId).tool, 'run_checks');
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === child.taskId).at(-1);
assert.deepEqual(plan.checks.map(check => check.name), spec.checks);
reviews.push({ operation: 'approve original check', id: approval.id, taskId: child.taskId, planId: plan.id, contractSha256: plan.contractSha256 }); runtime.decide(session.id, approval.id, true);
reviews.push({ operation: 'approve original check', id: approval.id, taskId: child.taskId, planId: plan.id, contractSha256: plan.contractSha256 });
if (desktopQa) { if (!cycledApproval) { cycledApproval = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-original-approval'); } await desktopQa.approve(session, approval); }
else runtime.decide(session.id, approval.id, true);
}
await new Promise(resolve => setTimeout(resolve, 30));
}
@@ -167,10 +174,13 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
const collection = await manager.readCollection(f.session, group); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'merge_prepared', group.summary);
const merge = f.store.core.change_set.get(group.merge.changeSetId); reviews.push({ operation: 'review parent batch', id: merge.id, fingerprint: merge.fingerprint });
await (await runtime.changeSetManager().start(f.session, merge.id, merge.fingerprint)).promise; assert.equal(merge.status, 'completed', merge.summary);
if (desktopQa) assert.equal((await desktopQa.review(f.session, merge.files.map(file => file.changeId))).id, merge.id);
else await (await runtime.changeSetManager().start(f.session, merge.id, merge.fingerprint)).promise;
assert.equal(merge.status, 'completed', merge.summary);
const checkPlan = await runtime.checkRunner().plan(f.session, spec.checks), check = await (await runtime.checkRunner().start(f.session, checkPlan.id)).promise;
assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify(f.projectRoot);
mark(label + ': separately reviewed parent merge passes all ' + spec.tests + ' functional tests and actual build/compile');
if (desktopQa) await desktopQa.capture(f.session, 'parent-checks');
const browserJobs = [];
if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
const steps = [ { action: 'assert_count', selector: '.task-list article', count: 3 },
@@ -191,6 +201,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n');
assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable);
mark(label + ': local Git contains only reviewed functional changes and preserves original staged/unstaged owner bytes');
if (desktopQa) { await desktopQa.capture(f.session, 'git-completed'); assert.equal(cycledRequest, true); assert.equal(cycledApproval, true); mark(label + ': real Electron reviews/approvals and window cycles retain Task/Run/eventSeq/results at 1440px and 390px'); }
const requests = engineRuns.flatMap(job => job.requests), metrics = { durationMs: Date.now() - startedAt, baselineTree,
contextLimit: group.config.contextLimit, budget: group.budget, model: group.model.model, tests: spec.tests,
engineeringFailures: childChecks.filter(job => job.status === 'failed').length,
@@ -202,10 +213,11 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
responseSequence: modelRequests.map(({ phase, responsePlan }) => ({ phase, responsePlan })) };
result = { success: true, scope: 'actual-native-Harness-with-owned-HTTP-model-fixture', version, kind, sampleName, comparison, metrics, parentTaskId: group.taskId, groupId: group.id,
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
writable, reviews, modelRequests, protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
writable, reviews, modelRequests, ...(desktopQa ? { desktop: { events: desktopQa.events, screenshots: desktopQa.screenshots, scope: desktopQa.scope, errors: desktopQa.errors } } : {}), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
return result;
} catch (error) { result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
finally {
await desktopQa?.close();
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
fs.cpSync(f.root, path.join(sample, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
for (const session of f.store.sessions.values()) fs.writeFileSync(path.join(sample, 'session-' + session.id + '.json'), JSON.stringify({ project: f.store.project(session.projectId), session, execution: f.store.executionSnapshot(session) }, null, 2));
@@ -243,4 +255,4 @@ try {
report = { success: true, targetVersion: '1.14.0', testedVersion: version, directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
scope: 'Pinned actual Harness and workbench execution with an owned HTTP model fixture. Checks, approvals, failures, parent merges, Edge and Git are real. Model reasoning and Purge benefit remain separate.' };
} catch (error) { report = { success: false, targetVersion: '1.14.0', testedVersion: version, directory, installation, cases, results, inputs: frozen, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }