feat: add source-pinned default-off prompt experiments with native comparisons
This commit is contained in:
@@ -8,21 +8,29 @@ import { Provider } from '../src/server/provider.mjs';
|
||||
import { Runtime } from '../src/server/runtime.mjs';
|
||||
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
||||
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
||||
import { originalEngineRun } from '../src/shared/contracts/engine-execution.mjs';
|
||||
import { originalEngineRun, verifyEngineModelInput } from '../src/shared/contracts/engine-execution.mjs';
|
||||
import { resolvePurgeSource, assertPurgeModelInput, nativeSystemTexts, PURGE_DEFAULT_PROMPT } from '../src/core/engines/purge-strategy.mjs';
|
||||
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-controlled-projects.mjs — actual pinned Harness, owned HTTP model fixture, three real Node/React/Python worktrees, failing tests/repairs, approvals, parent validation, Edge and Git; no configured-model requests. Requires AGENT_WORKBENCH_DEEPSEEK_ROOT and AGENT_WORKBENCH_TEST_PYTHON.'); process.exit(0);
|
||||
const args = process.argv.slice(2);
|
||||
assert.ok(args.every(value => ['--help', '--purge-comparison'].includes(value)), 'Unknown option; use --help.');
|
||||
if (args.includes('--help')) {
|
||||
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. Comparison verifies eight independent Node projects: two Purge versions, two repetitions, enabled/disabled. Set AGENT_WORKBENCH_DEEPSEEK_ROOT; default also requires AGENT_WORKBENCH_TEST_PYTHON, comparison requires AGENT_WORKBENCH_PURGE_178_ROOT / AGENT_WORKBENCH_PURGE_181_ROOT. No configured-model requests.'); process.exit(0);
|
||||
}
|
||||
assert.equal(process.argv.length, 2);
|
||||
const comparisonMode = args.includes('--purge-comparison');
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
||||
const directory = path.resolve('.qa', 'controlled-projects-' + randomUUID()), results = [], cases = [];
|
||||
const directory = path.resolve('.qa', (comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT);
|
||||
assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
const python = process.env.AGENT_WORKBENCH_TEST_PYTHON;
|
||||
assert.ok(python && path.isAbsolute(python) && fs.existsSync(python), 'Provide the existing isolated Python 3.10+ interpreter.');
|
||||
if (!comparisonMode) assert.ok(python && path.isAbsolute(python) && fs.existsSync(python), 'Provide the existing isolated Python 3.10+ interpreter.');
|
||||
const sources = comparisonMode ? ['1.1.78', '1.1.81'].map((expected, index) => {
|
||||
const root = process.env[index === 0 ? 'AGENT_WORKBENCH_PURGE_178_ROOT' : 'AGENT_WORKBENCH_PURGE_181_ROOT'];
|
||||
assert.ok(root && path.isAbsolute(root), 'Provide both existing Purge source directories.');
|
||||
const source = resolvePurgeSource(process.cwd(), root); assert.equal(source.status, 'available', source.message); assert.equal(source.version, expected); return source;
|
||||
}) : [];
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
||||
function inputs() {
|
||||
@@ -58,8 +66,9 @@ function specification(kind, root) {
|
||||
'schedule.py': 'import re\nfrom datetime import date\n\ndef normalize_schedule(values):\n dates = []\n for value in values:\n if not isinstance(value, str) or not re.fullmatch(r"\\d{4}-\\d{2}-\\d{2}", value):\n raise ValueError("ISO calendar date required")\n dates.append(date.fromisoformat(value))\n return [value.isoformat() for value in sorted(set(dates))]\n',
|
||||
} };
|
||||
}
|
||||
async function runSample(kind) {
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, kind);
|
||||
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
|
||||
const startedAt = Date.now(), label = sampleName;
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
|
||||
fs.mkdirSync(sample); let runtime, group, result; const reviews = [], modelRequests = [];
|
||||
try {
|
||||
const baseline = kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
|
||||
@@ -68,6 +77,7 @@ async function runSample(kind) {
|
||||
write(f, 'owner-staged.txt', 'Owner baseline\n'); write(f, 'owner-unstaged.txt', 'Owner baseline\n');
|
||||
git(f, ['init', '--initial-branch=main']); git(f, ['config', 'core.autocrlf', 'false']); git(f, ['config', 'user.name', 'Workbench QA']); git(f, ['config', 'user.email', 'qa@workbench.invalid']);
|
||||
git(f, ['add', '--', '.']); git(f, ['commit', '--no-gpg-sign', '-m', 'Seal ' + kind + ' controlled-engine acceptance']);
|
||||
const baselineTree = git(f, ['rev-parse', 'HEAD^{tree}']).trim();
|
||||
write(f, 'owner-staged.txt', 'Owner staged change\n'); git(f, ['add', '--', 'owner-staged.txt']);
|
||||
write(f, 'owner-staged.txt', 'Owner working copy after staging\n'); write(f, 'owner-unstaged.txt', 'Owner unstaged change\n'); write(f, 'owner-untracked.txt', 'Owner untracked file\n');
|
||||
const spec = specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort();
|
||||
@@ -96,7 +106,9 @@ async function runSample(kind) {
|
||||
} else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, spec.tests);
|
||||
else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: spec.checks })];
|
||||
else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })];
|
||||
modelRequests.push({ requestBody: input, callIds: calls.map(call => call.id), phase });
|
||||
modelRequests.push({ requestBody: input, callIds: calls.map(call => call.id), phase,
|
||||
responsePlan: calls.map(call => ({ name: call.function.name, arguments: { ...JSON.parse(call.function.arguments),
|
||||
...(call.function.name === 'run_checks' ? { plan_id: 'owned-current-check-plan' } : {}) } })) });
|
||||
return { choices: [{ message: { role: 'assistant', content: calls.length ? '' : 'Owned native run complete; host proof is still required.', ...(calls.length ? { tool_calls: calls } : {}) },
|
||||
finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 123, completion_tokens: 45, total_tokens: 168 } };
|
||||
});
|
||||
@@ -105,13 +117,13 @@ async function runSample(kind) {
|
||||
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine-runs') });
|
||||
runtime.checkRunner().pythonExecutable = python; const manager = runtime.workerManager();
|
||||
group = await manager.plan(f.session, { goal: 'Deliver ' + kind + ' with the pinned controlled Harness, retained failures, explicit review, parent validation and local Git.',
|
||||
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256 },
|
||||
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256, ...(strategy ? { strategy } : {}) },
|
||||
budget: { maxTokens: 200000, maxSteps: 64, maxToolCalls: 128, maxDurationMs: 600000 },
|
||||
children: [{ label: 'engine-' + kind, goal: 'Repair and validate the complete ' + kind + ' functional suite without replacing tests.', paths: writable }] });
|
||||
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'ready', group.summary);
|
||||
const child = group.children[0], childRoot = f.store.project(child.projectId).root;
|
||||
assert.notEqual(childRoot, f.projectRoot); assert.deepEqual(ownerBytes(f), original); verify(childRoot); if (kind === 'react') install(childRoot, 'child');
|
||||
mark(kind + ': pinned native engine and one owned Git worktree preserve parent staging and protected bytes');
|
||||
mark(label + ': pinned native engine and one owned Git worktree preserve parent staging and protected bytes');
|
||||
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
|
||||
const session = f.store.session(child.sessionId), ids = session.changes.filter(change => change.status === 'pending').map(change => change.id);
|
||||
if (ids.length) {
|
||||
@@ -142,10 +154,15 @@ async function runSample(kind) {
|
||||
for (const job of engineRuns) {
|
||||
originalEngineRun(f.store, childSession, job); assert.equal(job.status, 'completed', job.error); assert.equal(job.native.cleanup.treeConfirmed, true);
|
||||
assert.equal(job.native.stderr, ''); assert.equal(job.native.activity.endEvents[0].data.reason.kind, 'completed');
|
||||
for (const request of job.requests) {
|
||||
const saved = verifyEngineModelInput(f.store, childSession, job, request);
|
||||
if (strategy) { assertPurgeModelInput(saved.native, job.installation.strategy); assert.equal(job.native.readiness.promptExperimentSha256, job.installation.strategy.sha256); }
|
||||
else assert.ok(nativeSystemTexts(saved.native).every(text => !text.includes('[Agent Workbench engineering prompt experiment]')));
|
||||
}
|
||||
}
|
||||
assert.equal(f.store.core.task.get(child.taskId).usage.requests, modelRequests.length);
|
||||
assert.equal(f.store.core.task.get(child.taskId).usage.totalTokens, modelRequests.length * 168);
|
||||
mark(kind + ': actual Harness uses owned model/tool bridges, retains failing tests, and completes reviewed repairs with one accounting entry per HTTP request');
|
||||
mark(label + ': actual Harness uses owned model/tool bridges, retains failing tests, and completes reviewed repairs with one accounting entry per HTTP request');
|
||||
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'collected', group.summary);
|
||||
const collection = await manager.readCollection(f.session, group); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
|
||||
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'merge_prepared', group.summary);
|
||||
@@ -153,7 +170,7 @@ async function runSample(kind) {
|
||||
await (await runtime.changeSetManager().start(f.session, merge.id, merge.fingerprint)).promise; assert.equal(merge.status, 'completed', merge.summary);
|
||||
const checkPlan = await runtime.checkRunner().plan(f.session, spec.checks), check = await (await runtime.checkRunner().start(f.session, checkPlan.id)).promise;
|
||||
assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify(f.projectRoot);
|
||||
mark(kind + ': separately reviewed parent merge passes all ' + spec.tests + ' functional tests and actual build/compile');
|
||||
mark(label + ': separately reviewed parent merge passes all ' + spec.tests + ' functional tests and actual build/compile');
|
||||
const browserJobs = [];
|
||||
if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
|
||||
const steps = [ { action: 'assert_count', selector: '.task-list article', count: 3 },
|
||||
@@ -173,12 +190,21 @@ async function runSample(kind) {
|
||||
assert.equal(delivery.status, 'completed', delivery.summary); await manager.finalize(f.session, group.id, group.fingerprint); assert.equal(group.status, 'completed');
|
||||
assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n');
|
||||
assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable);
|
||||
mark(kind + ': local Git contains only reviewed functional changes and preserves original staged/unstaged owner bytes');
|
||||
result = { success: true, scope: 'actual-native-Harness-with-owned-HTTP-model-fixture', version, kind, parentTaskId: group.taskId, groupId: group.id,
|
||||
mark(label + ': local Git contains only reviewed functional changes and preserves original staged/unstaged owner bytes');
|
||||
const requests = engineRuns.flatMap(job => job.requests), metrics = { durationMs: Date.now() - startedAt, baselineTree,
|
||||
contextLimit: group.config.contextLimit, budget: group.budget, model: group.model.model, tests: spec.tests,
|
||||
engineeringFailures: childChecks.filter(job => job.status === 'failed').length,
|
||||
hostApprovals: reviews.filter(review => review.operation === 'approve original check').length, reviews: reviews.length,
|
||||
requests: modelRequests.length, estimatedInputTokens: requests.reduce((sum, request) => sum + request.estimatedInputTokens, 0),
|
||||
requestBytes: requests.reduce((sum, request) => sum + request.requestBytes, 0),
|
||||
systemBytes: modelRequests.reduce((sum, request) => sum + Buffer.byteLength(request.requestBody.messages.find(message => message.role === 'system')?.content ?? ''), 0),
|
||||
reportedFixtureTokens: modelRequests.length * 168, unknownRequests: f.store.core.task.get(child.taskId).usage.unknownRequests,
|
||||
responseSequence: modelRequests.map(({ phase, responsePlan }) => ({ phase, responsePlan })) };
|
||||
result = { success: true, scope: 'actual-native-Harness-with-owned-HTTP-model-fixture', version, kind, sampleName, comparison, metrics, parentTaskId: group.taskId, groupId: group.id,
|
||||
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
|
||||
writable, reviews, modelRequests, protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
||||
return result;
|
||||
} catch (error) { result = { success: false, version, kind, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
} catch (error) { result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.cpSync(f.root, path.join(sample, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
@@ -189,8 +215,32 @@ async function runSample(kind) {
|
||||
}
|
||||
let report;
|
||||
try {
|
||||
for (const kind of ['node', 'react', 'python']) results.push(await runSample(kind)); assert.deepEqual(inputs(), frozen);
|
||||
report = { success: true, targetVersion: '1.14.0', testedVersion: version, directory, installation, cases, results, inputs: frozen, realModelRequests: 0,
|
||||
const comparisons = [];
|
||||
if (comparisonMode) {
|
||||
const prompt = PURGE_DEFAULT_PROMPT + '\n保留字面值 {{engineering_literal}},依据实际失败证据完成同题修复。';
|
||||
for (const source of sources) for (const repetition of [1, 2]) {
|
||||
const pair = [];
|
||||
for (const enabled of repetition === 1 ? [false, true] : [true, false]) {
|
||||
const comparison = { sourceVersion: source.version, sourceSha256: source.sourceSha256, repetition, enabled };
|
||||
const result = await runSample('node', { sampleName: source.version + '-r' + repetition + '-' + (enabled ? 'enabled' : 'disabled'), comparison,
|
||||
strategy: enabled ? { kind: 'purge-prompt-experiment', root: source.root, sourceSha256: source.sourceSha256, prompt } : null });
|
||||
results.push(result); pair.push(result);
|
||||
}
|
||||
const control = pair.find(result => !result.comparison.enabled), experiment = pair.find(result => result.comparison.enabled);
|
||||
for (const key of ['baselineTree', 'contextLimit', 'budget', 'model', 'tests', 'responseSequence', 'hostApprovals', 'engineeringFailures', 'reportedFixtureTokens', 'requests']) assert.deepEqual(experiment.metrics[key], control.metrics[key], 'Comparable original ' + key);
|
||||
assert.deepEqual(experiment.writable, control.writable); assert.deepEqual(experiment.protectedFiles, control.protectedFiles);
|
||||
assert.ok(experiment.metrics.systemBytes > control.metrics.systemBytes);
|
||||
comparisons.push({ sourceVersion: source.version, repetition, control: control.sampleName, experiment: experiment.sampleName,
|
||||
estimatedInputTokenDelta: experiment.metrics.estimatedInputTokens - control.metrics.estimatedInputTokens,
|
||||
requestByteDelta: experiment.metrics.requestBytes - control.metrics.requestBytes, systemByteDelta: experiment.metrics.systemBytes - control.metrics.systemBytes,
|
||||
durationDeltaMs: experiment.metrics.durationMs - control.metrics.durationMs, controlMetrics: control.metrics, experimentMetrics: experiment.metrics,
|
||||
modelBenefit: 'not-measured-owned-response-fixture', defaultEnabled: false });
|
||||
mark(source.version + ' repetition ' + repetition + ': identical original task/baseline/tests/response algorithm and budgets; actual prompt overhead and host approvals recorded');
|
||||
}
|
||||
assert.equal(results.length, 8); assert.equal(comparisons.length, 4);
|
||||
} else for (const kind of ['node', 'react', 'python']) results.push(await runSample(kind));
|
||||
assert.deepEqual(inputs(), frozen);
|
||||
report = { success: true, targetVersion: '1.14.0', testedVersion: version, directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
|
||||
scope: 'Pinned actual Harness and workbench execution with an owned HTTP model fixture. Checks, approvals, failures, parent merges, Edge and Git are real. Model reasoning and Purge benefit remain separate.' };
|
||||
} catch (error) { report = { success: false, targetVersion: '1.14.0', testedVersion: version, directory, installation, cases, results, inputs: frozen, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
|
||||
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/controlled-projects-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
|
||||
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture } from '../tests/helpers.mjs';
|
||||
import { workerHttp } from '../tests/fixtures/workers-http.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
||||
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
||||
import { resolvePurgeSource, PURGE_SOURCE_FILES, assertPurgeModelInput } from '../src/core/engines/purge-strategy.mjs';
|
||||
import { verifyEngineModelInput, originalEngineRun } from '../src/shared/contracts/engine-execution.mjs';
|
||||
|
||||
assert.ok(process.argv.slice(2).every(value => value === '--help'), 'Unknown argument; use --help.');
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-purge-ui.mjs — actual Edge and Harness, two declared Purge source copies, default-off/changed source/literal prompt/owned Run evidence; no configured-model requests. Requires AGENT_WORKBENCH_DEEPSEEK_ROOT and AGENT_WORKBENCH_PURGE_178_ROOT / AGENT_WORKBENCH_PURGE_181_ROOT.'); process.exit(0); }
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, directory = path.resolve('.qa', 'controlled-purge-ui-' + randomUUID());
|
||||
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT); assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const sources = ['1.1.78', '1.1.81'].map((version, index) => {
|
||||
const root = process.env[index === 0 ? 'AGENT_WORKBENCH_PURGE_178_ROOT' : 'AGENT_WORKBENCH_PURGE_181_ROOT']; assert.ok(root && path.isAbsolute(root));
|
||||
const original = resolvePurgeSource(process.cwd(), root); assert.equal(original.status, 'available', original.message); assert.equal(original.version, version);
|
||||
const copy = path.join(directory, 'declared-source-copy-' + version);
|
||||
for (const relative of PURGE_SOURCE_FILES) { const target = path.join(copy, relative); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.copyFileSync(path.join(root, relative), target); }
|
||||
return { original, copy, source: resolvePurgeSource(process.cwd(), copy) };
|
||||
});
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), host = await workerHttp(f), original = ownerBytes(f);
|
||||
f.store.saveConfig({ contextLimit: 64000 });
|
||||
host.runtime.controlledEngines = new EngineRunner(f.store, host.provider, { directoryRoot: path.join(f.root, 'engine-runs') });
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
function inputs() {
|
||||
const files = ['package.json', 'package-lock.json'];
|
||||
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) { const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file); } };
|
||||
for (const root of ['src', 'scripts', 'tests', 'benchmarks', 'dist-client', 'dist-server']) walk(root);
|
||||
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
|
||||
}
|
||||
const frozen = inputs(), assertions = [], screenshots = [], pageErrors = [], requestFailures = [], expectedRejections = [];
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 }, acceptDownloads: true });
|
||||
page.on('pageerror', error => pageErrors.push(error.message)); page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const panel = () => page.getByLabel('隔离协作控制台', { exact: true }), experiment = () => panel().getByLabel('Purge 提示实验', { exact: true });
|
||||
const toggle = () => experiment().getByRole('checkbox', { name: '启用 Purge 提示实验', exact: true });
|
||||
const seal = () => panel().getByRole('button', { name: '封存分工计划', exact: true });
|
||||
const mark = text => { assertions.push(text); console.log('PASS ' + text); };
|
||||
async function screenshot(name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push({ file, sha256: hash(fs.readFileSync(file)) }); }
|
||||
async function closeInspector() { const close = page.getByRole('button', { name: '关闭检视区', exact: true }); if (page.viewportSize().width <= 940 && await close.isVisible()) { await close.click(); await expect(page.locator('.inspector')).toBeHidden(); } }
|
||||
async function workers() { await closeInspector(); await page.getByRole('button', { name: '隔离协作', exact: true }).click(); await expect(panel()).toBeVisible(); }
|
||||
async function source(root, version) { await experiment().getByLabel('Purge 源码目录', { exact: true }).fill(root); await experiment().getByRole('button', { name: '检测 Purge 来源', exact: true }).click(); await expect(experiment()).toContainText('已核对 Purge ' + version); }
|
||||
let result, releaseLate;
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); await workers(); await panel().getByLabel('执行方式', { exact: true }).selectOption('deepseek');
|
||||
await expect(toggle()).not.toBeChecked(); await expect(experiment()).toContainText('默认关闭');
|
||||
await panel().getByLabel('Harness 安装目录', { exact: true }).fill(installation.root); await panel().getByRole('button', { name: '检测受控安装', exact: true }).click(); await expect(panel()).toContainText('受控安装可用');
|
||||
await panel().getByLabel('父目标', { exact: true }).fill('保留来源和原权限,交付同题工程提示实验。');
|
||||
await panel().getByLabel('任务名称', { exact: true }).fill('engine-1');
|
||||
await panel().getByLabel('独立子目标', { exact: true }).fill('修改src/a.mjs并通过原检查。'); await panel().getByLabel('允许修改的文件或目录', { exact: true }).fill('src/a.mjs');
|
||||
await expect(seal()).toBeEnabled(); assert.equal(host.requests.length, 0); mark('desktop prompt experiment is off by default and does not require, import or run Purge when disabled');
|
||||
await toggle().check(); await experiment().getByLabel('Purge 源码目录', { exact: true }).fill(path.join(directory, 'missing-source'));
|
||||
await experiment().getByRole('button', { name: '检测 Purge 来源', exact: true }).click(); await expect(experiment()).toContainText('来源尚不可用'); await expect(seal()).toBeDisabled();
|
||||
await source(sources[0].copy, '1.1.78'); await experiment().getByLabel('增强工程提示', { exact: true }).fill('按原证据修复工程,保留 {{engineering_literal}} 字面值与实际审批。'); await expect(seal()).toBeEnabled();
|
||||
const tasksBefore = f.store.core.task.size; fs.appendFileSync(path.join(sources[0].copy, 'lib/identity.js'), '// declared post-detection mutation\n');
|
||||
await seal().click(); await expect(panel()).toContainText('Purge 来源与刚才的检测不一致'); assert.equal(f.store.core.task.size, tasksBefore); assert.equal(host.requests.length, 0);
|
||||
mark('missing sources block sealing and an actual source change after UI detection is refused before creating a Task');
|
||||
await source(sources[1].copy, '1.1.81'); await expect(experiment().locator('.purge-source-state')).toContainText(sources[1].source.sourceSha256);
|
||||
await expect(panel()).not.toContainText('Purge 来源与刚才的检测不一致'); mark('successfully detecting the replacement source clears the obsolete failed-sealing feedback');
|
||||
await toggle().uncheck(); await expect(experiment().getByLabel('Purge 源码目录', { exact: true })).toHaveCount(0); await expect(seal()).toBeEnabled(); await toggle().check();
|
||||
await screenshot('desktop-purge-source'); mark('both source versions are detected explicitly and disabling the experiment removes its required source and prompt fields');
|
||||
let lateSeen = false, lateFulfilled = false;
|
||||
const late = new Promise(resolve => { releaseLate = resolve; });
|
||||
await page.route('**/api/engines/purge/source*', async route => { lateSeen = true; await late; expectedRejections.push({ source: 'declared-late-source-response', status: 500 }); await route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: 'late-purge-source-error' }) }); lateFulfilled = true; });
|
||||
await experiment().getByRole('button', { name: '检测 Purge 来源', exact: true }).click(); await expect.poll(() => lateSeen).toBe(true); await toggle().uncheck(); releaseLate();
|
||||
await expect.poll(() => lateFulfilled).toBe(true); await expect(experiment()).not.toContainText('late-purge-source-error'); await page.unroute('**/api/engines/purge/source*'); await toggle().check(); await source(sources[1].copy, '1.1.81');
|
||||
mark('a late detection failure after disabling cannot overwrite or leak into the current experiment form');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await experiment().locator('header').scrollIntoViewIfNeeded();
|
||||
await expect.poll(async () => (await experiment().getByRole('button', { name: '检测 Purge 来源', exact: true }).boundingBox()).height).toBeGreaterThanOrEqual(42);
|
||||
const touch = await experiment().getByRole('button', { name: '检测 Purge 来源', exact: true }).boundingBox(); assert.ok(touch.height >= 42);
|
||||
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false); await expect(seal()).toBeEnabled(); await screenshot('mobile-purge-source');
|
||||
await seal().click(); await expect(panel()).toContainText('计划已封存'); const group = [...f.store.core.collaboration.values()].at(-1), bound = group.children[0].engine.strategy;
|
||||
assert.equal(bound.enabled, true); assert.equal(bound.source.version, '1.1.81'); assert.equal(bound.source.sourceSha256, sources[1].source.sourceSha256); assert.ok(bound.prompt.includes('{{engineering_literal}}'));
|
||||
await expect(panel()).toContainText('本计划的 Purge 提示实验'); assert.equal(host.requests.length, 0); assert.deepEqual(ownerBytes(f), original);
|
||||
mark('390px controls remain readable and touchable; sealing retains the exact selected source and literal prompt without model requests');
|
||||
await panel().getByRole('button', { name: '创建引擎工作区', exact: true }).click(); await page.getByRole('dialog', { name: '确认创建隔离工作区', exact: true }).getByRole('button', { name: '确认创建', exact: true }).click();
|
||||
await expect(panel().getByRole('button', { name: '开始引擎任务', exact: true })).toBeEnabled({ timeout: 30000 });
|
||||
await panel().getByRole('button', { name: '开始引擎任务', exact: true }).click(); await page.getByRole('dialog', { name: '确认受限子执行', exact: true }).getByRole('button', { name: '确认开始', exact: true }).click();
|
||||
await expect(panel()).toContainText('审查子任务改动', { timeout: 45000 });
|
||||
await expect.poll(() => group.status, { timeout: 45000 }).toBe('awaiting_review');
|
||||
const job = [...f.store.core.engine_run.values()].at(-1), session = f.store.session(job.sessionId);
|
||||
assert.equal(job.status, 'completed', job.error); assert.equal(job.native.cleanup.treeConfirmed, true); assert.equal(job.native.stderr, ''); assert.deepEqual(job.installation.strategy, bound);
|
||||
for (const request of job.requests) { const input = verifyEngineModelInput(f.store, session, job, request); assertPurgeModelInput(input.native, bound); assert.ok(input.body.messages[0].content.includes('{{engineering_literal}}')); }
|
||||
assert.equal(job.native.readiness.promptExperimentSha256, bound.sha256); assert.deepEqual(ownerBytes(f), original);
|
||||
mark('actual Harness applies the exact non-interpolated prompt once in every owned model request and closes its complete process tree');
|
||||
await panel().getByRole('button', { name: '审查子任务改动', exact: true }).click(); await expect.poll(() => page.evaluate(() => localStorage.getItem('awb-session'))).toBe(session.id);
|
||||
await closeInspector(); await page.getByRole('button', { name: '本地代理引擎', exact: true }).click(); const runs = page.getByLabel('受控引擎工程运行', { exact: true });
|
||||
await expect(runs).toContainText('Purge 提示实验 · 1.1.81'); await runs.getByText('查看本次实验策略', { exact: true }).click(); await expect(runs).toContainText(bound.sha256); await expect(runs).toContainText(bound.prompt);
|
||||
const downloadPromise = page.waitForEvent('download'); await runs.getByRole('button', { name: '下载完整报告', exact: true }).click(); const download = await downloadPromise;
|
||||
const reportPath = path.join(directory, 'original-engine-report.json'); await download.saveAs(reportPath);
|
||||
const originalReport = originalEngineRun(f.store, session, job); assert.equal(hash(fs.readFileSync(reportPath)), job.reportSha256); assert.equal(fs.readFileSync(reportPath, 'utf8'), originalReport);
|
||||
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false); await screenshot('mobile-purge-run');
|
||||
mark('owned Run UI displays the original source/text/strategy hashes and downloads the exact report bytes after real native execution');
|
||||
await page.keyboard.press('Escape'); await workers(); await panel().getByRole('button', { name: '返回父目标', exact: true }).click(); await expect.poll(() => page.evaluate(() => localStorage.getItem('awb-session'))).toBe(f.session.id); await workers();
|
||||
await panel().getByRole('button', { name: '结束协作并保留现场', exact: true }).click(); await page.getByRole('dialog', { name: '确认保留现场', exact: true }).getByRole('button', { name: '确认保留现场结束', exact: true }).click();
|
||||
await expect(panel().getByRole('button', { name: '创建下一父目标', exact: true })).toBeEnabled(); await panel().getByRole('button', { name: '创建下一父目标', exact: true }).click();
|
||||
await panel().getByLabel('执行方式', { exact: true }).selectOption('deepseek'); await expect(toggle()).not.toBeChecked(); await expect(experiment()).toContainText('默认关闭');
|
||||
mark('retaining the original Run and starting the next parent goal restores default-off without changing prior strategy, report or usage');
|
||||
assert.equal(pageErrors.length, 0); assert.equal(requestFailures.length, 0); assert.deepEqual(inputs(), frozen); assert.deepEqual(ownerBytes(f), original);
|
||||
result = { passed: true, version, targetVersion: '1.14.0', assertions, screenshots, pageErrors, requestFailures, expectedRejections, sources, groupId: group.id,
|
||||
engineRun: job, inputs: frozen, realModelRequests: 0, scope: 'Actual Edge/Harness and an owned in-memory model fixture. Source changes affect declared nine-file copies only; no upstream plugin code executes. Real model benefit remains unmeasured.' };
|
||||
} catch (error) { result = { passed: false, version, assertions, screenshots, pageErrors, requestFailures, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
|
||||
finally {
|
||||
releaseLate?.(); await host.runtime.stopAll(); await browser.close(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.cpSync(f.root, path.join(directory, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/controlled-purge-ui-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') }));
|
||||
for (const dispose of disposers.reverse()) await dispose(); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ passed: result.passed, nodes: assertions.length, directory, error: result.error ?? null }));
|
||||
}
|
||||
Reference in New Issue
Block a user