281 lines
30 KiB
JavaScript
281 lines
30 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { fixture, compatibleFixture, call, assertToolPairs } from '../tests/helpers.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
|
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
|
import { originalEngineRun, verifyEngineModelInput } from '../src/shared/contracts/engine-execution.mjs';
|
|
import { resolvePurgeSource, assertPurgeModelInput, nativeSystemTexts, PURGE_DEFAULT_PROMPT } from '../src/core/engines/purge-strategy.mjs';
|
|
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
|
|
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
|
import { desktopProjectQa } from './desktop-project-qa.mjs';
|
|
import { specification } from '../tests/fixtures/engineering-specification.mjs';
|
|
import { responsesEngineeringHttp } from '../tests/fixtures/responses-engineering-http.mjs';
|
|
import { nativeContextSpecification } from '../tests/fixtures/native-context-project.mjs';
|
|
|
|
const args = process.argv.slice(2);
|
|
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop', '--responses', '--context-16k'].includes(value)), 'Unknown option; use --help.');
|
|
if (args.includes('--help')) {
|
|
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison | --desktop | --context-16k] [--responses] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. --context-16k uses a new copy of the sealed three-module/nine-test pilot, original failure/log paging and a complete reviewed batch at 16K. --responses uses actual Responses SSE and ModelHub routing. Set AGENT_WORKBENCH_DEEPSEEK_ROOT and AGENT_WORKBENCH_TEST_PYTHON; context mode needs no Python. No configured-model requests.'); process.exit(0);
|
|
}
|
|
const comparisonMode = args.includes('--purge-comparison'), desktopMode = args.includes('--desktop');
|
|
const contextMode = args.includes('--context-16k');assert.ok(!contextMode || !comparisonMode && !desktopMode, '16K bridge acceptance is a separate run.');
|
|
const responsesMode = args.includes('--responses'); assert.ok(!responsesMode || !comparisonMode, 'Protocol acceptance and Purge comparison remain separate.');
|
|
assert.ok(!comparisonMode || !desktopMode, 'Desktop engineering and prompt comparison are separate acceptance runs.');
|
|
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
|
const directory = path.resolve('.qa', (contextMode ? 'controlled-context-' : desktopMode ? 'desktop-projects-' : comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
|
|
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT);
|
|
assert.equal(installation.controlledStatus, 'available', installation.message);
|
|
const python = process.env.AGENT_WORKBENCH_TEST_PYTHON;
|
|
if (!comparisonMode && !contextMode) assert.ok(python && path.isAbsolute(python) && fs.existsSync(python), 'Provide the existing isolated Python 3.10+ interpreter.');
|
|
const sources = comparisonMode ? ['1.1.78', '1.1.81'].map((expected, index) => {
|
|
const root = process.env[index === 0 ? 'AGENT_WORKBENCH_PURGE_178_ROOT' : 'AGENT_WORKBENCH_PURGE_181_ROOT'];
|
|
assert.ok(root && path.isAbsolute(root), 'Provide both existing Purge source directories.');
|
|
const source = resolvePurgeSource(process.cwd(), root); assert.equal(source.status, 'available', source.message); assert.equal(source.version, expected); return source;
|
|
}) : [];
|
|
fs.mkdirSync(directory, { recursive: true });
|
|
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
|
function inputs() {
|
|
const files = ['package.json', 'package-lock.json'];
|
|
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
|
const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file);
|
|
} };
|
|
for (const root of ['src', 'scripts', 'tests', 'benchmarks', 'dist-client', 'dist-server', ...(desktopMode ? ['dist-desktop'] : [])]) walk(root);
|
|
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
|
|
}
|
|
const frozen = inputs(), mark = message => { cases.push(message); console.log('PASS ' + message); };
|
|
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
|
|
const startedAt = Date.now(), label = sampleName;
|
|
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
|
|
fs.mkdirSync(sample); let runtime, group, result, desktopQa, model, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
|
|
try {
|
|
const baseline = contextMode ? 'docs/live-model-purge-1.24-evidence/source-baselines/r1-off' : kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
|
|
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
|
write(f, '.gitignore', 'node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n');
|
|
write(f, 'owner-staged.txt', 'Owner baseline\n'); write(f, 'owner-unstaged.txt', 'Owner baseline\n');
|
|
git(f, ['init', '--initial-branch=main']); git(f, ['config', 'core.autocrlf', 'false']); git(f, ['config', 'user.name', 'Workbench QA']); git(f, ['config', 'user.email', 'qa@workbench.invalid']);
|
|
git(f, ['add', '--', '.']); git(f, ['commit', '--no-gpg-sign', '-m', 'Seal ' + kind + ' controlled-engine acceptance']);
|
|
const baselineTree = git(f, ['rev-parse', 'HEAD^{tree}']).trim();
|
|
write(f, 'owner-staged.txt', 'Owner staged change\n'); git(f, ['add', '--', 'owner-staged.txt']);
|
|
write(f, 'owner-staged.txt', 'Owner working copy after staging\n'); write(f, 'owner-unstaged.txt', 'Owner unstaged change\n'); write(f, 'owner-untracked.txt', 'Owner untracked file\n');
|
|
const spec = contextMode ? nativeContextSpecification : specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort();
|
|
const original = ownerBytes(f), protectedNames = git(f, ['ls-files', '-z']).split('\0').filter(file => file && !writable.includes(file) && !file.startsWith('owner-'));
|
|
const protectedFiles = Object.fromEntries(protectedNames.map(file => [file, fs.readFileSync(path.join(f.projectRoot, file))]));
|
|
const verify = root => { for (const [file, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(root, file)), bytes, 'Protected ' + file); };
|
|
const install = (root, label) => {
|
|
const child = spawnSync(process.execPath, [npmCli, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: root, windowsHide: true, timeout: 120000, maxBuffer: 8 * 1024 * 1024 });
|
|
fs.writeFileSync(path.join(sample, label + '-npm-ci-log.txt'), child.stdout ?? Buffer.alloc(0)); fs.writeFileSync(path.join(sample, label + '-npm-ci-stderr-log.txt'), child.stderr ?? Buffer.alloc(0));
|
|
if (child.error) throw child.error; assert.equal(child.status, 0, 'Inspect the original offline installation logs.'); verify(root);
|
|
};
|
|
if (kind === 'react') install(f.projectRoot, 'parent');
|
|
let phase = 'new';
|
|
model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
|
|
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
|
|
assertToolPairs(input.messages);
|
|
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
|
|
if (desktopQa && !cycledRequest) { cycledRequest = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-owned-model-request'); }
|
|
const checks = [...f.store.core.check_job.values()].filter(job => job.taskId === worker.taskId), job = checks.at(-1);
|
|
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === worker.taskId).at(-1);
|
|
const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Fixed local acceptance repair; retain all tests and declarations.' }));
|
|
let calls = [];
|
|
if(contextMode){
|
|
const pending=session.changes.filter(change=>change.status==='pending');
|
|
const batch=[...f.store.core.change_set.values()].findLast(set=>set.taskId===worker.taskId&&set.status==='prepared');
|
|
if(pending.length){if(!batch)calls=[call(randomUUID(),'prepare_change_set',{change_ids:pending.map(change=>change.id)})];}
|
|
else if(phase==='new'){
|
|
calls=[...writable,'test/expense.test.mjs'].map(file=>call(randomUUID(),'read_file',{path:file,limit:16000}));phase='observed';
|
|
}else if(phase!=='fixed'&&job){
|
|
assert.equal(job.status,'failed');assert.equal(job.results[0].tests.failed,9);
|
|
if(phase!=='failure-log-read'){
|
|
calls=[call(randomUUID(),'read_artifact',{artifact_id:job.results[0].logArtifactId,offset:0,limit:16000})];phase='failure-log-read';
|
|
}else{calls=propose(spec.fixes);phase='fixed';}
|
|
}else if(phase==='fixed'&&job?.status==='passed')assert.equal(job.results[0].tests.passed,9);
|
|
else if(!plan||job?.planId===plan.id)calls=[call(randomUUID(),'plan_checks',{names:spec.checks})];
|
|
else calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
|
|
}
|
|
else if (session.changes.some(change => change.status === 'pending')) { /* Explicit review follows native completion. */ }
|
|
else if (phase === 'new' && spec.initial) { calls = propose(spec.initial); phase = 'initial'; }
|
|
else if (phase !== 'fixed' && job) {
|
|
assert.equal(job.status, 'failed', job.summary); assert.ok(job.results.some(result => result.tests?.failed > 0)); calls = propose(spec.fixes); phase = 'fixed';
|
|
} else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, spec.tests);
|
|
else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: spec.checks })];
|
|
else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })];
|
|
modelRequests.push({ requestBody: input, callIds: calls.map(call => call.id), phase,
|
|
responsePlan: calls.map(call => ({ name: call.function.name, arguments: { ...JSON.parse(call.function.arguments),
|
|
...(call.function.name === 'run_checks' ? { plan_id: 'owned-current-check-plan' } : {}) } })) });
|
|
return { choices: [{ message: { role: 'assistant', content: calls.length ? '' : 'Owned native run complete; host proof is still required.', ...(calls.length ? { tool_calls: calls } : {}) },
|
|
finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 123, completion_tokens: 45, total_tokens: 168 } };
|
|
});
|
|
f.store.saveConfig({ mode: 'compatible', model: 'owned-controlled-engineering', baseUrl: model.baseUrl, contextLimit: contextMode ? 16000 : 64000,
|
|
...(contextMode ? {maxOutputTokens:2048} : {}) });
|
|
const provider = new Provider(); provider.setKey('self-owned-controlled-model-fixture');
|
|
if (responsesMode) {
|
|
const hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned Responses engineering', protocol: 'responses', model: 'owned-controlled-engineering', baseUrl: model.baseUrl,
|
|
requiresKey: false, contextLimit: contextMode ? 16000 : 64000, ...(contextMode ? {maxOutputTokens:2048} : {}), streaming: true, capabilityDeclarations: { streaming: 'supported' } });
|
|
hub.select({ type: 'connection', id: connection.id });
|
|
}
|
|
runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
|
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine-runs') });
|
|
runtime.checkRunner().pythonExecutable = python; const manager = runtime.workerManager();
|
|
if (desktopMode) desktopQa = await desktopProjectQa({ store: f.store, provider, runtime, sample, kind });
|
|
group = await manager.plan(f.session, { goal: 'Deliver ' + kind + ' with the pinned controlled Harness, retained failures, explicit review, parent validation and local Git.',
|
|
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256, ...(strategy ? { strategy } : {}) },
|
|
budget: { maxTokens: 200000, maxSteps: 64, maxToolCalls: 128, maxDurationMs: 600000 },
|
|
children: [{ label: 'engine-' + kind, goal: 'Repair and validate the complete ' + kind + ' functional suite without replacing tests.', paths: writable }] });
|
|
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'ready', group.summary);
|
|
const child = group.children[0], childRoot = f.store.project(child.projectId).root;
|
|
assert.notEqual(childRoot, f.projectRoot); assert.deepEqual(ownerBytes(f), original); verify(childRoot); if (kind === 'react') install(childRoot, 'child');
|
|
if (desktopQa) await desktopQa.select(f.store.session(child.sessionId));
|
|
mark(label + ': pinned native engine and one owned Git worktree preserve parent staging and protected bytes');
|
|
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
|
|
const session = f.store.session(child.sessionId), ids = session.changes.filter(change => change.status === 'pending').map(change => change.id);
|
|
if (ids.length) {
|
|
assert.equal(group.status, 'awaiting_review', group.summary);
|
|
const nativeBatch=contextMode&&[...f.store.core.change_set.values()].findLast(set=>set.taskId===child.taskId&&set.status==='prepared');
|
|
const set = nativeBatch || (desktopQa ? await desktopQa.review(session, ids) : await runtime.changeSetManager().prepare(session, ids));
|
|
if(contextMode)assert.deepEqual(set.files.map(file=>file.path).sort(),writable);
|
|
reviews.push({ operation: 'review child batch', id: set.id, fingerprint: set.fingerprint, paths: set.files.map(file => file.path) });
|
|
if (!desktopQa) await (await runtime.changeSetManager().start(session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary);
|
|
}
|
|
const started = manager.start(f.session, group.id, group.fingerprint); let done = false, failure;
|
|
started.promise.then(() => { done = true; }, error => { failure = error; done = true; });
|
|
while (!done) {
|
|
for (const approval of session.approvals.filter(approval => approval.status === 'pending')) {
|
|
assert.equal(f.store.core.tool.get(approval.invocationId).tool, 'run_checks');
|
|
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === child.taskId).at(-1);
|
|
assert.deepEqual(plan.checks.map(check => check.name), spec.checks);
|
|
reviews.push({ operation: 'approve original check', id: approval.id, taskId: child.taskId, planId: plan.id, contractSha256: plan.contractSha256 });
|
|
if (desktopQa) { if (!cycledApproval) { cycledApproval = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-original-approval'); } await desktopQa.approve(session, approval); }
|
|
else runtime.decide(session.id, approval.id, true);
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 30));
|
|
}
|
|
if (failure) throw failure;
|
|
const native = [...f.store.core.engine_run.values()].filter(job => job.taskId === child.taskId).at(-1);
|
|
assert.equal(native.status, 'completed', native.error);
|
|
}
|
|
assert.equal(group.status, 'ready_to_collect', group.summary); assert.deepEqual(ownerBytes(f), original); verify(childRoot);
|
|
const childChecks = [...f.store.core.check_job.values()].filter(job => job.taskId === child.taskId);
|
|
assert.ok(childChecks.some(job => job.status === 'failed' && job.results.some(result => result.tests?.failed > 0)));
|
|
assert.equal(childChecks.at(-1).results[0].tests.passed, spec.tests); assert.equal(childChecks.at(-1).status, 'passed');
|
|
const engineRuns = [...f.store.core.engine_run.values()].filter(job => job.taskId === child.taskId), childSession = f.store.session(child.sessionId);
|
|
for (const job of engineRuns) {
|
|
originalEngineRun(f.store, childSession, job); assert.equal(job.status, 'completed', job.error); assert.equal(job.native.cleanup.treeConfirmed, true);
|
|
assert.equal(job.native.stderr, ''); assert.equal(job.native.activity.endEvents[0].data.reason.kind, 'completed');
|
|
for (const request of job.requests) {
|
|
const saved = verifyEngineModelInput(f.store, childSession, job, request);
|
|
if (strategy) { assertPurgeModelInput(saved.native, job.installation.strategy); assert.equal(job.native.readiness.promptExperimentSha256, job.installation.strategy.sha256); }
|
|
else assert.ok(nativeSystemTexts(saved.native).every(text => !text.includes('[Agent Workbench engineering prompt experiment]')));
|
|
}
|
|
}
|
|
assert.equal(f.store.core.task.get(child.taskId).usage.requests, modelRequests.length);
|
|
assert.equal(f.store.core.task.get(child.taskId).usage.totalTokens, modelRequests.length * 168);
|
|
if (responsesMode) {
|
|
const attempts = provider.modelHub(f.store).attempts;
|
|
assert.equal(attempts.length, model.requests.length); assert.ok(attempts.every(value => value.protocol === 'responses' && value.stream?.terminal && value.responseArtifactId));
|
|
fs.writeFileSync(path.join(sample, 'original-responses-requests.json'), JSON.stringify(model.requests, null, 2));
|
|
fs.writeFileSync(path.join(sample, 'original-responses-attempts.json'), JSON.stringify(attempts, null, 2));
|
|
}
|
|
mark(label + ': actual Harness uses owned model/tool bridges, retains failing tests, and completes reviewed repairs with one accounting entry per HTTP request');
|
|
if(contextMode){
|
|
const bounded=engineRuns.flatMap(job=>job.requests.map(request=>verifyEngineModelInput(f.store,childSession,job,request))).filter(input=>input.view.mode==='bounded');
|
|
assert.ok(bounded.length>0,'The original long failure/log history must exercise the bounded bridge.');
|
|
assert.ok(bounded.every(input=>input.view.beforeInputTokens>input.view.inputBudget&&input.plan.estimatedInputTokens<=input.view.inputBudget));
|
|
const sent=JSON.stringify(bounded.map(input=>input.body));assert.ok(sent.includes('logArtifactId'));assert.ok(sent.includes('failed'));
|
|
const failed=childChecks.find(job=>job.status==='failed'),log=f.store.core.artifact.get(failed.results[0].logArtifactId);
|
|
assert.ok(f.store.artifactStore.read(log).includes('fail 9'));assert.ok(engineRuns.some(job=>job.calls.some(call=>call.name==='read_artifact')));
|
|
assert.ok(engineRuns.some(job=>job.calls.some(call=>call.name==='prepare_change_set')));assert.ok(engineRuns.every(job=>!job.inputRejections?.length));
|
|
mark('16K: original nine failures and paged log survive verified native compaction; all three proposals form one reviewed batch without replay');
|
|
}
|
|
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'collected', group.summary);
|
|
const collection = await manager.readCollection(f.session, group); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
|
|
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'merge_prepared', group.summary);
|
|
const merge = f.store.core.change_set.get(group.merge.changeSetId); reviews.push({ operation: 'review parent batch', id: merge.id, fingerprint: merge.fingerprint });
|
|
if (desktopQa) assert.equal((await desktopQa.review(f.session, merge.files.map(file => file.changeId))).id, merge.id);
|
|
else await (await runtime.changeSetManager().start(f.session, merge.id, merge.fingerprint)).promise;
|
|
assert.equal(merge.status, 'completed', merge.summary);
|
|
const checkPlan = await runtime.checkRunner().plan(f.session, spec.checks), check = await (await runtime.checkRunner().start(f.session, checkPlan.id)).promise;
|
|
assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify(f.projectRoot);
|
|
mark(label + ': separately reviewed parent merge passes all ' + spec.tests + ' functional tests and actual build/compile');
|
|
if (desktopQa) await desktopQa.capture(f.session, 'parent-checks');
|
|
const browserJobs = [];
|
|
if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
|
|
const steps = [ { action: 'assert_count', selector: '.task-list article', count: 3 },
|
|
{ action: 'assert_text', selector: '.summary > div:nth-child(3) strong', text: viewport.width > 600 ? '8小时' : '8\n小时' },
|
|
{ action: 'select', selector: 'select[aria-label="任务状态"]', value: 'pending' }, { action: 'assert_count', selector: '.task-list article', count: 2 },
|
|
{ action: 'fill', selector: 'input[aria-label="任务搜索"]', value: ' 编写 ' }, { action: 'assert_count', selector: '.task-list article', count: 1 },
|
|
{ action: 'fill', selector: 'input[aria-label="任务搜索"]', value: '' }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'all' },
|
|
{ action: 'click', selector: '.task-list article[data-task="api"] button' }, { action: 'assert_text', selector: '.task-list article[data-task="api"] .tag', text: '已完成' },
|
|
{ action: 'assert_text', selector: '.task-list article[data-task="tests"] .tag', text: '待完成' }, { action: 'assert_text', selector: '.summary > div:nth-child(2) strong', text: '2' },
|
|
{ action: 'assert_text', selector: '.summary > div:nth-child(3) strong', text: viewport.width > 600 ? '5小时' : '5\n小时' } ];
|
|
const plan = await runtime.browserManager().plan(f.session, { viewport, steps }), job = await runtime.browserManager().start(f.session, plan.id).promise;
|
|
assert.equal(job.status, 'passed', job.summary); assert.equal(job.cleanup.exited, true); browserJobs.push(job);
|
|
for (const image of job.screenshots) fs.writeFileSync(path.join(sample, 'page-' + viewport.width + '-' + image.artifactId + '.png'), f.store.artifactStore.readBytes(f.store.core.artifact.get(image.artifactId)));
|
|
}
|
|
if (kind === 'react') mark('react: real Edge desktop/390px filtering, trimmed search, one-task toggle and complete totals pass');
|
|
const deliveryPlan = await runtime.gitDelivery().plan(f.session, { message: 'Reviewed ' + kind + ' controlled-engine delivery' }), delivery = await (await runtime.gitDelivery().start(f.session, deliveryPlan.id)).promise;
|
|
assert.equal(delivery.status, 'completed', delivery.summary); await manager.finalize(f.session, group.id, group.fingerprint); assert.equal(group.status, 'completed');
|
|
assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n');
|
|
assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable);
|
|
mark(label + ': local Git contains only reviewed functional changes and preserves original staged/unstaged owner bytes');
|
|
if (desktopQa) { await desktopQa.capture(f.session, 'git-completed'); assert.equal(cycledRequest, true); assert.equal(cycledApproval, true); mark(label + ': real Electron reviews/approvals and window cycles retain Task/Run/eventSeq/results at 1440px and 390px'); }
|
|
const requests = engineRuns.flatMap(job => job.requests), metrics = { durationMs: Date.now() - startedAt, baselineTree,
|
|
contextLimit: group.config.contextLimit, budget: group.budget, model: group.model.model, tests: spec.tests,
|
|
engineeringFailures: childChecks.filter(job => job.status === 'failed').length,
|
|
hostApprovals: reviews.filter(review => review.operation === 'approve original check').length, reviews: reviews.length,
|
|
requests: modelRequests.length, estimatedInputTokens: requests.reduce((sum, request) => sum + request.estimatedInputTokens, 0),
|
|
requestBytes: requests.reduce((sum, request) => sum + request.requestBytes, 0),
|
|
systemBytes: modelRequests.reduce((sum, request) => sum + Buffer.byteLength(request.requestBody.messages.find(message => message.role === 'system')?.content ?? ''), 0),
|
|
reportedFixtureTokens: modelRequests.length * 168, unknownRequests: f.store.core.task.get(child.taskId).usage.unknownRequests,
|
|
responseSequence: modelRequests.map(({ phase, responsePlan }) => ({ phase, responsePlan })) };
|
|
result = { success: true, scope: 'actual-native-Harness-with-owned-HTTP-model-fixture', version, kind, sampleName, comparison, metrics, parentTaskId: group.taskId, groupId: group.id,
|
|
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
|
|
writable, reviews, modelRequests, ...(desktopQa ? { desktop: { events: desktopQa.events, screenshots: desktopQa.screenshots, scope: desktopQa.scope, errors: desktopQa.errors } } : {}), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
|
return result;
|
|
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, ownedFixtureErrors:model?.errors??[], engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
|
finally {
|
|
await desktopQa?.close();
|
|
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
fs.cpSync(f.root, path.join(sample, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
|
for (const session of f.store.sessions.values()) fs.writeFileSync(path.join(sample, 'session-' + session.id + '.json'), JSON.stringify({ project: f.store.project(session.projectId), session, execution: f.store.executionSnapshot(session) }, null, 2));
|
|
fs.writeFileSync(path.join(sample, 'result.json'), JSON.stringify(result, null, 2));
|
|
for (const dispose of disposers.reverse()) await dispose();
|
|
}
|
|
}
|
|
let report;
|
|
try {
|
|
const comparisons = [];
|
|
if (comparisonMode) {
|
|
const prompt = PURGE_DEFAULT_PROMPT + '\n保留字面值 {{engineering_literal}},依据实际失败证据完成同题修复。';
|
|
for (const source of sources) for (const repetition of [1, 2]) {
|
|
const pair = [];
|
|
for (const enabled of repetition === 1 ? [false, true] : [true, false]) {
|
|
const comparison = { sourceVersion: source.version, sourceSha256: source.sourceSha256, repetition, enabled };
|
|
const result = await runSample('node', { sampleName: source.version + '-r' + repetition + '-' + (enabled ? 'enabled' : 'disabled'), comparison,
|
|
strategy: enabled ? { kind: 'purge-prompt-experiment', root: source.root, sourceSha256: source.sourceSha256, prompt } : null });
|
|
results.push(result); pair.push(result);
|
|
}
|
|
const control = pair.find(result => !result.comparison.enabled), experiment = pair.find(result => result.comparison.enabled);
|
|
for (const key of ['baselineTree', 'contextLimit', 'budget', 'model', 'tests', 'responseSequence', 'hostApprovals', 'engineeringFailures', 'reportedFixtureTokens', 'requests']) assert.deepEqual(experiment.metrics[key], control.metrics[key], 'Comparable original ' + key);
|
|
assert.deepEqual(experiment.writable, control.writable); assert.deepEqual(experiment.protectedFiles, control.protectedFiles);
|
|
assert.ok(experiment.metrics.systemBytes > control.metrics.systemBytes);
|
|
comparisons.push({ sourceVersion: source.version, repetition, control: control.sampleName, experiment: experiment.sampleName,
|
|
estimatedInputTokenDelta: experiment.metrics.estimatedInputTokens - control.metrics.estimatedInputTokens,
|
|
requestByteDelta: experiment.metrics.requestBytes - control.metrics.requestBytes, systemByteDelta: experiment.metrics.systemBytes - control.metrics.systemBytes,
|
|
durationDeltaMs: experiment.metrics.durationMs - control.metrics.durationMs, controlMetrics: control.metrics, experimentMetrics: experiment.metrics,
|
|
modelBenefit: 'not-measured-owned-response-fixture', defaultEnabled: false });
|
|
mark(source.version + ' repetition ' + repetition + ': identical original task/baseline/tests/response algorithm and budgets; actual prompt overhead and host approvals recorded');
|
|
}
|
|
assert.equal(results.length, 8); assert.equal(comparisons.length, 4);
|
|
} else for (const kind of contextMode ? ['native-context'] : ['node', 'react', 'python']) results.push(await runSample(kind));
|
|
assert.deepEqual(inputs(), frozen);
|
|
report = { success: true, targetVersion: version, testedVersion: version, protocol: responsesMode ? 'responses-stream' : 'chat-completions', directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
|
|
scope: 'Pinned actual Harness and workbench execution with an owned HTTP model fixture. Checks, approvals, failures, parent merges, Edge and Git are real. Model reasoning and Purge benefit remain separate.' };
|
|
} catch (error) { report = { success: false, targetVersion: version, testedVersion: version, directory, installation, cases, results, inputs: frozen, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
|
|
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (contextMode ? (responsesMode ? 'responses-context' : 'controlled-context') : responsesMode ? 'responses-engineering' : desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
|