fix(engine): bound native history and preserve verified continuation context

This commit is contained in:
Agent Workbench Dev
2026-11-02 19:42:29 +08:00
parent b67b4f2aad
commit 0a58c0d09f
19 changed files with 600 additions and 37 deletions
+43 -12
View File
@@ -15,22 +15,24 @@ import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
import { desktopProjectQa } from './desktop-project-qa.mjs';
import { specification } from '../tests/fixtures/engineering-specification.mjs';
import { responsesEngineeringHttp } from '../tests/fixtures/responses-engineering-http.mjs';
import { nativeContextSpecification } from '../tests/fixtures/native-context-project.mjs';
const args = process.argv.slice(2);
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop', '--responses'].includes(value)), 'Unknown option; use --help.');
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop', '--responses', '--context-16k'].includes(value)), 'Unknown option; use --help.');
if (args.includes('--help')) {
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison | --desktop] [--responses] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. --responses uses actual Responses SSE and ModelHub routing with the same engineering algorithm. Comparison verifies eight independent Node projects. Set AGENT_WORKBENCH_DEEPSEEK_ROOT and AGENT_WORKBENCH_TEST_PYTHON; comparison requires both Purge source roots. No configured-model requests.'); process.exit(0);
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison | --desktop | --context-16k] [--responses] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. --context-16k uses a new copy of the sealed three-module/nine-test pilot, original failure/log paging and a complete reviewed batch at 16K. --responses uses actual Responses SSE and ModelHub routing. Set AGENT_WORKBENCH_DEEPSEEK_ROOT and AGENT_WORKBENCH_TEST_PYTHON; context mode needs no Python. No configured-model requests.'); process.exit(0);
}
const comparisonMode = args.includes('--purge-comparison'), desktopMode = args.includes('--desktop');
const contextMode = args.includes('--context-16k');assert.ok(!contextMode || !comparisonMode && !desktopMode, '16K bridge acceptance is a separate run.');
const responsesMode = args.includes('--responses'); assert.ok(!responsesMode || !comparisonMode, 'Protocol acceptance and Purge comparison remain separate.');
assert.ok(!comparisonMode || !desktopMode, 'Desktop engineering and prompt comparison are separate acceptance runs.');
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
const directory = path.resolve('.qa', (desktopMode ? 'desktop-projects-' : comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
const directory = path.resolve('.qa', (contextMode ? 'controlled-context-' : desktopMode ? 'desktop-projects-' : comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT);
assert.equal(installation.controlledStatus, 'available', installation.message);
const python = process.env.AGENT_WORKBENCH_TEST_PYTHON;
if (!comparisonMode) assert.ok(python && path.isAbsolute(python) && fs.existsSync(python), 'Provide the existing isolated Python 3.10+ interpreter.');
if (!comparisonMode && !contextMode) assert.ok(python && path.isAbsolute(python) && fs.existsSync(python), 'Provide the existing isolated Python 3.10+ interpreter.');
const sources = comparisonMode ? ['1.1.78', '1.1.81'].map((expected, index) => {
const root = process.env[index === 0 ? 'AGENT_WORKBENCH_PURGE_178_ROOT' : 'AGENT_WORKBENCH_PURGE_181_ROOT'];
assert.ok(root && path.isAbsolute(root), 'Provide both existing Purge source directories.');
@@ -52,7 +54,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
fs.mkdirSync(sample); let runtime, group, result, desktopQa, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
try {
const baseline = kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
const baseline = contextMode ? 'docs/live-model-purge-1.24-evidence/source-baselines/r1-off' : kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
write(f, '.gitignore', 'node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n');
write(f, 'owner-staged.txt', 'Owner baseline\n'); write(f, 'owner-unstaged.txt', 'Owner baseline\n');
@@ -61,7 +63,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
const baselineTree = git(f, ['rev-parse', 'HEAD^{tree}']).trim();
write(f, 'owner-staged.txt', 'Owner staged change\n'); git(f, ['add', '--', 'owner-staged.txt']);
write(f, 'owner-staged.txt', 'Owner working copy after staging\n'); write(f, 'owner-unstaged.txt', 'Owner unstaged change\n'); write(f, 'owner-untracked.txt', 'Owner untracked file\n');
const spec = specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort();
const spec = contextMode ? nativeContextSpecification : specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort();
const original = ownerBytes(f), protectedNames = git(f, ['ls-files', '-z']).split('\0').filter(file => file && !writable.includes(file) && !file.startsWith('owner-'));
const protectedFiles = Object.fromEntries(protectedNames.map(file => [file, fs.readFileSync(path.join(f.projectRoot, file))]));
const verify = root => { for (const [file, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(root, file)), bytes, 'Protected ' + file); };
@@ -81,7 +83,22 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === worker.taskId).at(-1);
const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Fixed local acceptance repair; retain all tests and declarations.' }));
let calls = [];
if (session.changes.some(change => change.status === 'pending')) { /* Explicit review follows native completion. */ }
if(contextMode){
const pending=session.changes.filter(change=>change.status==='pending');
const batch=[...f.store.core.change_set.values()].findLast(set=>set.taskId===worker.taskId&&set.status==='prepared');
if(pending.length){if(!batch)calls=[call(randomUUID(),'prepare_change_set',{change_ids:pending.map(change=>change.id)})];}
else if(phase==='new'){
calls=[...writable,'test/expense.test.mjs'].map(file=>call(randomUUID(),'read_file',{path:file,limit:16000}));phase='observed';
}else if(phase!=='fixed'&&job){
assert.equal(job.status,'failed');assert.equal(job.results[0].tests.failed,9);
if(phase!=='failure-log-read'){
calls=[call(randomUUID(),'read_artifact',{artifact_id:job.results[0].logArtifactId,offset:0,limit:16000})];phase='failure-log-read';
}else{calls=propose(spec.fixes);phase='fixed';}
}else if(phase==='fixed'&&job?.status==='passed')assert.equal(job.results[0].tests.passed,9);
else if(!plan||job?.planId===plan.id)calls=[call(randomUUID(),'plan_checks',{names:spec.checks})];
else calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
}
else if (session.changes.some(change => change.status === 'pending')) { /* Explicit review follows native completion. */ }
else if (phase === 'new' && spec.initial) { calls = propose(spec.initial); phase = 'initial'; }
else if (phase !== 'fixed' && job) {
assert.equal(job.status, 'failed', job.summary); assert.ok(job.results.some(result => result.tests?.failed > 0)); calls = propose(spec.fixes); phase = 'fixed';
@@ -94,11 +111,12 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
return { choices: [{ message: { role: 'assistant', content: calls.length ? '' : 'Owned native run complete; host proof is still required.', ...(calls.length ? { tool_calls: calls } : {}) },
finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 123, completion_tokens: 45, total_tokens: 168 } };
});
f.store.saveConfig({ mode: 'compatible', model: 'owned-controlled-engineering', baseUrl: model.baseUrl, contextLimit: 64000 });
f.store.saveConfig({ mode: 'compatible', model: 'owned-controlled-engineering', baseUrl: model.baseUrl, contextLimit: contextMode ? 16000 : 64000,
...(contextMode ? {maxOutputTokens:2048} : {}) });
const provider = new Provider(); provider.setKey('self-owned-controlled-model-fixture');
if (responsesMode) {
const hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned Responses engineering', protocol: 'responses', model: 'owned-controlled-engineering', baseUrl: model.baseUrl,
requiresKey: false, contextLimit: 64000, streaming: true, capabilityDeclarations: { streaming: 'supported' } });
requiresKey: false, contextLimit: contextMode ? 16000 : 64000, ...(contextMode ? {maxOutputTokens:2048} : {}), streaming: true, capabilityDeclarations: { streaming: 'supported' } });
hub.select({ type: 'connection', id: connection.id });
}
runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
@@ -117,7 +135,10 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
const session = f.store.session(child.sessionId), ids = session.changes.filter(change => change.status === 'pending').map(change => change.id);
if (ids.length) {
assert.equal(group.status, 'awaiting_review', group.summary); const set = desktopQa ? await desktopQa.review(session, ids) : await runtime.changeSetManager().prepare(session, ids);
assert.equal(group.status, 'awaiting_review', group.summary);
const nativeBatch=contextMode&&[...f.store.core.change_set.values()].findLast(set=>set.taskId===child.taskId&&set.status==='prepared');
const set = nativeBatch || (desktopQa ? await desktopQa.review(session, ids) : await runtime.changeSetManager().prepare(session, ids));
if(contextMode)assert.deepEqual(set.files.map(file=>file.path).sort(),writable);
reviews.push({ operation: 'review child batch', id: set.id, fingerprint: set.fingerprint, paths: set.files.map(file => file.path) });
if (!desktopQa) await (await runtime.changeSetManager().start(session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary);
}
@@ -161,6 +182,16 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
fs.writeFileSync(path.join(sample, 'original-responses-attempts.json'), JSON.stringify(attempts, null, 2));
}
mark(label + ': actual Harness uses owned model/tool bridges, retains failing tests, and completes reviewed repairs with one accounting entry per HTTP request');
if(contextMode){
const bounded=engineRuns.flatMap(job=>job.requests.map(request=>verifyEngineModelInput(f.store,childSession,job,request))).filter(input=>input.view.mode==='bounded');
assert.ok(bounded.length>0,'The original long failure/log history must exercise the bounded bridge.');
assert.ok(bounded.every(input=>input.view.beforeInputTokens>input.view.inputBudget&&input.plan.estimatedInputTokens<=input.view.inputBudget));
const sent=JSON.stringify(bounded.map(input=>input.body));assert.ok(sent.includes('logArtifactId'));assert.ok(sent.includes('failed'));
const failed=childChecks.find(job=>job.status==='failed'),log=f.store.core.artifact.get(failed.results[0].logArtifactId);
assert.ok(f.store.artifactStore.read(log).includes('fail 9'));assert.ok(engineRuns.some(job=>job.calls.some(call=>call.name==='read_artifact')));
assert.ok(engineRuns.some(job=>job.calls.some(call=>call.name==='prepare_change_set')));assert.ok(engineRuns.every(job=>!job.inputRejections?.length));
mark('16K: original nine failures and paged log survive verified native compaction; all three proposals form one reviewed batch without replay');
}
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'collected', group.summary);
const collection = await manager.readCollection(f.session, group); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'merge_prepared', group.summary);
@@ -241,9 +272,9 @@ try {
mark(source.version + ' repetition ' + repetition + ': identical original task/baseline/tests/response algorithm and budgets; actual prompt overhead and host approvals recorded');
}
assert.equal(results.length, 8); assert.equal(comparisons.length, 4);
} else for (const kind of ['node', 'react', 'python']) results.push(await runSample(kind));
} else for (const kind of contextMode ? ['native-context'] : ['node', 'react', 'python']) results.push(await runSample(kind));
assert.deepEqual(inputs(), frozen);
report = { success: true, targetVersion: version, testedVersion: version, protocol: responsesMode ? 'responses-stream' : 'chat-completions', directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
scope: 'Pinned actual Harness and workbench execution with an owned HTTP model fixture. Checks, approvals, failures, parent merges, Edge and Git are real. Model reasoning and Purge benefit remain separate.' };
} catch (error) { report = { success: false, targetVersion: version, testedVersion: version, directory, installation, cases, results, inputs: frozen, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (responsesMode ? 'responses-engineering' : desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (contextMode ? (responsesMode ? 'responses-context' : 'controlled-context') : responsesMode ? 'responses-engineering' : desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
+52
View File
@@ -0,0 +1,52 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { chromium,expect } from 'playwright/test';
import { fixture,launchWorkbench } from '../tests/helpers.mjs';
import { controlledWorker } from '../tests/fixtures/controlled-worker.mjs';
// A declared UI state fixture. Native execution and capacity are independently
// checked by verify-controlled-projects --context-16k, not by this sample.
const directory=path.resolve('.qa','engine-context-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),cases=[],screenshots=[],errors=[];
let browser,host,report;
try{
const context=controlledWorker(f),group=await context.manager.plan(f.session,context.definition);
await (await context.manager.prepare(f.session,group.id,group.fingerprint)).promise;
assert.equal(group.status,'ready',group.summary);
const child=group.children[0],session=f.store.session(child.sessionId),run=f.store.core.run.get(session.runId);
run.engine='deepseek-workbench';run.budgetPause={schema:1,reason:'engine_input_budget',contextLimit:16000,outputReservation:2048,safetyTokens:1600,
inputBudget:12352,requiredInputTokens:12437,originalInputTokens:13110,runRemainingTokens:198488,taskRemainingTokens:198488,
breakdown:{requestOverheadTokens:40,toolDefinitionTokens:2704,fixedInstructionTokens:7818,retainedHistoryAndViewTokens:1875}};
session.status='paused';f.store.finishRun(session,run,'engine_input_budget');f.store.save(session);
await context.runtime.stopAll();f.store.close();
host=await launchWorkbench(t,f.data);assert.notEqual(new URL(host.url).port,'4318');
browser=await chromium.launch({channel:'msedge',headless:true});
for(const width of [1440,390]){
const page=await browser.newPage({viewport:{width,height:1000}});page.on('pageerror',error=>errors.push(error.message));
await page.goto(host.url);await page.waitForLoadState('networkidle');
const project=page.getByLabel('当前项目',{exact:true});
if(!await project.isVisible())await page.getByRole('button',{name:'展开项目栏',exact:true}).click();
await project.selectOption(child.projectId);
const panel=page.getByLabel('任务预算诊断',{exact:true});await expect(panel).toBeVisible();
await expect(panel).toContainText('原生引擎输入或剩余预算不足');await expect(panel).toContainText('12,437');
await expect(panel).toContainText('完整目标与原生提示');await expect(panel).toContainText('198,488');await expect(panel).not.toContainText('不限');
await expect(panel.getByRole('button',{name:'核对预算并续接原任务',exact:true})).toHaveCount(0);
await page.reload();await expect(page.getByLabel('任务预算诊断',{exact:true})).toContainText('13,110');
const screenshot=path.join(directory,'native-budget-'+width+'.png');await panel.scrollIntoViewIfNeeded();await page.screenshot({path:screenshot,fullPage:true});screenshots.push(screenshot);
fs.writeFileSync(path.join(directory,'native-budget-'+width+'.html'),await page.content());
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+1),'No viewport overflow');
await panel.getByRole('button',{name:'打开隔离协作',exact:true}).click();
const dialog=page.getByRole('dialog',{name:'隔离协作',exact:true});await expect(dialog).toContainText('当前是受限子任务');
await dialog.getByRole('button',{name:'返回父目标',exact:true}).click();await expect(project).toHaveValue(group.projectId);
const original=await host.api('/sessions/'+child.sessionId);assert.equal(original.execution.task.usage.requests,0);
assert.equal(original.execution.run.budgetPause.reason,'engine_input_budget');assert.equal(original.execution.progress.totalTools,0);
cases.push(width+': native diagnostic persists, names exact capacity, and returns to the parent without dispatch or unsupported resume');await page.close();
}
assert.deepEqual(errors,[]);report={passed:true,cases,screenshots,errors,realModelRequests:0,scope:'Declared paused UI fixture; native execution is separate.'};
}catch(error){report={passed:false,cases,screenshots,errors,error:error.stack,realModelRequests:0};process.exitCode=1;}
finally{
await browser?.close();await host?.close();for(const dispose of disposers.reverse())await dispose();
fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));console.log(JSON.stringify({...report,directory}));
}
+8 -2
View File
@@ -2,6 +2,7 @@ import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { completionBody, requestMetrics } from '../src/shared/contracts/model-request.mjs';
import { nativeMessagesToChat } from '../src/shared/contracts/engine-execution.mjs';
import { nativeSystemTexts } from '../src/core/engines/purge-strategy.mjs';
@@ -33,7 +34,12 @@ function visit(relative='') {
}
}
visit();assert.deepEqual(files.sort(),[...entries.keys(),'manifest.json'].sort(),'Exact sealed file scope');
for(const input of manifest.sourceInputs){const data=fs.readFileSync(input.path);assert.equal(data.length,input.bytes,input.path);assert.equal(sha(data),input.sha256,'Source used for offline reconstruction: '+input.path);}
assert.match(manifest.sourceCommit,/^[a-f0-9]{40}$/);
for(const input of manifest.sourceInputs){
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$/);assert.ok(!input.path.split('/').includes('..'));
const data=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:8*1024*1024});
assert.equal(data.length,input.bytes,input.path);assert.equal(sha(data),input.sha256,'Historical source used by the sealed pilot: '+input.path);
}
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,381);
for(const a of artifacts.values()){assert.equal(entries.get(a.file)?.sha256,a.sha256,a.id);assert.equal(entries.get(a.file)?.bytes,a.bytes,a.id);}
const setup=from('receipts/setup-public-receipt.json'),analysis=from('receipts/analysis.json'),preservation=from('receipts/state-preservation.json');
@@ -89,7 +95,7 @@ assert.equal(manifest.requests,requests);assert.equal(analysis.requests,requests
assert.equal(analysis.aggregate.enabled.reportedTokens,138758);assert.equal(analysis.aggregate.disabled.reportedTokens,78875);
for(const source of reconciliation.sourceReceipts)for(const [file,identity]of Object.entries(source.files))assert.equal(entries.get('source-baselines/'+source.sample+'/'+file)?.sha256,identity.sha256);
console.log(JSON.stringify({passed:true,manifestSha256:sha(fs.readFileSync(path.join(root,'manifest.json'))),evidenceFiles:manifest.count,evidenceBytes:manifest.bytes,
sourceInputs:manifest.sourceInputs.length,samples:manifest.sampleCount,requests,sseResponses,reportedTokens,unknownUsageRequests:unknown,
sourceInputs:manifest.sourceInputs.length,sourceCommit:manifest.sourceCommit,samples:manifest.sampleCount,requests,sseResponses,reportedTokens,unknownUsageRequests:unknown,
supervision:{fixedCheckApprovals:approvals,appliedBatches:applications,continuations,gitConfirmations:0,targetedImplementationHints:0},
promptComparison:{delivered:0,enabledReportedTokens:138758,disabledReportedTokens:78875,nativeInputRejections:4},originalPluginBenefit:'not_tested',
scope:'Offline byte/source/request/usage/input reconstruction only; no model request, no original failure rewrite, no claim that autonomous delivery passed.'}));