fix(models): validate engine protocol budgets and freeze release acceptance inputs

This commit is contained in:
Agent Workbench Dev
2026-10-31 23:05:36 +08:00
parent 31ed219477
commit a72dc0fbc1
49 changed files with 21491 additions and 45 deletions
+20 -6
View File
@@ -13,13 +13,15 @@ import { resolvePurgeSource, assertPurgeModelInput, nativeSystemTexts, PURGE_DEF
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
import { desktopProjectQa } from './desktop-project-qa.mjs';
import { responsesEngineeringHttp } from '../tests/fixtures/responses-engineering-http.mjs';
const args = process.argv.slice(2);
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop'].includes(value)), 'Unknown option; use --help.');
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop', '--responses'].includes(value)), 'Unknown option; use --help.');
if (args.includes('--help')) {
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison | --desktop] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. Comparison verifies eight independent Node projects: two Purge versions, two repetitions, enabled/disabled. Set AGENT_WORKBENCH_DEEPSEEK_ROOT; default also requires AGENT_WORKBENCH_TEST_PYTHON, comparison requires AGENT_WORKBENCH_PURGE_178_ROOT / AGENT_WORKBENCH_PURGE_181_ROOT. No configured-model requests.'); process.exit(0);
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison | --desktop] [--responses] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. --responses uses actual Responses SSE and ModelHub routing with the same engineering algorithm. Comparison verifies eight independent Node projects. Set AGENT_WORKBENCH_DEEPSEEK_ROOT and AGENT_WORKBENCH_TEST_PYTHON; comparison requires both Purge source roots. No configured-model requests.'); process.exit(0);
}
const comparisonMode = args.includes('--purge-comparison'), desktopMode = args.includes('--desktop');
const responsesMode = args.includes('--responses'); assert.ok(!responsesMode || !comparisonMode, 'Protocol acceptance and Purge comparison remain separate.');
assert.ok(!comparisonMode || !desktopMode, 'Desktop engineering and prompt comparison are separate acceptance runs.');
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
const directory = path.resolve('.qa', (desktopMode ? 'desktop-projects-' : comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
@@ -93,7 +95,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
};
if (kind === 'react') install(f.projectRoot, 'parent');
let phase = 'new';
const model = await compatibleFixture(t, async (route, input) => {
const model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
assertToolPairs(input.messages);
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
@@ -116,7 +118,13 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 123, completion_tokens: 45, total_tokens: 168 } };
});
f.store.saveConfig({ mode: 'compatible', model: 'owned-controlled-engineering', baseUrl: model.baseUrl, contextLimit: 64000 });
const provider = new Provider(); provider.setKey('self-owned-controlled-model-fixture'); runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
const provider = new Provider(); provider.setKey('self-owned-controlled-model-fixture');
if (responsesMode) {
const hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned Responses engineering', protocol: 'responses', model: 'owned-controlled-engineering', baseUrl: model.baseUrl,
requiresKey: false, contextLimit: 64000, streaming: true, capabilityDeclarations: { streaming: 'supported' } });
hub.select({ type: 'connection', id: connection.id });
}
runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine-runs') });
runtime.checkRunner().pythonExecutable = python; const manager = runtime.workerManager();
if (desktopMode) desktopQa = await desktopProjectQa({ store: f.store, provider, runtime, sample, kind });
@@ -169,6 +177,12 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
}
assert.equal(f.store.core.task.get(child.taskId).usage.requests, modelRequests.length);
assert.equal(f.store.core.task.get(child.taskId).usage.totalTokens, modelRequests.length * 168);
if (responsesMode) {
const attempts = provider.modelHub(f.store).attempts;
assert.equal(attempts.length, model.requests.length); assert.ok(attempts.every(value => value.protocol === 'responses' && value.stream?.terminal && value.responseArtifactId));
fs.writeFileSync(path.join(sample, 'original-responses-requests.json'), JSON.stringify(model.requests, null, 2));
fs.writeFileSync(path.join(sample, 'original-responses-attempts.json'), JSON.stringify(attempts, null, 2));
}
mark(label + ': actual Harness uses owned model/tool bridges, retains failing tests, and completes reviewed repairs with one accounting entry per HTTP request');
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'collected', group.summary);
const collection = await manager.readCollection(f.session, group); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
@@ -252,7 +266,7 @@ try {
assert.equal(results.length, 8); assert.equal(comparisons.length, 4);
} else for (const kind of ['node', 'react', 'python']) results.push(await runSample(kind));
assert.deepEqual(inputs(), frozen);
report = { success: true, targetVersion: version, testedVersion: version, directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
report = { success: true, targetVersion: version, testedVersion: version, protocol: responsesMode ? 'responses-stream' : 'chat-completions', directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
scope: 'Pinned actual Harness and workbench execution with an owned HTTP model fixture. Checks, approvals, failures, parent merges, Edge and Git are real. Model reasoning and Purge benefit remain separate.' };
} catch (error) { report = { success: false, targetVersion: version, testedVersion: version, directory, installation, cases, results, inputs: frozen, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (responsesMode ? 'responses-engineering' : desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }