fix: bind observed models and calibrate every model dispatch budget
This commit is contained in:
@@ -52,7 +52,7 @@ const frozen = inputs(), mark = message => { cases.push(message); console.log('P
|
||||
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
|
||||
const startedAt = Date.now(), label = sampleName;
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
|
||||
fs.mkdirSync(sample); let runtime, group, result, desktopQa, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
|
||||
fs.mkdirSync(sample); let runtime, group, result, desktopQa, model, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
|
||||
try {
|
||||
const baseline = contextMode ? 'docs/live-model-purge-1.24-evidence/source-baselines/r1-off' : kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
|
||||
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
@@ -74,7 +74,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
};
|
||||
if (kind === 'react') install(f.projectRoot, 'parent');
|
||||
let phase = 'new';
|
||||
const model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
|
||||
model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
|
||||
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
|
||||
assertToolPairs(input.messages);
|
||||
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
|
||||
@@ -237,7 +237,7 @@ async function runSample(kind, { sampleName = kind, strategy = null, comparison
|
||||
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
|
||||
writable, reviews, modelRequests, ...(desktopQa ? { desktop: { events: desktopQa.events, screenshots: desktopQa.screenshots, scope: desktopQa.scope, errors: desktopQa.errors } } : {}), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
||||
return result;
|
||||
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, ownedFixtureErrors:model?.errors??[], engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
await desktopQa?.close();
|
||||
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
|
||||
Reference in New Issue
Block a user