fix: bind observed models and calibrate every model dispatch budget
This commit is contained in:
@@ -282,12 +282,12 @@ export class ModelHub {
|
||||
}
|
||||
} finally { this.waiting--; }
|
||||
}
|
||||
async complete(config, body, signal, onProgress, plan, execute) {
|
||||
async complete(config, body, signal, onProgress, plan, execute,beforeDispatch) {
|
||||
const operation = this.provider.requestOperation?.(signal);
|
||||
try { return await this.completeOwned(config, body, operation?.signal ?? signal, onProgress, plan, execute); }
|
||||
try { return await this.completeOwned(config, body, operation?.signal ?? signal, onProgress, plan, execute,beforeDispatch); }
|
||||
finally { operation?.finish(); }
|
||||
}
|
||||
async completeOwned(config, body, signal, onProgress, plan, execute) {
|
||||
async completeOwned(config, body, signal, onProgress, plan, execute,beforeDispatch) {
|
||||
const routing = config.modelRouting, excluded = new Set(), deadline = Date.now() + Math.min(routing.queueTimeoutMs, 60000);
|
||||
const requestId = plan?.id ?? randomUUID(), runId = plan?.runId ?? requestId;
|
||||
const requirements = config.capabilityProbe ? [] : ['text', ...(requestVariants(body).some(value => value.tools?.length) ? ['tools'] : []),
|
||||
@@ -303,6 +303,7 @@ export class ModelHub {
|
||||
let lastError;
|
||||
for (let count = 1; count <= routing.maxAttempts; count++) {
|
||||
signal?.throwIfAborted();
|
||||
beforeDispatch?.({previousFailure:lastError?.diagnostic});
|
||||
let lease;
|
||||
try { lease = await this.acquire(eligibleRouting, runId, excluded, signal, deadline, requirements); }
|
||||
catch (cause) { throw lastError ?? cause; }
|
||||
|
||||
Reference in New Issue
Block a user