27 lines
2.6 KiB
JavaScript
27 lines
2.6 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import {hash} from './desktop-f-state.mjs';
|
|
export const batch=path.resolve('.qa/live-retest-e-20261102-c'),setup=JSON.parse(fs.readFileSync(path.join(batch,'setup.json')));
|
|
export async function control(route,input){const boot=await(await fetch(setup.origin+'/api/bootstrap')).json();
|
|
const r=await fetch(setup.origin+'/api'+route,{method:input===undefined?'GET':'POST',headers:{'X-Workbench-Token':boot.token,'Content-Type':'application/json',Connection:'close'},
|
|
...(input===undefined?{}:{body:JSON.stringify(input)}),signal:AbortSignal.timeout(30000)});const v=await r.json();assert.ok(r.ok,route+': '+(v.error??r.status));return v;}
|
|
export async function batchBudget(){const center=await control('/model-center'),original=new Set(setup.originalModelCenter.attempts.map(a=>a.id));
|
|
const attempts=center.attempts.filter(a=>!original.has(a.id));
|
|
const reported=attempts.reduce((n,a)=>n+(a.usage?.total_tokens??0),0);
|
|
assert.ok(attempts.length<100,'Batch request limit reached');assert.ok(reported<700000,'Batch reported token limit reached');
|
|
assert.deepEqual(center.selected,{type:'connection',id:setup.connection.id});
|
|
return {requests:attempts.length,reportedTokens:reported};}
|
|
export function assertRuntime(){const identity=JSON.parse(fs.readFileSync(path.join(batch,'runtime-identity.json')));
|
|
for(const f of identity.files)assert.equal(hash(fs.readFileSync(path.join(identity.runtime,f.path))),f.sha256,f.path);}
|
|
export async function beginSample(name,tokens){assertRuntime();await batchBudget();
|
|
const center=await control('/model-center'),c=center.connections.find(c=>c.id===setup.connection.id);
|
|
const saved=JSON.parse(fs.readFileSync(path.join(batch,'connection.json')));assert.equal(c.revision,saved.revision);assert.equal(c.contextLimit,32000);assert.equal(c.hasKey,true);
|
|
const originalConfig=setup.originalConfig;
|
|
await control('/config',{...originalConfig,maxSteps:24,contextLimit:32000,maxRunDurationMs:600000,maxRunTokens:tokens,maxTaskTokens:tokens});
|
|
await control('/model-center/select',{type:'connection',id:c.id});
|
|
const boot=await control('/bootstrap');assert.equal(boot.runtimeRoot,setup.runtime);assert.equal(boot.config.maxTaskTokens,tokens);
|
|
const contract={name,tokens,requests:24,durationMs:600000,connectionId:c.id,revision:c.revision,model:c.model,protocol:c.protocol,contextLimit:32000,
|
|
createdAt:new Date().toISOString(),originalImplementationSupplied:false,automaticReplay:false,sourceIdentity:setup.identitySha256};
|
|
fs.writeFileSync(path.join(batch,name+'-budget.json'),JSON.stringify(contract,null,2)+'\n',{flag:'wx'});return contract;}
|