feat: add fixed engineering evaluations and controlled candidate adoption
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { workerPath, workerPathKey } from './workers.mjs';
|
||||
|
||||
export const EVALUATION_ACTIVE = new Set(['preparing', 'baseline_running', 'candidate_running', 'collecting', 'adopting']);
|
||||
export const evaluationConflict = message => Object.assign(new Error(message), { status: 409 });
|
||||
export const evaluationHash = value => createHash('sha256').update(typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value)).digest('hex');
|
||||
const object = (value, keys, label) => {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(key => !keys.includes(key))) throw evaluationConflict(label + '字段无效。');
|
||||
};
|
||||
const text = (value, limit, label) => {
|
||||
if (typeof value !== 'string' || !value.trim() || value.length > limit || !value.isWellFormed() || /[\x00-\x08\x0b\x0c\x0e-\x1f]/.test(value)) throw evaluationConflict(label + '不能为空或超出长度范围。');
|
||||
return value.trim();
|
||||
};
|
||||
const integer = (value, fallback, min, max, label) => {
|
||||
const result = value ?? fallback;
|
||||
if (!Number.isSafeInteger(result) || result < min || result > max) throw evaluationConflict(label + '超出允许范围。');
|
||||
return result;
|
||||
};
|
||||
export function evaluationSourcePath(value) {
|
||||
const relative = workerPath(value);
|
||||
if (!/\.(?:[cm]?[jt]sx?|py|go|rs|cs|vue|css|html)$/.test(relative)
|
||||
|| /(?:^|\/)(?:tests?|__tests__|fixtures?|scripts?|\.github|runtime|releases|vendor|node_modules)(?:\/|$)/i.test(relative)
|
||||
|| /(?:^|[./_-])(?:test|spec|config|setup|conftest)(?:[./_-]|$)/i.test(relative)) throw evaluationConflict('候选范围仅接受明确的源码文件;测试、运行声明和验收脚本保持原输入。');
|
||||
return relative;
|
||||
}
|
||||
export function evaluationInput(input, config) {
|
||||
object(input, ['title', 'samples', 'budget', 'evidenceMode'], '评测');
|
||||
if (!Array.isArray(input.samples) || !input.samples.length || input.samples.length > 3) throw evaluationConflict('每批评测需要1到3个固定样本。');
|
||||
const count = input.samples.length;
|
||||
const samples = input.samples.map(item => {
|
||||
object(item, ['label', 'goal', 'criteria', 'paths', 'checks'], '样本');
|
||||
if (!Array.isArray(item.paths) || !item.paths.length || item.paths.length > 16) throw evaluationConflict('样本需要1到16个明确源码文件。');
|
||||
const paths = item.paths.map(evaluationSourcePath).sort();
|
||||
if (new Set(paths.map(workerPathKey)).size !== paths.length) throw evaluationConflict('源码文件不能重复。');
|
||||
if (!Array.isArray(item.checks) || !item.checks.length || item.checks.length > 6 || item.checks.some(name => typeof name !== 'string' || !/^(?:test(?::[a-z0-9_-]+)?|typecheck|check|lint|build|python:(?:test|compile))$/.test(name))
|
||||
|| new Set(item.checks).size !== item.checks.length) throw evaluationConflict('检查需要1到6个不同的已识别名称。');
|
||||
return { label: text(item.label, 64, '样本名称'), goal: text(item.goal, 6000, '需求'), criteria: text(item.criteria, 4000, '验收条件'), paths, checks: [...item.checks] };
|
||||
});
|
||||
if (new Set(samples.map(item => item.label)).size !== count) throw evaluationConflict('样本名称不能重复。');
|
||||
const raw = input.budget ?? {}; object(raw, ['maxTokens', 'maxSteps', 'maxToolCalls', 'maxDurationMs'], '总预算');
|
||||
const maximumTokens = Math.min(config.maxTaskTokens ?? 1000000, (config.maxRunTokens ?? 200000) * count);
|
||||
const budget = {
|
||||
maxTokens: integer(raw.maxTokens, maximumTokens, count * 2000, maximumTokens, '总token预算'),
|
||||
maxSteps: integer(raw.maxSteps, Math.min(200, (config.maxSteps ?? 12) * count), count * 2, 200, '总请求步数'),
|
||||
maxToolCalls: integer(raw.maxToolCalls, Math.min(512, count * 64), count * 2, 512, '总工具次数'),
|
||||
maxDurationMs: integer(raw.maxDurationMs, config.maxRunDurationMs ?? 600000, count * 1000, Math.min(3600000, config.maxRunDurationMs ?? 600000), '总执行时限'),
|
||||
};
|
||||
const evidenceMode = input.evidenceMode ?? 'configured-model';
|
||||
if (!['configured-model', 'local-fixture'].includes(evidenceMode)) throw evaluationConflict('评测来源需要明确为实际配置模型或本地固定响应。');
|
||||
return { title: text(input.title, 120, '评测名称'), samples, budget, evidenceMode };
|
||||
}
|
||||
export function evaluationSeal(batch) {
|
||||
return { schemaVersion: 1, id: batch.id, projectId: batch.projectId, sessionId: batch.sessionId, taskId: batch.taskId, runId: batch.runId,
|
||||
title: batch.title, source: batch.source, budget: batch.budget, evidenceMode: batch.evidenceMode, model: batch.model,
|
||||
samples: batch.samples.map(({ id, label, goal, criteria, paths, checks }) => ({ id, label, goal, criteria, paths, checks })) };
|
||||
}
|
||||
export function evaluationSampleSeal(batch, sample) {
|
||||
return { schemaVersion: 1, batchId: batch.id, batchFingerprint: batch.fingerprint, sampleId: sample.id, groupId: sample.groupId,
|
||||
baseCommit: batch.source.head, definition: evaluationSeal(batch).samples.find(value => value.id === sample.id),
|
||||
files: sample.files, commands: sample.commands, children: sample.children };
|
||||
}
|
||||
export function evaluationSummary(batch) {
|
||||
const outcomes = batch.samples.map(sample => sample.report?.summary ?? { original: sample.baselineAttemptedAt ? 'unknown' : 'pending', candidate: ['failed','stopped','unknown'].includes(sample.status) ? sample.status : 'pending' });
|
||||
const count = (role, status) => outcomes.filter(item => item[role] === status).length;
|
||||
return { samples: batch.samples.length, finished: batch.samples.filter(item => !!item.report).length,
|
||||
original: Object.fromEntries(['passed', 'failed', 'stopped', 'unknown', 'pending'].map(status => [status, count('original', status)])),
|
||||
candidate: Object.fromEntries(['passed', 'failed', 'stopped', 'unknown', 'pending'].map(status => [status, count('candidate', status)])),
|
||||
inferenceValidated: false, scope: batch.evidenceMode === 'local-fixture' ? '固定响应仅验证执行链和工程检查,不计模型自主能力。' : '保留当前固定样本的实际结果,不外推任意工程成功率。' };
|
||||
}
|
||||
@@ -4,6 +4,7 @@ export const EXECUTION_SCHEMA = 1;
|
||||
export const ACTIVE_STATES = new Set(['running', 'awaiting_approval', 'stopping']);
|
||||
export const CORE_KINDS = ['task', 'run', 'step', 'tool', 'artifact', 'check_plan', 'check_job', 'delivery_plan', 'delivery_job', 'change_set', 'context_plan', 'workspace', 'research_target', 'research_job', 'engine_job', 'engine_run', 'browser_plan', 'browser_job', 'task_plan', 'task_proof', 'memory', 'task_checkpoint', 'mcp_job', 'collaboration', 'worker', 'input_attachment'];
|
||||
CORE_KINDS.push('resume_plan');
|
||||
CORE_KINDS.push('evaluation');
|
||||
export const timestamp = () => new Date().toISOString();
|
||||
export const isId = value => typeof value === 'string' && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value);
|
||||
|
||||
|
||||
@@ -59,7 +59,8 @@ export function collaborationInput(input, config) {
|
||||
|
||||
export function collaborationSeal(group) {
|
||||
return { schemaVersion: 1, id: group.id, projectId: group.projectId, sessionId: group.sessionId, taskId: group.taskId, runId: group.runId,
|
||||
goal: group.goal, budget: group.budget, model: group.model, config: group.config, source: group.source, children: group.children };
|
||||
goal: group.goal, budget: group.budget, model: group.model, config: group.config, source: group.source, children: group.children,
|
||||
...(group.evaluation ? { evaluation: group.evaluation } : {}) };
|
||||
}
|
||||
export function workerSeal(worker) {
|
||||
return { schemaVersion: 1, id: worker.id, collaborationId: worker.collaborationId, parentProjectId: worker.parentProjectId,
|
||||
@@ -67,7 +68,7 @@ export function workerSeal(worker) {
|
||||
projectId: worker.projectId, sessionId: worker.sessionId, taskId: worker.taskId, workspaceId: worker.workspaceId,
|
||||
label: worker.label, goal: worker.goal, paths: worker.paths, budget: worker.budget, model: worker.model, baseCommit: worker.baseCommit,
|
||||
baselineArtifactId: worker.baselineArtifactId, baselineSha256: worker.baselineSha256, baselineRunId: worker.baselineRunId,
|
||||
...(worker.engine ? { engine: worker.engine } : {}) };
|
||||
...(worker.engine ? { engine: worker.engine } : {}), ...(worker.evaluation ? { evaluation: worker.evaluation } : {}) };
|
||||
}
|
||||
export function workerForProject(store, projectId) {
|
||||
return [...(store.core.worker?.values() ?? [])].find(value => value.projectId === projectId) ?? null;
|
||||
@@ -83,7 +84,8 @@ export function workerForSession(store, session) {
|
||||
|| definition.workspaceId !== worker.workspaceId || JSON.stringify(definition.paths) !== JSON.stringify(worker.paths)
|
||||
|| JSON.stringify(definition.budget) !== JSON.stringify(worker.budget) || definition.goal !== worker.goal
|
||||
|| JSON.stringify(group.model) !== JSON.stringify(worker.model)
|
||||
|| JSON.stringify(definition.engine) !== JSON.stringify(worker.engine)) throw workerConflict('子任务与父任务的封存分工不一致。');
|
||||
|| JSON.stringify(definition.engine) !== JSON.stringify(worker.engine)
|
||||
|| JSON.stringify(definition.evaluation) !== JSON.stringify(worker.evaluation)) throw workerConflict('子任务与父任务的封存分工不一致。');
|
||||
const task = store.core.task.get(worker.taskId);
|
||||
if (!task || task.goal !== worker.goal || task.workerId !== worker.id || task.budget.maxTokens !== worker.budget.maxTokens) throw workerConflict('子任务目标、身份或累计预算已变化。');
|
||||
const original = store.owned('artifact', worker.contractArtifactId, session.id);
|
||||
@@ -98,6 +100,8 @@ export function assertWorkerWriteScope(store, session, value, { readOnly = false
|
||||
const worker = workerForSession(store, session);
|
||||
if (!worker) return;
|
||||
const portable = workerPath(value);
|
||||
if (!readOnly && worker.evaluation && store.core.evaluation?.get(worker.evaluation.batchId)?.samples.some(sample => sample.id === worker.evaluation.sampleId && sample.report)) throw workerConflict('评测样本已封存,不能再修改候选。');
|
||||
if (worker.evaluation && (worker.evaluation.role === 'original' || !worker.paths.some(path => workerPathKey(path) === workerPathKey(portable)))) throw workerConflict('原版仅供固定检查;候选只能修改封存的明确源码文件。');
|
||||
if (!pathInWorkerScope(portable, worker.paths)) throw workerConflict('文件超出子任务的固定修改范围:' + portable);
|
||||
if (!readOnly && ['unknown', 'completed', 'resolved_external'].includes(worker.status)) throw workerConflict('子任务已封存或结果待核对,不能修改原成果。');
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user