71 lines
6.2 KiB
JavaScript
71 lines
6.2 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import { workerPath, workerPathKey } from './workers.mjs';
|
|
|
|
export const EVALUATION_ACTIVE = new Set(['preparing', 'baseline_running', 'candidate_running', 'collecting', 'adopting']);
|
|
export const evaluationConflict = message => Object.assign(new Error(message), { status: 409 });
|
|
export const evaluationHash = value => createHash('sha256').update(typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value)).digest('hex');
|
|
const object = (value, keys, label) => {
|
|
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(key => !keys.includes(key))) throw evaluationConflict(label + '字段无效。');
|
|
};
|
|
const text = (value, limit, label) => {
|
|
if (typeof value !== 'string' || !value.trim() || value.length > limit || !value.isWellFormed() || /[\x00-\x08\x0b\x0c\x0e-\x1f]/.test(value)) throw evaluationConflict(label + '不能为空或超出长度范围。');
|
|
return value.trim();
|
|
};
|
|
const integer = (value, fallback, min, max, label) => {
|
|
const result = value ?? fallback;
|
|
if (!Number.isSafeInteger(result) || result < min || result > max) throw evaluationConflict(label + '超出允许范围。');
|
|
return result;
|
|
};
|
|
export function evaluationSourcePath(value) {
|
|
const relative = workerPath(value);
|
|
if (!/\.(?:[cm]?[jt]sx?|py|go|rs|cs|vue|css|html)$/.test(relative)
|
|
|| /(?:^|\/)(?:tests?|__tests__|fixtures?|scripts?|\.github|runtime|releases|vendor|node_modules)(?:\/|$)/i.test(relative)
|
|
|| /(?:^|[./_-])(?:test|spec|config|setup|conftest)(?:[./_-]|$)/i.test(relative)) throw evaluationConflict('候选范围仅接受明确的源码文件;测试、运行声明和验收脚本保持原输入。');
|
|
return relative;
|
|
}
|
|
export function evaluationInput(input, config) {
|
|
object(input, ['title', 'samples', 'budget', 'evidenceMode'], '评测');
|
|
if (!Array.isArray(input.samples) || !input.samples.length || input.samples.length > 3) throw evaluationConflict('每批评测需要1到3个固定样本。');
|
|
const count = input.samples.length;
|
|
const samples = input.samples.map(item => {
|
|
object(item, ['label', 'goal', 'criteria', 'paths', 'checks'], '样本');
|
|
if (!Array.isArray(item.paths) || !item.paths.length || item.paths.length > 16) throw evaluationConflict('样本需要1到16个明确源码文件。');
|
|
const paths = item.paths.map(evaluationSourcePath).sort();
|
|
if (new Set(paths.map(workerPathKey)).size !== paths.length) throw evaluationConflict('源码文件不能重复。');
|
|
if (!Array.isArray(item.checks) || !item.checks.length || item.checks.length > 6 || item.checks.some(name => typeof name !== 'string' || !/^(?:test(?::[a-z0-9_-]+)?|typecheck|check|lint|build|python:(?:test|compile))$/.test(name))
|
|
|| new Set(item.checks).size !== item.checks.length) throw evaluationConflict('检查需要1到6个不同的已识别名称。');
|
|
return { label: text(item.label, 64, '样本名称'), goal: text(item.goal, 6000, '需求'), criteria: text(item.criteria, 4000, '验收条件'), paths, checks: [...item.checks] };
|
|
});
|
|
if (new Set(samples.map(item => item.label)).size !== count) throw evaluationConflict('样本名称不能重复。');
|
|
const raw = input.budget ?? {}; object(raw, ['maxTokens', 'maxSteps', 'maxToolCalls', 'maxDurationMs'], '总预算');
|
|
const maximumTokens = Math.min(config.maxTaskTokens ?? 1000000, (config.maxRunTokens ?? 200000) * count);
|
|
const budget = {
|
|
maxTokens: integer(raw.maxTokens, maximumTokens, count * 2000, maximumTokens, '总token预算'),
|
|
maxSteps: integer(raw.maxSteps, Math.min(200, (config.maxSteps ?? 12) * count), count * 2, 200, '总请求步数'),
|
|
maxToolCalls: integer(raw.maxToolCalls, Math.min(512, count * 64), count * 2, 512, '总工具次数'),
|
|
maxDurationMs: integer(raw.maxDurationMs, config.maxRunDurationMs ?? 600000, count * 1000, Math.min(3600000, config.maxRunDurationMs ?? 600000), '总执行时限'),
|
|
};
|
|
const evidenceMode = input.evidenceMode ?? 'configured-model';
|
|
if (!['configured-model', 'local-fixture'].includes(evidenceMode)) throw evaluationConflict('评测来源需要明确为实际配置模型或本地固定响应。');
|
|
return { title: text(input.title, 120, '评测名称'), samples, budget, evidenceMode };
|
|
}
|
|
export function evaluationSeal(batch) {
|
|
return { schemaVersion: 1, id: batch.id, projectId: batch.projectId, sessionId: batch.sessionId, taskId: batch.taskId, runId: batch.runId,
|
|
title: batch.title, source: batch.source, budget: batch.budget, evidenceMode: batch.evidenceMode, model: batch.model,
|
|
samples: batch.samples.map(({ id, label, goal, criteria, paths, checks }) => ({ id, label, goal, criteria, paths, checks })) };
|
|
}
|
|
export function evaluationSampleSeal(batch, sample) {
|
|
return { schemaVersion: 1, batchId: batch.id, batchFingerprint: batch.fingerprint, sampleId: sample.id, groupId: sample.groupId,
|
|
baseCommit: batch.source.head, definition: evaluationSeal(batch).samples.find(value => value.id === sample.id),
|
|
files: sample.files, commands: sample.commands, children: sample.children,
|
|
...(sample.environmentInputs ? { environmentInputs:sample.environmentInputs } : {}) };
|
|
}
|
|
export function evaluationSummary(batch) {
|
|
const outcomes = batch.samples.map(sample => sample.report?.summary ?? { original: sample.baselineAttemptedAt ? 'unknown' : 'pending', candidate: ['failed','stopped','unknown'].includes(sample.status) ? sample.status : ['candidate_running','candidate_complete','awaiting_review','paused','collecting','adopting','sealed'].includes(sample.status) ? 'unknown' : 'pending' });
|
|
const count = (role, status) => outcomes.filter(item => item[role] === status).length;
|
|
return { samples: batch.samples.length, finished: batch.samples.filter(item => !!item.report).length,
|
|
original: Object.fromEntries(['passed', 'failed', 'stopped', 'unknown', 'pending'].map(status => [status, count('original', status)])),
|
|
candidate: Object.fromEntries(['passed', 'failed', 'stopped', 'unknown', 'pending'].map(status => [status, count('candidate', status)])),
|
|
inferenceValidated: false, scope: batch.evidenceMode === 'local-fixture' ? '固定响应仅验证执行链和工程检查,不计模型自主能力。' : '保留当前固定样本的实际结果,不外推任意工程成功率。' };
|
|
}
|