Files
Agent-Workbench/scripts/live-ab/preflight.mjs
T
2026-11-05 03:23:44 +08:00

77 lines
6.6 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { publicSnapshot, hash } from './state.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
import { root, origin, dataRoot, connectionId, save, meta, client, captureAttempts, readRoot, experiment } from './common.mjs';
// Single no-tool provider identity preflight. One request max, strict identity, stop on failure.
assert.equal(fs.existsSync(path.join(root, 'preflight')), false, 'Never replay or overwrite a real sample.');
const original = readRoot();
assert.equal(experiment.key, 'deepseek');
const nativeParameters = experiment.parameters;
const prefix = 'preflight';
const { assertBatchBudget } = await import('./common.mjs');
assertBatchBudget();
const { api, boot, close } = await client();
let fatal;
try {
await inspectorClosed();
assert.equal(boot.version, '1.25.3');
const host = await api('/host/status'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false);
const before = await publicSnapshot(origin, dataRoot()); save(prefix + '/before.json', before);
assert.deepEqual(before.active, []);
const connection = before.modelCenter.connections.find(value => value.id === connectionId);
assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.credentialRevision, experiment.credentialRevision);
assert.equal(connection.protocol, experiment.protocol); assert.equal(connection.model, experiment.model);
assert.equal(connection.revision, original.connectionRevision);
assert.equal(connection.contextLimit, 16000); assert.equal(connection.modelIdentityPolicy ?? 'record', 'record');
const originalAttempts = meta('modelAttempts') ?? []; save(prefix + '/original-attempts.json', originalAttempts);
const targetName = connection.name + ' · low';
const nonce = randomUUID(), prompt = `这是独立模型连接身份预检,不是工程任务。只回复 OK ${nonce},不要调用工具、读写文件或执行命令。`;
save(prefix + '/contract.json', { kind: 'single-real-provider-identity-preflight', connectionId, connection,
version: boot.version, runtimeRoot: boot.runtimeRoot, maxRequests: 1, maxAttempts: 1, maxOutputTokens: 2048, nativeParameters,
maxTokens: 16000, contextLimit: 16000, toolDefinitions: 0, toolExecution: 0,
poolSwitching: false, automaticReplay: false, modelIdentityPolicy: 'require-match in this experiment snapshot only',
noGlobalConfigurationWrite: true, noCredentialExtractionOrResave: true, prompt, nonce,
originalBaselineSha256: hash(fs.readFileSync(path.join(root, prefix + '/before.json'))) });
const projectRoot = path.join(root, prefix + '/identity-only'); fs.mkdirSync(projectRoot);
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# AB provider identity preflight\nNo engineering or command execution.\n', { flag: 'wx' });
const project = await api('/projects', { root: projectRoot });
const session = await api('/sessions', { projectId: project.id }); save(prefix + '/owned.json', { project, session });
const input = { pid: host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, sessionId: session.id, projectId: project.id,
nativeParameters, connectionId, connectionRevision: connection.revision, credentialRevision: connection.credentialRevision,
baseUrl: connection.baseUrl, model: connection.model, protocol: connection.protocol, title: `AB · ${targetName}单次预检`, prompt };
save(prefix + '/input.json', input);
console.log(JSON.stringify({ stage: 'single-provider-preflight', connectionId, model: connection.model, protocol: connection.protocol, baseUrl: connection.baseUrl, maxRequests: 1, toolDefinitions: 0 }));
const invoked = await invokeProbe(input); save(prefix + '/expression.js', invoked.expression); save(prefix + '/host-result.json', invoked.result);
const previous = new Set(originalAttempts.map(value => value.id));
const attempts = (meta('modelAttempts') ?? []).filter(value => !previous.has(value.id)); save(prefix + '/attempts.json', attempts);
assert.ok(attempts.length <= 1); for (const attempt of attempts) { assert.equal(attempt.sessionId, session.id); assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.poolId, null); }
const state = await api('/sessions/' + session.id); save(prefix + '/session.json', state);
save(prefix + '/export.json', await api('/sessions/' + session.id + '/export'));
const captures = await captureAttempts(api, attempts, prefix + '/captures');
const attempt = attempts[0];
const passed = attempts.length === 1 && attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched'
&& attempt.modelIdentity.complete === true && attempt.modelIdentity.policy === 'require-match' && attempt.responseRedacted === false
&& attempt.usageStatus === 'reported' && attempt.usage.total_tokens <= 16000 && captures.length === 2
&& invoked.result.actualTools === 0 && state.status === 'completed' && invoked.result.message?.content?.includes(nonce);
const result = { passed, engineeringTestAllowed: passed, testedAt: new Date().toISOString(), version: boot.version, nativeParameters, reasoningMode: 'on-low',
connectionId, sessionId: session.id, projectId: project.id, taskId: invoked.result.taskId,
requestedModel: connection.model, reportedModels: attempt?.modelIdentity?.reportedModels ?? [], identity: attempt?.modelIdentity,
requestUrl: attempt?.requestUrl, requests: attempts.length, usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched',
actualTools: invoked.result.actualTools, stopReason: invoked.result.stopReason, captures,
noRetryOrPoolSwitch: attempts.every(value => value.ordinal === 1 && value.poolId === null),
modelWeightsAuthenticated: false, engineeringDeliveryTested: false, purgeBenefitTested: false,
defaultSelectionChanged: false, connectionConfigChanged: false, credentialValuesReadOrSaved: false };
save(prefix + '/result.json', result); console.log(JSON.stringify({ passed, requests: result.requests, usage: result.usage, reportedModels: result.reportedModels, stopReason: result.stopReason }));
} catch (error) {
fatal = { name: error.name, message: error.message, stack: error.stack, recordedAt: new Date().toISOString(), automaticReplay: false };
save(prefix + '/driver-error.json', fatal); process.exitCode = 1;
} finally {
save(prefix + '/after.json', await publicSnapshot(origin, dataRoot()));
save(prefix + '/inspector-final.json', { closed: await inspectorClosed() });
await close();
}
if (fatal) console.error(fatal.message);