77 lines
6.6 KiB
JavaScript
77 lines
6.6 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { publicSnapshot, hash } from './state.mjs';
|
|
import { invokeProbe, inspectorClosed } from './inspector.mjs';
|
|
import { root, origin, dataRoot, connectionId, save, meta, client, captureAttempts, readRoot, experiment } from './common.mjs';
|
|
// Single no-tool provider identity preflight. One request max, strict identity, stop on failure.
|
|
assert.equal(fs.existsSync(path.join(root, 'preflight')), false, 'Never replay or overwrite a real sample.');
|
|
const original = readRoot();
|
|
assert.equal(experiment.key, 'deepseek');
|
|
const nativeParameters = experiment.parameters;
|
|
const prefix = 'preflight';
|
|
const { assertBatchBudget } = await import('./common.mjs');
|
|
assertBatchBudget();
|
|
const { api, boot, close } = await client();
|
|
let fatal;
|
|
try {
|
|
await inspectorClosed();
|
|
assert.equal(boot.version, '1.25.3');
|
|
const host = await api('/host/status'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false);
|
|
const before = await publicSnapshot(origin, dataRoot()); save(prefix + '/before.json', before);
|
|
assert.deepEqual(before.active, []);
|
|
const connection = before.modelCenter.connections.find(value => value.id === connectionId);
|
|
assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.credentialRevision, experiment.credentialRevision);
|
|
assert.equal(connection.protocol, experiment.protocol); assert.equal(connection.model, experiment.model);
|
|
assert.equal(connection.revision, original.connectionRevision);
|
|
assert.equal(connection.contextLimit, 16000); assert.equal(connection.modelIdentityPolicy ?? 'record', 'record');
|
|
const originalAttempts = meta('modelAttempts') ?? []; save(prefix + '/original-attempts.json', originalAttempts);
|
|
const targetName = connection.name + ' · low';
|
|
const nonce = randomUUID(), prompt = `这是独立模型连接身份预检,不是工程任务。只回复 OK ${nonce},不要调用工具、读写文件或执行命令。`;
|
|
save(prefix + '/contract.json', { kind: 'single-real-provider-identity-preflight', connectionId, connection,
|
|
version: boot.version, runtimeRoot: boot.runtimeRoot, maxRequests: 1, maxAttempts: 1, maxOutputTokens: 2048, nativeParameters,
|
|
maxTokens: 16000, contextLimit: 16000, toolDefinitions: 0, toolExecution: 0,
|
|
poolSwitching: false, automaticReplay: false, modelIdentityPolicy: 'require-match in this experiment snapshot only',
|
|
noGlobalConfigurationWrite: true, noCredentialExtractionOrResave: true, prompt, nonce,
|
|
originalBaselineSha256: hash(fs.readFileSync(path.join(root, prefix + '/before.json'))) });
|
|
const projectRoot = path.join(root, prefix + '/identity-only'); fs.mkdirSync(projectRoot);
|
|
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# AB provider identity preflight\nNo engineering or command execution.\n', { flag: 'wx' });
|
|
const project = await api('/projects', { root: projectRoot });
|
|
const session = await api('/sessions', { projectId: project.id }); save(prefix + '/owned.json', { project, session });
|
|
const input = { pid: host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, sessionId: session.id, projectId: project.id,
|
|
nativeParameters, connectionId, connectionRevision: connection.revision, credentialRevision: connection.credentialRevision,
|
|
baseUrl: connection.baseUrl, model: connection.model, protocol: connection.protocol, title: `AB · ${targetName}单次预检`, prompt };
|
|
save(prefix + '/input.json', input);
|
|
console.log(JSON.stringify({ stage: 'single-provider-preflight', connectionId, model: connection.model, protocol: connection.protocol, baseUrl: connection.baseUrl, maxRequests: 1, toolDefinitions: 0 }));
|
|
const invoked = await invokeProbe(input); save(prefix + '/expression.js', invoked.expression); save(prefix + '/host-result.json', invoked.result);
|
|
const previous = new Set(originalAttempts.map(value => value.id));
|
|
const attempts = (meta('modelAttempts') ?? []).filter(value => !previous.has(value.id)); save(prefix + '/attempts.json', attempts);
|
|
assert.ok(attempts.length <= 1); for (const attempt of attempts) { assert.equal(attempt.sessionId, session.id); assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.poolId, null); }
|
|
const state = await api('/sessions/' + session.id); save(prefix + '/session.json', state);
|
|
save(prefix + '/export.json', await api('/sessions/' + session.id + '/export'));
|
|
const captures = await captureAttempts(api, attempts, prefix + '/captures');
|
|
const attempt = attempts[0];
|
|
const passed = attempts.length === 1 && attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched'
|
|
&& attempt.modelIdentity.complete === true && attempt.modelIdentity.policy === 'require-match' && attempt.responseRedacted === false
|
|
&& attempt.usageStatus === 'reported' && attempt.usage.total_tokens <= 16000 && captures.length === 2
|
|
&& invoked.result.actualTools === 0 && state.status === 'completed' && invoked.result.message?.content?.includes(nonce);
|
|
const result = { passed, engineeringTestAllowed: passed, testedAt: new Date().toISOString(), version: boot.version, nativeParameters, reasoningMode: 'on-low',
|
|
connectionId, sessionId: session.id, projectId: project.id, taskId: invoked.result.taskId,
|
|
requestedModel: connection.model, reportedModels: attempt?.modelIdentity?.reportedModels ?? [], identity: attempt?.modelIdentity,
|
|
requestUrl: attempt?.requestUrl, requests: attempts.length, usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched',
|
|
actualTools: invoked.result.actualTools, stopReason: invoked.result.stopReason, captures,
|
|
noRetryOrPoolSwitch: attempts.every(value => value.ordinal === 1 && value.poolId === null),
|
|
modelWeightsAuthenticated: false, engineeringDeliveryTested: false, purgeBenefitTested: false,
|
|
defaultSelectionChanged: false, connectionConfigChanged: false, credentialValuesReadOrSaved: false };
|
|
save(prefix + '/result.json', result); console.log(JSON.stringify({ passed, requests: result.requests, usage: result.usage, reportedModels: result.reportedModels, stopReason: result.stopReason }));
|
|
} catch (error) {
|
|
fatal = { name: error.name, message: error.message, stack: error.stack, recordedAt: new Date().toISOString(), automaticReplay: false };
|
|
save(prefix + '/driver-error.json', fatal); process.exitCode = 1;
|
|
} finally {
|
|
save(prefix + '/after.json', await publicSnapshot(origin, dataRoot()));
|
|
save(prefix + '/inspector-final.json', { closed: await inspectorClosed() });
|
|
await close();
|
|
}
|
|
if (fatal) console.error(fatal.message);
|