Files
Agent-Workbench/docs/reliability-1.25.2-evidence/source-bytes/scripts/live-x/preflight.mjs.txt
T

71 lines
6.1 KiB
Plaintext

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { publicSnapshot, hash } from './state.mjs';
import { invokeGptProbe, inspectorClosed } from './inspector.mjs';
import { root, origin, dataRoot, connectionId, save, meta, client, captureAttempts, readRoot } from './common.mjs';
// Single no-tool GPT identity preflight. One request max, strict identity, stop on failure.
assert.equal(fs.existsSync(path.join(root, 'preflight')), false, 'Never replay or overwrite a real sample.');
const original = readRoot();
const { api, boot, close } = await client();
let fatal;
try {
await inspectorClosed();
assert.equal(boot.version, '1.25.2');
const host = await api('/host/status'); assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false);
const before = await publicSnapshot(origin, dataRoot()); save('preflight/before.json', before);
assert.deepEqual(before.active, []);
const connection = before.modelCenter.connections.find(value => value.id === connectionId);
assert.equal(connection.enabled, true); assert.equal(connection.hasKey, true); assert.equal(connection.credentialRevision, 1);
assert.equal(connection.protocol, 'chat-completions'); assert.equal(connection.model, 'gpt-6.1-sol');
assert.equal(connection.revision, original.connectionRevision);
assert.equal(connection.contextLimit, 16000); assert.equal(connection.modelIdentityPolicy ?? 'record', 'record');
const originalAttempts = meta('modelAttempts') ?? []; save('preflight/original-attempts.json', originalAttempts);
const nonce = randomUUID(), prompt = `这是独立GPT连接身份预检,不是工程任务。只回复 OK ${nonce},不要调用工具、读写文件或执行命令。`;
save('preflight/contract.json', { kind: 'single-real-GPT-identity-preflight', connectionId, connection,
version: boot.version, runtimeRoot: boot.runtimeRoot, maxRequests: 1, maxAttempts: 1, maxOutputTokens: 256,
maxTokens: 16000, contextLimit: 16000, toolDefinitions: 0, toolExecution: 0,
poolSwitching: false, automaticReplay: false, modelIdentityPolicy: 'require-match in this experiment snapshot only',
noGlobalConfigurationWrite: true, noCredentialExtractionOrResave: true, prompt, nonce,
originalBaselineSha256: hash(fs.readFileSync(path.join(root, 'preflight/before.json'))) });
const projectRoot = path.join(root, 'preflight/identity-only'); fs.mkdirSync(projectRoot);
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# X GPT identity preflight\nNo engineering or command execution.\n', { flag: 'wx' });
const project = await api('/projects', { root: projectRoot });
const session = await api('/sessions', { projectId: project.id }); save('preflight/owned.json', { project, session });
const input = { pid: host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, sessionId: session.id, projectId: project.id,
connectionId, connectionRevision: connection.revision, credentialRevision: connection.credentialRevision,
baseUrl: connection.baseUrl, model: connection.model, title: 'X · jojocode GPT连接单次预检', prompt };
save('preflight/input.json', input);
console.log(JSON.stringify({ stage: 'single-GPT-preflight', connectionId, model: connection.model, requestUrl: connection.baseUrl + '/chat/completions', maxRequests: 1, toolDefinitions: 0 }));
const invoked = await invokeGptProbe(input); save('preflight/expression.js', invoked.expression); save('preflight/host-result.json', invoked.result);
const previous = new Set(originalAttempts.map(value => value.id));
const attempts = (meta('modelAttempts') ?? []).filter(value => !previous.has(value.id)); save('preflight/attempts.json', attempts);
assert.ok(attempts.length <= 1); for (const attempt of attempts) { assert.equal(attempt.sessionId, session.id); assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.poolId, null); }
const state = await api('/sessions/' + session.id); save('preflight/session.json', state);
save('preflight/export.json', await api('/sessions/' + session.id + '/export'));
const captures = await captureAttempts(api, attempts, 'preflight/captures');
const attempt = attempts[0];
const passed = attempts.length === 1 && attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched'
&& attempt.modelIdentity.complete === true && attempt.modelIdentity.policy === 'require-match' && attempt.responseRedacted === false
&& attempt.usageStatus === 'reported' && attempt.usage.total_tokens <= 16000 && captures.length === 2
&& invoked.result.actualTools === 0 && state.status === 'completed' && invoked.result.message?.content?.includes(nonce);
const result = { passed, engineeringTestAllowed: passed, testedAt: new Date().toISOString(), version: boot.version,
connectionId, sessionId: session.id, projectId: project.id, taskId: invoked.result.taskId,
requestedModel: connection.model, reportedModels: attempt?.modelIdentity?.reportedModels ?? [], identity: attempt?.modelIdentity,
requestUrl: attempt?.requestUrl, requests: attempts.length, usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched',
actualTools: invoked.result.actualTools, stopReason: invoked.result.stopReason, captures,
noRetryOrPoolSwitch: attempts.every(value => value.ordinal === 1 && value.poolId === null),
modelWeightsAuthenticated: false, engineeringDeliveryTested: false, purgeBenefitTested: false,
defaultSelectionChanged: false, connectionConfigChanged: false, credentialValuesReadOrSaved: false };
save('preflight/result.json', result); console.log(JSON.stringify({ passed, requests: result.requests, usage: result.usage, reportedModels: result.reportedModels, stopReason: result.stopReason }));
} catch (error) {
fatal = { name: error.name, message: error.message, stack: error.stack, recordedAt: new Date().toISOString(), automaticReplay: false };
save('preflight/driver-error.json', fatal); process.exitCode = 1;
} finally {
save('preflight/after.json', await publicSnapshot(origin, dataRoot()));
save('preflight/inspector-final.json', { closed: await inspectorClosed() });
await close();
}
if (fatal) console.error(fatal.message);