52 lines
3.3 KiB
JavaScript
52 lines
3.3 KiB
JavaScript
// Offline analysis of the original receipts; never reruns a failed request.
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { root, save, sha, experiment } from './common.mjs';
|
|
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
|
const preflight = read('preflight/result.json');
|
|
const requests = [];
|
|
const add = (stage, attempts, captures) => {
|
|
for (const attempt of attempts) {
|
|
const response = captures.find(value => value.attemptId === attempt.id && value.kind === 'response');
|
|
let done = false, finishReasons = [], reasoningDeltas = 0;
|
|
if (response) {
|
|
const raw = fs.readFileSync(path.join(root, response.path));
|
|
assert.equal(raw.length, response.bytes); assert.equal(sha(raw), response.sha256);
|
|
const text = raw.toString('utf8'); done = text.includes('[DONE]') || text.includes('event: message_stop');
|
|
for (const line of text.split(/\r?\n/).filter(value => value.startsWith('data: ') && value !== 'data: [DONE]')) {
|
|
const chunk = JSON.parse(line.slice(6));
|
|
for (const choice of chunk.choices ?? []) {
|
|
if (choice.finish_reason) finishReasons.push(choice.finish_reason);
|
|
if (choice.delta?.reasoning_content) reasoningDeltas++;
|
|
}
|
|
if (chunk.delta?.stop_reason) finishReasons.push(chunk.delta.stop_reason);
|
|
}
|
|
}
|
|
requests.push({ stage, id: attempt.id, status: attempt.status, requested: attempt.model,
|
|
reported: attempt.modelIdentity?.reportedModels, identity: attempt.modelIdentity?.status,
|
|
identityComplete: attempt.modelIdentity?.complete, usageStatus: attempt.usageStatus, usage: attempt.usage,
|
|
requestUrl: attempt.requestUrl, protocol: attempt.protocol, done, finishReasons, reasoningDeltas });
|
|
}
|
|
};
|
|
add('preflight', read('preflight/attempts.json'), preflight.captures);
|
|
let node;
|
|
if (fs.existsSync(path.join(root, 'node/result.json'))) {
|
|
node = read('node/result.json');
|
|
assert.equal(sha(fs.readFileSync(path.join(root, 'node/goal.txt'))), 'b5421d0bb502126f49a5153729027f13ef82db86ab19a9e5b0c23a49070bb905');
|
|
add('node', read('node/attempts.json'), node.captures);
|
|
}
|
|
const result = { route: experiment.name, preflightPassed: preflight.passed,
|
|
engineeringStarted: !!node, engineeringSuccess: node?.success ?? false, requests,
|
|
requestCount: requests.length, reportedTokens: requests.reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0),
|
|
everyReportedIdentityMatched: requests.every(value => value.identity === 'matched' && value.identityComplete),
|
|
everyAttemptCompleted: requests.every(value => value.status === 'completed'),
|
|
note: 'Model identity, transport/stream completion and engineering success are separate observations.',
|
|
toolCallsOnPreflight: preflight.actualTools, failure: node?.failure, noAutomaticReplay: true, verifierModelRequests: 0 };
|
|
save('analysis.json', result);
|
|
console.log(JSON.stringify({ route: result.route, requestCount: result.requestCount,
|
|
reportedTokens: result.reportedTokens, engineeringSuccess: result.engineeringSuccess,
|
|
everyReportedIdentityMatched: result.everyReportedIdentityMatched,
|
|
everyAttemptCompleted: result.everyAttemptCompleted, lastFinish: requests.at(-1)?.finishReasons,
|
|
lastReasoningTokens: requests.at(-1)?.usage?.completion_tokens_details?.reasoning_tokens }));
|