// Offline analysis of the original receipts; never reruns a failed request. import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { root, save, sha, experiment } from './common.mjs'; const read = name => JSON.parse(fs.readFileSync(path.join(root, name))); const preflight = read('preflight/result.json'); const requests = []; const add = (stage, attempts, captures) => { for (const attempt of attempts) { const response = captures.find(value => value.attemptId === attempt.id && value.kind === 'response'); let done = false, finishReasons = [], reasoningDeltas = 0; if (response) { const raw = fs.readFileSync(path.join(root, response.path)); assert.equal(raw.length, response.bytes); assert.equal(sha(raw), response.sha256); const text = raw.toString('utf8'); done = text.includes('[DONE]') || text.includes('event: message_stop'); for (const line of text.split(/\r?\n/).filter(value => value.startsWith('data: ') && value !== 'data: [DONE]')) { const chunk = JSON.parse(line.slice(6)); for (const choice of chunk.choices ?? []) { if (choice.finish_reason) finishReasons.push(choice.finish_reason); if (choice.delta?.reasoning_content) reasoningDeltas++; } if (chunk.delta?.stop_reason) finishReasons.push(chunk.delta.stop_reason); } } requests.push({ stage, id: attempt.id, status: attempt.status, requested: attempt.model, reported: attempt.modelIdentity?.reportedModels, identity: attempt.modelIdentity?.status, identityComplete: attempt.modelIdentity?.complete, usageStatus: attempt.usageStatus, usage: attempt.usage, requestUrl: attempt.requestUrl, protocol: attempt.protocol, done, finishReasons, reasoningDeltas }); } }; add('preflight', read('preflight/attempts.json'), preflight.captures); let node; if (fs.existsSync(path.join(root, 'node/result.json'))) { node = read('node/result.json'); assert.equal(sha(fs.readFileSync(path.join(root, 'node/goal.txt'))), 'b5421d0bb502126f49a5153729027f13ef82db86ab19a9e5b0c23a49070bb905'); add('node', read('node/attempts.json'), node.captures); } const result = { route: experiment.name, preflightPassed: preflight.passed, engineeringStarted: !!node, engineeringSuccess: node?.success ?? false, requests, requestCount: requests.length, reportedTokens: requests.reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0), everyReportedIdentityMatched: requests.every(value => value.identity === 'matched' && value.identityComplete), everyAttemptCompleted: requests.every(value => value.status === 'completed'), note: 'Model identity, transport/stream completion and engineering success are separate observations.', toolCallsOnPreflight: preflight.actualTools, failure: node?.failure, noAutomaticReplay: true, verifierModelRequests: 0 }; save('analysis.json', result); console.log(JSON.stringify({ route: result.route, requestCount: result.requestCount, reportedTokens: result.reportedTokens, engineeringSuccess: result.engineeringSuccess, everyReportedIdentityMatched: result.everyReportedIdentityMatched, everyAttemptCompleted: result.everyAttemptCompleted, lastFinish: requests.at(-1)?.finishReasons, lastReasoningTokens: requests.at(-1)?.usage?.completion_tokens_details?.reasoning_tokens }));