Files
Agent-Workbench/scripts/live-z/analyze.mjs
T

52 lines
3.3 KiB
JavaScript

// Offline analysis of the original receipts; never reruns a failed request.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { root, save, sha, experiment } from './common.mjs';
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
const preflight = read('preflight/result.json');
const requests = [];
const add = (stage, attempts, captures) => {
for (const attempt of attempts) {
const response = captures.find(value => value.attemptId === attempt.id && value.kind === 'response');
let done = false, finishReasons = [], reasoningDeltas = 0;
if (response) {
const raw = fs.readFileSync(path.join(root, response.path));
assert.equal(raw.length, response.bytes); assert.equal(sha(raw), response.sha256);
const text = raw.toString('utf8'); done = text.includes('[DONE]') || text.includes('event: message_stop');
for (const line of text.split(/\r?\n/).filter(value => value.startsWith('data: ') && value !== 'data: [DONE]')) {
const chunk = JSON.parse(line.slice(6));
for (const choice of chunk.choices ?? []) {
if (choice.finish_reason) finishReasons.push(choice.finish_reason);
if (choice.delta?.reasoning_content) reasoningDeltas++;
}
if (chunk.delta?.stop_reason) finishReasons.push(chunk.delta.stop_reason);
}
}
requests.push({ stage, id: attempt.id, status: attempt.status, requested: attempt.model,
reported: attempt.modelIdentity?.reportedModels, identity: attempt.modelIdentity?.status,
identityComplete: attempt.modelIdentity?.complete, usageStatus: attempt.usageStatus, usage: attempt.usage,
requestUrl: attempt.requestUrl, protocol: attempt.protocol, done, finishReasons, reasoningDeltas });
}
};
add('preflight', read('preflight/attempts.json'), preflight.captures);
let node;
if (fs.existsSync(path.join(root, 'node/result.json'))) {
node = read('node/result.json');
assert.equal(sha(fs.readFileSync(path.join(root, 'node/goal.txt'))), 'b5421d0bb502126f49a5153729027f13ef82db86ab19a9e5b0c23a49070bb905');
add('node', read('node/attempts.json'), node.captures);
}
const result = { route: experiment.name, preflightPassed: preflight.passed,
engineeringStarted: !!node, engineeringSuccess: node?.success ?? false, requests,
requestCount: requests.length, reportedTokens: requests.reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0),
everyReportedIdentityMatched: requests.every(value => value.identity === 'matched' && value.identityComplete),
everyAttemptCompleted: requests.every(value => value.status === 'completed'),
note: 'Model identity, transport/stream completion and engineering success are separate observations.',
toolCallsOnPreflight: preflight.actualTools, failure: node?.failure, noAutomaticReplay: true, verifierModelRequests: 0 };
save('analysis.json', result);
console.log(JSON.stringify({ route: result.route, requestCount: result.requestCount,
reportedTokens: result.reportedTokens, engineeringSuccess: result.engineeringSuccess,
everyReportedIdentityMatched: result.everyReportedIdentityMatched,
everyAttemptCompleted: result.everyAttemptCompleted, lastFinish: requests.at(-1)?.finishReasons,
lastReasoningTokens: requests.at(-1)?.usage?.completion_tokens_details?.reasoning_tokens }));