73 lines
7.0 KiB
JavaScript
73 lines
7.0 KiB
JavaScript
// Analyze observed wire shape, public tools and source coverage. Do not inspect
|
|
// or evaluate private reasoning semantics; only count its returned characters.
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { root, save, sha, dataRoot } from './common.mjs';
|
|
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
|
const state = read('node/session.json'), outcome = read('node/model-outcome.json'), attempts = read('node/attempts.json');
|
|
const readCapture = (attempt, kind) => fs.readFileSync(path.join(root, outcome.captures.find(c => c.attemptId === attempt.id && c.kind === kind).path));
|
|
const requests = attempts.map((attempt, index) => {
|
|
const request = JSON.parse(readCapture(attempt, 'request')), calls = new Map(), finishes = [];
|
|
let terminalSeen = false, textCharacters = 0, reasoningCharacters = 0;
|
|
for (const line of readCapture(attempt, 'response').toString('utf8').split(/\r?\n/)) {
|
|
if (!line.startsWith('data: ')) continue;
|
|
const data = line.slice(6); if (data === '[DONE]') { terminalSeen = true; continue; }
|
|
const event = JSON.parse(data);
|
|
for (const choice of event.choices ?? []) {
|
|
if (choice.finish_reason) finishes.push(choice.finish_reason);
|
|
textCharacters += (choice.delta?.content ?? '').length; reasoningCharacters += (choice.delta?.reasoning_content ?? '').length;
|
|
for (const fragment of choice.delta?.tool_calls ?? []) {
|
|
const call = calls.get(fragment.index) ?? { name: '', arguments: '' };
|
|
call.name += fragment.function?.name ?? ''; call.arguments += fragment.function?.arguments ?? ''; calls.set(fragment.index, call);
|
|
}
|
|
}
|
|
}
|
|
assert.deepEqual(request.thinking, { type: 'enabled' }); assert.equal(request.reasoning_effort, 'low'); assert.equal(request.max_tokens, 2048);
|
|
return { ordinal: index + 1, attemptId: attempt.id, status: attempt.status, identity: attempt.modelIdentity, usage: attempt.usage,
|
|
parameters: { thinking: request.thinking, reasoning_effort: request.reasoning_effort, max_tokens: request.max_tokens },
|
|
finishes, terminalSeen, textCharacters, reasoningCharacters,
|
|
calls: [...calls.values()].map(call => { let input, argumentsComplete = false; try { input = JSON.parse(call.arguments); argumentsComplete = true; } catch {}
|
|
return { name: call.name, argumentCharacters: call.arguments.length, argumentsComplete, ...(call.name === 'read_file' ? { input } : {}) }; }) };
|
|
});
|
|
const fileReads = state.events.filter(e => e.kind === 'tool' && e.tool === 'read_file').map(event => {
|
|
const result = JSON.parse(event.output), source = fs.readFileSync(path.join(root, 'node/source-before', result.path), 'utf8');
|
|
assert.equal(sha(source), result.sha256); assert.equal(source.slice(result.offset, result.next_offset), result.content);
|
|
return { input: event.input, artifactId: event.artifactId, path: result.path, sha256: result.sha256, range: [result.offset, result.next_offset],
|
|
characters: result.returned_characters, total: result.total_characters, truncated: result.truncated };
|
|
});
|
|
const duplicateRanges = fileReads.filter((row, index) => fileReads.slice(0, index).some(old => old.path === row.path && old.sha256 === row.sha256 && JSON.stringify(old.range) === JSON.stringify(row.range)));
|
|
const lastRequest = JSON.parse(readCapture(attempts.at(-1), 'request'));
|
|
const workingMessage = lastRequest.messages.find(message => typeof message.content === 'string' && message.content.includes('"kind":"task-working-set"'));
|
|
assert.ok(workingMessage); const working = JSON.parse(workingMessage.content.slice(workingMessage.content.indexOf('{')));
|
|
const retainedFiles = working.data.entries.filter(entry => entry.observation?.kind === 'file').map(entry => {
|
|
const source = fs.readFileSync(path.join(root, 'node/source-before', entry.observation.path), 'utf8');
|
|
assert.equal(sha(entry.content), entry.content_sha256); assert.equal(entry.content, source.slice(...entry.sent_range));
|
|
return { path: entry.observation.path, sentRange: entry.sent_range, totalSourceCharacters: source.length,
|
|
retainedCharacters: entry.content.length, sourceSha256: entry.observation.sha256, rangeComplete: entry.file_read?.range_complete ?? !entry.truncated };
|
|
});
|
|
const before = read('node/before.json'), after = read('node/after.json'), oldArtifacts = new Set(before.persistent.artifacts.map(a => a.path));
|
|
const added = after.persistent.artifacts.filter(a => !oldArtifacts.has(a.path));
|
|
for (const row of added) {
|
|
const bytes = fs.readFileSync(path.join(dataRoot(), 'artifacts', row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
|
|
save('node/artifacts/' + row.path, bytes);
|
|
}
|
|
save('node/owned-artifacts.json', { files: added, originalPhysicalFiles: before.persistent.artifacts.length, allNewPhysicalFilesCaptured: true });
|
|
const checks = state.execution.checks.map(job => ({ id: job.id, status: job.status, results: job.results.map(result => ({ name: result.name, status: result.status, tests: result.tests, durationMs: result.durationMs, logArtifactId: result.logArtifactId })) }));
|
|
const analysis = { passed: true, purpose: 'Preservation and failure analysis; engineering did not pass', requests, fileReads,
|
|
exactDuplicateReadRanges: duplicateRanges.length, retainedFilesInLastActualRequest: retainedFiles,
|
|
currentCheckJobPresentInLastRequest: checks.every(job => JSON.stringify(lastRequest).includes(job.id)),
|
|
sourceFilesChanged: state.changes.length, checks, actualToolCalls: state.events.filter(e => e.kind === 'tool').length,
|
|
supervision: { checkApprovals: outcome.approvals.length, applications: outcome.applications.length, reviewContinuations: Math.max(0, outcome.continuations.length - 1), budgetAllocations: outcome.continuations.length },
|
|
elapsedSeconds: outcome.elapsedSeconds, reportedCacheHitTokens: attempts.reduce((sum, a) => sum + (a.usage?.prompt_cache_hit_tokens ?? 0), 0),
|
|
finalReservation: state.execution.contextPlans.at(-1)?.tokenReservation, ownedPhysicalArtifacts: added.length,
|
|
privateReasoningSemanticsReviewed: false, modelRequestsInAnalysis: 0, automaticReplay: false,
|
|
conclusions: { stoppedBy: 'upstream finish_reason=length at the sealed 2048 output ceiling; DONE was seen',
|
|
modelStartedImplementation: true, acceptedProposals: 0, lastResponseToolCalls: 'one complete propose_file followed by a second incomplete propose_file; the response was rejected as a whole',
|
|
all2048TokensWereReasoning: 'not supported: the last usage lacks a reasoning-token split and the stream contains tool arguments',
|
|
repeatReadingResolvedGenerally: false, reactPythonExecuted: false, originalPurgeBenefitTested: false } };
|
|
assert.equal(analysis.currentCheckJobPresentInLastRequest, true); assert.equal(analysis.sourceFilesChanged, 0);
|
|
save('node/analysis.json', analysis);
|
|
console.log(JSON.stringify({ passed: true, requests: requests.length, exactDuplicateReadRanges: duplicateRanges.length, retainedFiles, checks,
|
|
lastResponse: requests.at(-1), ownedPhysicalArtifacts: added.length, reportedCacheHitTokens: analysis.reportedCacheHitTokens, realModelRequests: 0 }));
|