// Analyze observed wire shape, public tools and source coverage. Do not inspect // or evaluate private reasoning semantics; only count its returned characters. import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { root, save, sha, dataRoot } from './common.mjs'; const read = name => JSON.parse(fs.readFileSync(path.join(root, name))); const state = read('node/session.json'), outcome = read('node/model-outcome.json'), attempts = read('node/attempts.json'); const readCapture = (attempt, kind) => fs.readFileSync(path.join(root, outcome.captures.find(c => c.attemptId === attempt.id && c.kind === kind).path)); const requests = attempts.map((attempt, index) => { const request = JSON.parse(readCapture(attempt, 'request')), calls = new Map(), finishes = []; let terminalSeen = false, textCharacters = 0, reasoningCharacters = 0; for (const line of readCapture(attempt, 'response').toString('utf8').split(/\r?\n/)) { if (!line.startsWith('data: ')) continue; const data = line.slice(6); if (data === '[DONE]') { terminalSeen = true; continue; } const event = JSON.parse(data); for (const choice of event.choices ?? []) { if (choice.finish_reason) finishes.push(choice.finish_reason); textCharacters += (choice.delta?.content ?? '').length; reasoningCharacters += (choice.delta?.reasoning_content ?? '').length; for (const fragment of choice.delta?.tool_calls ?? []) { const call = calls.get(fragment.index) ?? { name: '', arguments: '' }; call.name += fragment.function?.name ?? ''; call.arguments += fragment.function?.arguments ?? ''; calls.set(fragment.index, call); } } } assert.deepEqual(request.thinking, { type: 'enabled' }); assert.equal(request.reasoning_effort, 'low'); assert.equal(request.max_tokens, 2048); return { ordinal: index + 1, attemptId: attempt.id, status: attempt.status, identity: attempt.modelIdentity, usage: attempt.usage, parameters: { thinking: request.thinking, reasoning_effort: request.reasoning_effort, max_tokens: request.max_tokens }, finishes, terminalSeen, textCharacters, reasoningCharacters, calls: [...calls.values()].map(call => { let input, argumentsComplete = false; try { input = JSON.parse(call.arguments); argumentsComplete = true; } catch {} return { name: call.name, argumentCharacters: call.arguments.length, argumentsComplete, ...(call.name === 'read_file' ? { input } : {}) }; }) }; }); const fileReads = state.events.filter(e => e.kind === 'tool' && e.tool === 'read_file').map(event => { const result = JSON.parse(event.output), source = fs.readFileSync(path.join(root, 'node/source-before', result.path), 'utf8'); assert.equal(sha(source), result.sha256); assert.equal(source.slice(result.offset, result.next_offset), result.content); return { input: event.input, artifactId: event.artifactId, path: result.path, sha256: result.sha256, range: [result.offset, result.next_offset], characters: result.returned_characters, total: result.total_characters, truncated: result.truncated }; }); const duplicateRanges = fileReads.filter((row, index) => fileReads.slice(0, index).some(old => old.path === row.path && old.sha256 === row.sha256 && JSON.stringify(old.range) === JSON.stringify(row.range))); const lastRequest = JSON.parse(readCapture(attempts.at(-1), 'request')); const workingMessage = lastRequest.messages.find(message => typeof message.content === 'string' && message.content.includes('"kind":"task-working-set"')); assert.ok(workingMessage); const working = JSON.parse(workingMessage.content.slice(workingMessage.content.indexOf('{'))); const retainedFiles = working.data.entries.filter(entry => entry.observation?.kind === 'file').map(entry => { const source = fs.readFileSync(path.join(root, 'node/source-before', entry.observation.path), 'utf8'); assert.equal(sha(entry.content), entry.content_sha256); assert.equal(entry.content, source.slice(...entry.sent_range)); return { path: entry.observation.path, sentRange: entry.sent_range, totalSourceCharacters: source.length, retainedCharacters: entry.content.length, sourceSha256: entry.observation.sha256, rangeComplete: entry.file_read?.range_complete ?? !entry.truncated }; }); const before = read('node/before.json'), after = read('node/after.json'), oldArtifacts = new Set(before.persistent.artifacts.map(a => a.path)); const added = after.persistent.artifacts.filter(a => !oldArtifacts.has(a.path)); for (const row of added) { const bytes = fs.readFileSync(path.join(dataRoot(), 'artifacts', row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256); save('node/artifacts/' + row.path, bytes); } save('node/owned-artifacts.json', { files: added, originalPhysicalFiles: before.persistent.artifacts.length, allNewPhysicalFilesCaptured: true }); const checks = state.execution.checks.map(job => ({ id: job.id, status: job.status, results: job.results.map(result => ({ name: result.name, status: result.status, tests: result.tests, durationMs: result.durationMs, logArtifactId: result.logArtifactId })) })); const analysis = { passed: true, purpose: 'Preservation and failure analysis; engineering did not pass', requests, fileReads, exactDuplicateReadRanges: duplicateRanges.length, retainedFilesInLastActualRequest: retainedFiles, currentCheckJobPresentInLastRequest: checks.every(job => JSON.stringify(lastRequest).includes(job.id)), sourceFilesChanged: state.changes.length, checks, actualToolCalls: state.events.filter(e => e.kind === 'tool').length, supervision: { checkApprovals: outcome.approvals.length, applications: outcome.applications.length, reviewContinuations: Math.max(0, outcome.continuations.length - 1), budgetAllocations: outcome.continuations.length }, elapsedSeconds: outcome.elapsedSeconds, reportedCacheHitTokens: attempts.reduce((sum, a) => sum + (a.usage?.prompt_cache_hit_tokens ?? 0), 0), finalReservation: state.execution.contextPlans.at(-1)?.tokenReservation, ownedPhysicalArtifacts: added.length, privateReasoningSemanticsReviewed: false, modelRequestsInAnalysis: 0, automaticReplay: false, conclusions: { stoppedBy: 'upstream finish_reason=length at the sealed 2048 output ceiling; DONE was seen', modelStartedImplementation: true, acceptedProposals: 0, lastResponseToolCalls: 'one complete propose_file followed by a second incomplete propose_file; the response was rejected as a whole', all2048TokensWereReasoning: 'not supported: the last usage lacks a reasoning-token split and the stream contains tool arguments', repeatReadingResolvedGenerally: false, reactPythonExecuted: false, originalPurgeBenefitTested: false } }; assert.equal(analysis.currentCheckJobPresentInLastRequest, true); assert.equal(analysis.sourceFilesChanged, 0); save('node/analysis.json', analysis); console.log(JSON.stringify({ passed: true, requests: requests.length, exactDuplicateReadRanges: duplicateRanges.length, retainedFiles, checks, lastResponse: requests.at(-1), ownedPhysicalArtifacts: added.length, reportedCacheHitTokens: analysis.reportedCacheHitTokens, realModelRequests: 0 }));