import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; const root = path.resolve('docs/live-context-o-evidence'); const sha = bytes => createHash('sha256').update(bytes).digest('hex'); const file = name => { const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target; }; const read = name => JSON.parse(fs.readFileSync(file(name), 'utf8')); const manifest = read('manifest.json'); assert.equal(manifest.kind, 'owned-live-context-o-evidence'); const names = new Set(); let bytes = 0; for (const row of manifest.files) { assert.ok(!names.has(row.path)); names.add(row.path); const content = fs.readFileSync(file(row.path)); assert.equal(content.length, row.bytes); assert.equal(sha(content), row.sha256, row.path); bytes += content.length; } const owned = reference => { assert.ok(names.has(reference)); return read(reference); }; const real = [owned(manifest.references.preflight), owned(manifest.references.originalNode), owned(manifest.references.fixedNode)]; let requests = 0, reportedTokens = 0, unknownRequests = 0, estimatedUnknownTokens = 0; for (const [index, result] of real.entries()) { const count = result.requests ?? result.usage.requests; assert.ok(count <= (index === 0 ? 1 : 8)); const total = result.usage.total_tokens ?? result.usage.totalTokens; assert.ok(total <= (index === 0 ? 16000 : 90000)); const unknown = result.usage.unknownRequests ?? 0; const estimatedUnknown = result.usage.estimatedUnknownTokens ?? 0; assert.ok(total + estimatedUnknown <= (index === 0 ? 16000 : 90000)); requests += count; reportedTokens += total; unknownRequests += unknown; estimatedUnknownTokens += estimatedUnknown; assert.equal(result.modelWeightsAuthenticated, false); const folder = index === 0 ? 'preflight-resumed' : index === 1 ? 'node' : 'node-after-fix'; const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, count); let actualTotal = 0, actualUnknown = 0; for (const attempt of attempts) { assert.equal(attempt.connectionId, '93d0cf6d-4f32-4918-9da8-6f125aa8a6e5'); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1); assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions'); assert.equal(attempt.requestUrl, 'https://api.apikey.fan/v1/chat/completions'); const request = result.captures.find(c => c.attemptId === attempt.id && c.kind === 'request'); assert.ok(request); const requestBytes = fs.readFileSync(file('run/' + request.path)); assert.equal(sha(requestBytes), request.sha256); assert.equal(request.sha256, attempt.requestSha256); assert.equal(JSON.parse(requestBytes).model, attempt.model); const response = result.captures.find(c => c.attemptId === attempt.id && c.kind === 'response'); assert.ok(response); const responseBytes = fs.readFileSync(file('run/' + response.path)); assert.equal(sha(responseBytes), response.sha256); assert.equal(response.sha256, attempt.responseSha256); if (attempt.status !== 'completed') { assert.equal(result.success, false); assert.equal(index, 2); assert.equal(attempt.status, 'failed'); assert.equal(attempt.httpStatus, 502); assert.equal(attempt.usage, null); assert.equal(attempt.usageStatus, 'unknown'); actualUnknown++; assert.equal(attempt.modelIdentity.status, 'incomplete'); assert.equal(attempt.modelIdentity.complete, false); assert.equal(attempt.diagnostic.origin, 'upstream'); assert.equal(attempt.diagnostic.retryAfterMs, 60000); assert.equal(attempt.diagnostic.upstream.headers['content-type'], 'text/html; charset=UTF-8'); assert.match(responseBytes.toString('utf8'), /502: Bad gateway/); assert.equal(attempt.diagnostic.upstream.artifactSha256, response.sha256); assert.equal(attempt.diagnostic.upstream.capture.complete, true); continue; } const events = responseBytes.toString('utf8').split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trim()).filter(line => line && line !== '[DONE]').map(line => JSON.parse(line)); const models = [...new Set(events.map(event => event.model).filter(Boolean))]; assert.deepEqual(models, ['gpt-6.1-sol']); const usage = events.findLast(event => event.usage)?.usage; assert.ok(usage); assert.equal(usage.prompt_tokens, attempt.usage.prompt_tokens); assert.equal(usage.completion_tokens, attempt.usage.completion_tokens); actualTotal += usage.total_tokens ?? usage.prompt_tokens + usage.completion_tokens; assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true); } assert.equal(actualTotal, total); assert.equal(actualUnknown, unknown); if (index > 0) { const restoration = owned('run/' + folder + '/restoration.json'); assert.equal(restoration.passed, true); assert.equal(restoration.originalCredentialRevision, 1); assert.equal(restoration.finalCredentialRevision, 1); if (index === 2) { assert.equal(restoration.originalPublicProjectsExact, true); assert.equal(restoration.originalAttemptsExact, owned('run/node-after-fix/original-attempts.json').length); assert.equal(restoration.rawConfigExceptRevisionExact, true); assert.equal(restoration.registryExceptDeclaredRevisionTimestampOrderExact, true); assert.equal(restoration.otherMetadataRowsExact, true); } } } assert.equal(real[0].passed, true); assert.equal(real[0].actualTools, 0); assert.equal(real[1].success, false); assert.equal(real[1].stopReason, 'context_limit'); assert.equal(real[1].usage.requests, 2); assert.equal(real[1].usage.totalTokens, 11269); assert.equal(real[2].success, false); assert.equal(real[2].state, 'failed'); assert.equal(real[2].usage.requests, 2); assert.equal(real[2].usage.totalTokens, 4838); assert.equal(real[2].usage.unknownRequests, 1); assert.equal(real[2].usage.estimatedUnknownTokens, 11095); assert.deepEqual(real[2].approvals, []); assert.deepEqual(real[2].applications, []); assert.equal(real[2].modelImplementationSuppliedBySupervisor, false); for (const name of ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']) { assert.deepEqual(fs.readFileSync(file('run/node/source-before/' + name)), fs.readFileSync(file('run/node/source-after/' + name))); assert.deepEqual(fs.readFileSync(file('run/node-after-fix/source-before/' + name)), fs.readFileSync(file('run/node-after-fix/source-after/' + name))); } const formal = owned(manifest.references.formal), freeze = owned(manifest.references.freeze), release = owned(manifest.references.release); assert.equal(formal.success, true); assert.equal(formal.pid, 36552); assert.equal(formal.hasKey, true); assert.equal(formal.version, '1.24.5'); assert.equal(formal.sourceCommit, manifest.sourceCommit); assert.equal(freeze.sourceCommit, manifest.sourceCommit); assert.equal(release.documentation.sourceCommit, manifest.sourceCommit); assert.equal(formal.retention.originalRows, 'exact'); assert.equal(formal.retention.originalExports, 'exact'); assert.equal(formal.retention.originalArtifactBytes, 'exact'); assert.equal(formal.retention.originalModelCenter, 'exact'); const browserReview = owned('run/release/browser-review-result.json'), nativeReview = owned('run/release/native-review-result.json'); assert.equal(browserReview.success, true); assert.equal(browserReview.views.length, 2); assert.equal(browserReview.actualImagesReviewed, true); assert.deepEqual(browserReview.errors, []); assert.deepEqual(browserReview.rejectedWrites, []); assert.equal(nativeReview.success, true); assert.equal(nativeReview.version, manifest.version); assert.equal(nativeReview.checked.length, 3); assert.equal(nativeReview.noTaskStart, true); assert.equal(nativeReview.realModelRequests, 0); const archives = owned('release/archives-validation.json'), portable = owned('release/release-validation.json'); assert.equal(archives.success, true); assert.equal(archives.version, manifest.version); assert.equal(archives.archives.length, 3); for (const archive of archives.archives) { assert.equal(archive.crcPassed, true); assert.equal(archive.manifestAndDirectoryMatch, true); } assert.equal(portable.success, true); assert.equal(portable.appRoot, release.portableRoot); const rebuild = owned('release/independent-source-rebuild/result.json'); assert.equal(rebuild.success, true); assert.equal(rebuild.sourceCommit, manifest.sourceCommit); assert.equal(rebuild.inputCount, 552); assert.deepEqual(rebuild.mismatches, []); for (const log of rebuild.logs) assert.equal(sha(fs.readFileSync(file('release/independent-source-rebuild/' + log.name + '.log'))), log.sha256); const installation = owned('release/isolated-install/result.json'); assert.equal(installation.success, true); assert.equal(installation.version, manifest.version); assert.equal(installation.results.length, 12); for (const { proof, script } of installation.installedUI.proofs) { assert.equal(proof.success, true); assert.equal(proof.version, manifest.version); assert.equal(proof.realModelRequests, 0); assert.equal(proof.checks.length, script === 'verify-file-results-ui.mjs' ? 16 : 9); assert.deepEqual(proof.errors, []); owned('ui/installed/' + path.basename(proof.directory ?? proof.root) + '/result.json'); } let sourceUiGroups = 0; for (const [folder, count] of [['model-selection-k-3bd2d5b6-b0c2-4592-89a7-5221f614cc91', 7], ['model-identity-ui-71450f91-9e8f-48d7-b7b9-ea6c69bb1b28', 3], ['task-plan-ui-5b6fee37-1b8d-46e4-984f-f4202e6a66e3', 8], ['messages-ui-m-c3f7030c-d9c0-4eba-a608-b5c4c9429180', 7]]) { const proof = owned('ui/source/' + folder + '/result.json'); assert.equal(proof.success, true); assert.equal(proof.version ?? proof.testedVersion, manifest.version); assert.equal((proof.checks ?? proof.passed).length, count); sourceUiGroups += count; } for (const label of ['source', 'portable', 'installed']) { const result = owned('run/compiled-context-' + label + '/result.json'); assert.equal(result.success, true); assert.equal(result.requests, 3); assert.equal(result.externalModelRequests, 0); assert.equal(result.contextLimit, 16000); assert.equal(result.outputReservation, 2048); assert.equal(result.safetyTokens, 1600); assert.equal(result.rawToolEvidenceVerified, true); assert.equal(result.stopReason, 'verification_incomplete'); } if (real[2].success) { assert.ok(real[2].commit); assert.equal(real[2].tests.total, 9); assert.equal(real[2].tests.passed, 9); assert.equal(real[2].tests.failed, 0); assert.equal(real[2].identitiesValid, true); } console.log(JSON.stringify({ success: true, manifestSha256: sha(fs.readFileSync(file('manifest.json'))), files: manifest.files.length, bytes, requests, reportedTokens, unknownRequests, estimatedUnknownTokens, originalNodeDelivered: false, fixedNodeDelivered: real[2].success, sourceUiGroups, isolatedInstallerCases: installation.results.length, installedFileUiCases: 25, originalPurgeBenefitTested: false, sourceCommit: manifest.sourceCommit, candidate: manifest.candidate, scope: 'Offline original-byte, outgoing/requested/reported identifiers, usage, bounded samples and declared restoration checks. No model requests, credential access, task replay or weight authentication.' }));