Files
Agent-Workbench/scripts/verify-live-context-o.mjs
T
2026-11-03 22:28:14 +08:00

135 lines
11 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
const root = path.resolve('docs/live-context-o-evidence');
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
const file = name => { const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target; };
const read = name => JSON.parse(fs.readFileSync(file(name), 'utf8'));
const manifest = read('manifest.json');
assert.equal(manifest.kind, 'owned-live-context-o-evidence');
const names = new Set(); let bytes = 0;
for (const row of manifest.files) {
assert.ok(!names.has(row.path)); names.add(row.path);
const content = fs.readFileSync(file(row.path)); assert.equal(content.length, row.bytes); assert.equal(sha(content), row.sha256, row.path); bytes += content.length;
}
const owned = reference => { assert.ok(names.has(reference)); return read(reference); };
const real = [owned(manifest.references.preflight), owned(manifest.references.originalNode), owned(manifest.references.fixedNode)];
let requests = 0, reportedTokens = 0, unknownRequests = 0, estimatedUnknownTokens = 0;
for (const [index, result] of real.entries()) {
const count = result.requests ?? result.usage.requests;
assert.ok(count <= (index === 0 ? 1 : 8));
const total = result.usage.total_tokens ?? result.usage.totalTokens;
assert.ok(total <= (index === 0 ? 16000 : 90000));
const unknown = result.usage.unknownRequests ?? 0;
const estimatedUnknown = result.usage.estimatedUnknownTokens ?? 0;
assert.ok(total + estimatedUnknown <= (index === 0 ? 16000 : 90000));
requests += count; reportedTokens += total; unknownRequests += unknown;
estimatedUnknownTokens += estimatedUnknown;
assert.equal(result.modelWeightsAuthenticated, false);
const folder = index === 0 ? 'preflight-resumed' : index === 1 ? 'node' : 'node-after-fix';
const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, count);
let actualTotal = 0, actualUnknown = 0;
for (const attempt of attempts) {
assert.equal(attempt.connectionId, '93d0cf6d-4f32-4918-9da8-6f125aa8a6e5');
assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions');
assert.equal(attempt.requestUrl, 'https://api.apikey.fan/v1/chat/completions');
const request = result.captures.find(c => c.attemptId === attempt.id && c.kind === 'request');
assert.ok(request); const requestBytes = fs.readFileSync(file('run/' + request.path));
assert.equal(sha(requestBytes), request.sha256); assert.equal(request.sha256, attempt.requestSha256);
assert.equal(JSON.parse(requestBytes).model, attempt.model);
const response = result.captures.find(c => c.attemptId === attempt.id && c.kind === 'response');
assert.ok(response); const responseBytes = fs.readFileSync(file('run/' + response.path));
assert.equal(sha(responseBytes), response.sha256); assert.equal(response.sha256, attempt.responseSha256);
if (attempt.status !== 'completed') {
assert.equal(result.success, false); assert.equal(index, 2); assert.equal(attempt.status, 'failed');
assert.equal(attempt.httpStatus, 502); assert.equal(attempt.usage, null); assert.equal(attempt.usageStatus, 'unknown'); actualUnknown++;
assert.equal(attempt.modelIdentity.status, 'incomplete'); assert.equal(attempt.modelIdentity.complete, false);
assert.equal(attempt.diagnostic.origin, 'upstream'); assert.equal(attempt.diagnostic.retryAfterMs, 60000);
assert.equal(attempt.diagnostic.upstream.headers['content-type'], 'text/html; charset=UTF-8');
assert.match(responseBytes.toString('utf8'), /502: Bad gateway/);
assert.equal(attempt.diagnostic.upstream.artifactSha256, response.sha256);
assert.equal(attempt.diagnostic.upstream.capture.complete, true);
continue;
}
const events = responseBytes.toString('utf8').split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trim()).filter(line => line && line !== '[DONE]').map(line => JSON.parse(line));
const models = [...new Set(events.map(event => event.model).filter(Boolean))]; assert.deepEqual(models, ['gpt-6.1-sol']);
const usage = events.findLast(event => event.usage)?.usage; assert.ok(usage);
assert.equal(usage.prompt_tokens, attempt.usage.prompt_tokens); assert.equal(usage.completion_tokens, attempt.usage.completion_tokens);
actualTotal += usage.total_tokens ?? usage.prompt_tokens + usage.completion_tokens;
assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true);
}
assert.equal(actualTotal, total);
assert.equal(actualUnknown, unknown);
if (index > 0) {
const restoration = owned('run/' + folder + '/restoration.json'); assert.equal(restoration.passed, true);
assert.equal(restoration.originalCredentialRevision, 1); assert.equal(restoration.finalCredentialRevision, 1);
if (index === 2) {
assert.equal(restoration.originalPublicProjectsExact, true);
assert.equal(restoration.originalAttemptsExact, owned('run/node-after-fix/original-attempts.json').length);
assert.equal(restoration.rawConfigExceptRevisionExact, true); assert.equal(restoration.registryExceptDeclaredRevisionTimestampOrderExact, true);
assert.equal(restoration.otherMetadataRowsExact, true);
}
}
}
assert.equal(real[0].passed, true); assert.equal(real[0].actualTools, 0);
assert.equal(real[1].success, false); assert.equal(real[1].stopReason, 'context_limit');
assert.equal(real[1].usage.requests, 2); assert.equal(real[1].usage.totalTokens, 11269);
assert.equal(real[2].success, false); assert.equal(real[2].state, 'failed'); assert.equal(real[2].usage.requests, 2);
assert.equal(real[2].usage.totalTokens, 4838); assert.equal(real[2].usage.unknownRequests, 1); assert.equal(real[2].usage.estimatedUnknownTokens, 11095);
assert.deepEqual(real[2].approvals, []); assert.deepEqual(real[2].applications, []); assert.equal(real[2].modelImplementationSuppliedBySupervisor, false);
for (const name of ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']) {
assert.deepEqual(fs.readFileSync(file('run/node/source-before/' + name)), fs.readFileSync(file('run/node/source-after/' + name)));
assert.deepEqual(fs.readFileSync(file('run/node-after-fix/source-before/' + name)), fs.readFileSync(file('run/node-after-fix/source-after/' + name)));
}
const formal = owned(manifest.references.formal), freeze = owned(manifest.references.freeze), release = owned(manifest.references.release);
assert.equal(formal.success, true); assert.equal(formal.pid, 36552); assert.equal(formal.hasKey, true);
assert.equal(formal.version, '1.24.5'); assert.equal(formal.sourceCommit, manifest.sourceCommit); assert.equal(freeze.sourceCommit, manifest.sourceCommit);
assert.equal(release.documentation.sourceCommit, manifest.sourceCommit);
assert.equal(formal.retention.originalRows, 'exact'); assert.equal(formal.retention.originalExports, 'exact');
assert.equal(formal.retention.originalArtifactBytes, 'exact'); assert.equal(formal.retention.originalModelCenter, 'exact');
const browserReview = owned('run/release/browser-review-result.json'), nativeReview = owned('run/release/native-review-result.json');
assert.equal(browserReview.success, true); assert.equal(browserReview.views.length, 2); assert.equal(browserReview.actualImagesReviewed, true);
assert.deepEqual(browserReview.errors, []); assert.deepEqual(browserReview.rejectedWrites, []);
assert.equal(nativeReview.success, true); assert.equal(nativeReview.version, manifest.version); assert.equal(nativeReview.checked.length, 3);
assert.equal(nativeReview.noTaskStart, true); assert.equal(nativeReview.realModelRequests, 0);
const archives = owned('release/archives-validation.json'), portable = owned('release/release-validation.json');
assert.equal(archives.success, true); assert.equal(archives.version, manifest.version); assert.equal(archives.archives.length, 3);
for (const archive of archives.archives) { assert.equal(archive.crcPassed, true); assert.equal(archive.manifestAndDirectoryMatch, true); }
assert.equal(portable.success, true); assert.equal(portable.appRoot, release.portableRoot);
const rebuild = owned('release/independent-source-rebuild/result.json');
assert.equal(rebuild.success, true); assert.equal(rebuild.sourceCommit, manifest.sourceCommit); assert.equal(rebuild.inputCount, 552);
assert.deepEqual(rebuild.mismatches, []);
for (const log of rebuild.logs) assert.equal(sha(fs.readFileSync(file('release/independent-source-rebuild/' + log.name + '.log'))), log.sha256);
const installation = owned('release/isolated-install/result.json');
assert.equal(installation.success, true); assert.equal(installation.version, manifest.version); assert.equal(installation.results.length, 12);
for (const { proof, script } of installation.installedUI.proofs) {
assert.equal(proof.success, true); assert.equal(proof.version, manifest.version); assert.equal(proof.realModelRequests, 0);
assert.equal(proof.checks.length, script === 'verify-file-results-ui.mjs' ? 16 : 9); assert.deepEqual(proof.errors, []);
owned('ui/installed/' + path.basename(proof.directory ?? proof.root) + '/result.json');
}
let sourceUiGroups = 0;
for (const [folder, count] of [['model-selection-k-3bd2d5b6-b0c2-4592-89a7-5221f614cc91', 7],
['model-identity-ui-71450f91-9e8f-48d7-b7b9-ea6c69bb1b28', 3], ['task-plan-ui-5b6fee37-1b8d-46e4-984f-f4202e6a66e3', 8],
['messages-ui-m-c3f7030c-d9c0-4eba-a608-b5c4c9429180', 7]]) {
const proof = owned('ui/source/' + folder + '/result.json');
assert.equal(proof.success, true); assert.equal(proof.version ?? proof.testedVersion, manifest.version);
assert.equal((proof.checks ?? proof.passed).length, count); sourceUiGroups += count;
}
for (const label of ['source', 'portable', 'installed']) {
const result = owned('run/compiled-context-' + label + '/result.json');
assert.equal(result.success, true); assert.equal(result.requests, 3); assert.equal(result.externalModelRequests, 0);
assert.equal(result.contextLimit, 16000); assert.equal(result.outputReservation, 2048); assert.equal(result.safetyTokens, 1600);
assert.equal(result.rawToolEvidenceVerified, true); assert.equal(result.stopReason, 'verification_incomplete');
}
if (real[2].success) {
assert.ok(real[2].commit); assert.equal(real[2].tests.total, 9); assert.equal(real[2].tests.passed, 9); assert.equal(real[2].tests.failed, 0);
assert.equal(real[2].identitiesValid, true);
}
console.log(JSON.stringify({ success: true, manifestSha256: sha(fs.readFileSync(file('manifest.json'))), files: manifest.files.length, bytes,
requests, reportedTokens, unknownRequests, estimatedUnknownTokens, originalNodeDelivered: false, fixedNodeDelivered: real[2].success,
sourceUiGroups, isolatedInstallerCases: installation.results.length, installedFileUiCases: 25,
originalPurgeBenefitTested: false, sourceCommit: manifest.sourceCommit, candidate: manifest.candidate,
scope: 'Offline original-byte, outgoing/requested/reported identifiers, usage, bounded samples and declared restoration checks. No model requests, credential access, task replay or weight authentication.' }));