Files
Agent-Workbench/scripts/verify-live-delivery-p.mjs
2026-11-03 22:45:26 +08:00

133 lines
12 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
const root = path.resolve('docs/live-delivery-p-evidence');
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
const file = name => {
assert.equal(typeof name, 'string'); assert.ok(!/[\\\x00\r\n]/.test(name) && !name.split('/').includes('..'));
const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target;
};
const read = name => JSON.parse(fs.readFileSync(file(name), 'utf8'));
const manifest = read('manifest.json'); assert.equal(manifest.kind, 'owned-live-delivery-p-evidence');
const names = new Set(); let bytes = 0;
for (const row of manifest.files) {
assert.ok(!names.has(row.path)); names.add(row.path);
const content = fs.readFileSync(file(row.path)); assert.equal(content.length, row.bytes); assert.equal(sha(content), row.sha256, row.path); bytes += content.length;
}
const list = (folder = root) => fs.readdirSync(folder, { withFileTypes: true }).flatMap(entry => {
assert.equal(entry.isSymbolicLink(), false); const target = path.join(folder, entry.name);
return entry.isDirectory() ? list(target) : [path.relative(root, target).replaceAll('\\', '/')];
});
assert.deepEqual(list().sort(), ['manifest.json', ...names].sort());
const owned = name => { assert.ok(names.has(name)); return read(name); };
const initial = owned(manifest.references.initial), preflight = owned(manifest.references.preflight), node = owned(manifest.references.node);
const final = owned(manifest.references.final), network = owned(manifest.references.network);
const connectionId = '5b4daed9-b2f7-4de0-b949-573eba70c7b3';
for (const [folder, outcome] of [['preflight', preflight], ['node', node]]) {
const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, 1);
const attempt = attempts[0]; assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions'); assert.equal(attempt.streaming, false);
assert.equal(attempt.requestUrl, 'https://jojocode.com/v1/chat/completions'); assert.equal(attempt.credentialRevision, 1);
assert.equal(attempt.modelIdentity.policy, 'require-match');
const request = outcome.captures.find(value => value.kind === 'request'); assert.ok(request);
assert.equal(request.attemptId, attempt.id); assert.ok(names.has('run/' + request.path));
const requestBytes = fs.readFileSync(file('run/' + request.path));
assert.equal(sha(requestBytes), request.sha256); assert.equal(request.sha256, attempt.requestSha256); assert.equal(requestBytes.length, attempt.requestBytes);
const body = JSON.parse(requestBytes); assert.equal(body.model, 'gpt-6.1-sol'); assert.notEqual(body.stream, true);
const restoration = owned('run/' + folder + '/restoration.json'); assert.equal(restoration.passed, true);
if (folder === 'preflight') {
assert.equal(preflight.passed, true); assert.equal(body.tools?.length ?? 0, 0); assert.equal(body.max_tokens, 256);
assert.equal(preflight.actualTools, 0); assert.equal(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported');
assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true);
const response = preflight.captures.find(value => value.kind === 'response'); assert.ok(response);
const responseBytes = fs.readFileSync(file('run/' + response.path)); assert.equal(sha(responseBytes), response.sha256);
assert.equal(response.sha256, attempt.responseSha256); assert.equal(response.sha256, preflight.identity.responseSource.sha256);
const original = JSON.parse(responseBytes); assert.equal(original.model, 'gpt-6.1-sol'); assert.deepEqual(original.usage, attempt.usage);
assert.equal(original.usage.total_tokens, 112); assert.deepEqual(preflight.usage, original.usage);
assert.equal(original.choices[0].message.tool_calls?.length ?? 0, 0);
assert.ok(original.choices[0].message.content.includes(owned('run/preflight/contract.json').nonce));
assert.equal(restoration.originalConfigRegistryAndPoolExact, true); assert.equal(restoration.inspectorClosed, true);
} else {
assert.equal(body.max_tokens, 2048); assert.ok(body.tools.length > 0); assert.equal(node.success, false);
assert.equal(attempt.status, 'failed'); assert.equal(attempt.httpStatus, null); assert.equal(attempt.usage, null); assert.equal(attempt.usageStatus, 'unknown');
assert.equal(attempt.responseArtifactId, undefined); assert.equal(node.captures.length, 1);
assert.equal(attempt.modelIdentity.complete, false); assert.equal(attempt.diagnostic.code, 'UND_ERR_CONNECT_TIMEOUT');
assert.equal(attempt.diagnostic.network.stage, 'connect'); assert.equal(attempt.diagnostic.network.replaySafety, 'before_model_send');
assert.equal(attempt.diagnostic.network.responseReceived, false); assert.equal(attempt.diagnostic.retryDecision, 'attempt_limit');
assert.equal(attempt.diagnostic.network.transport.connectTimeoutMs, 10000);
assert.equal(node.usage.requests, 1); assert.equal(node.usage.totalTokens, 0); assert.equal(node.usage.unknownRequests, 1); assert.equal(node.usage.estimatedUnknownTokens, 10141);
const session = owned('run/node/session.json'), plan = session.execution.contextPlans[0];
assert.equal(plan.tokenReservation.reservedTokens, node.usage.estimatedUnknownTokens);
assert.equal(plan.budget.contextLimit, 16000); assert.equal(plan.budget.outputReservation, 2048); assert.equal(plan.budget.safetyTokens, 1600);
assert.equal(session.execution.run.budget.maxSteps, 8); assert.equal(session.execution.run.budget.maxTokens, 90000);
assert.equal(session.execution.run.toolCalls, 0); assert.deepEqual(session.execution.checks, []); assert.deepEqual(session.changes, []);
assert.deepEqual(node.approvals, []); assert.deepEqual(node.applications, []); assert.equal(node.modelImplementationSuppliedBySupervisor, false);
assert.equal(restoration.budgetsAndSelectionRestored, true); assert.equal(restoration.poolsUnchanged, true);
assert.equal(restoration.originalCredentialRevision, 1); assert.equal(restoration.finalCredentialRevision, 1);
assert.equal(restoration.registryExceptDeclaredRevisionTimestampOrderAndIdentityDefaultsExact, true);
assert.deepEqual(restoration.identityDefaultNormalization, { modelIdentityPolicy: 'record', modelIdentityAliases: [], modelIdentityAliasSource: '' });
}
}
assert.equal(network.realModelRequests, 0); assert.equal(network.httpRequests, 0); assert.equal(network.keyValuesReadOrSent, false);
assert.equal(network.tlsCertificateValidationEnabled, true); assert.equal(network.productConfigurationChanged, false);
assert.equal(network.results.length, 4);
const trials = network.results.flatMap(value => value.trials); assert.equal(trials.length, 12); assert.equal(trials.filter(value => value.success).length, 11);
const failure = trials.find(value => !value.success); assert.equal(failure.mode, 'ipv6-only-diagnostic'); assert.equal(failure.code, 'QA_TLS_TIMEOUT');
assert.ok(trials.filter(value => value.success).every(value => value.certificateAuthorized === true));
assert.equal(final.passed, true); assert.equal(final.pid, 36552); assert.equal(final.version, '1.24.5');
assert.equal(final.originalProjects, 59); assert.equal(final.originalSessions, 75); assert.equal(final.originalArtifacts, 3605); assert.equal(final.originalAttempts, 207);
assert.equal(final.finalProjects, 61); assert.equal(final.finalSessions, 77); assert.equal(final.addedAttempts, 2);
assert.equal(final.originalBudgetsSelectionAndPoolRestored, true); assert.equal(final.originalSessionExportsRowsAndArtifactsExact, true);
assert.equal(final.exactSinceNodeRestoration, true); assert.equal(final.inspectorClosed, true); assert.equal(final.keyReadOrResaved, false);
const snapshot = owned('run/final/snapshot.json');
for (const row of initial.baseline.exports) assert.deepEqual(snapshot.exports.find(value => value.id === row.id), row);
for (const table of ['sessions', 'records', 'events', 'outbox']) {
const rows = new Map(snapshot.database.tables[table].rows.map(value => [value.id, value]));
for (const row of initial.baseline.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row);
}
const artifacts = new Map(snapshot.persistent.artifacts.map(value => [value.path, value]));
for (const row of initial.baseline.persistent.artifacts) assert.deepEqual(artifacts.get(row.path), row);
assert.deepEqual(snapshot.modelCenter.selected, initial.center.selected); assert.deepEqual(snapshot.modelCenter.pools, initial.center.pools);
assert.deepEqual(final.connectionRevision, [3, 5]); assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision, [61, 65]);
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
const beforeMeta = owned('run/node/original-public-metadata.json'), afterMeta = owned('run/node/final-public-metadata.json');
assert.deepEqual({ ...afterMeta.config, configRevision: beforeMeta.config.configRevision }, beforeMeta.config);
const registry = structuredClone(afterMeta.registry); registry.revision = beforeMeta.registry.revision;
const currentConnection = registry.connections.find(value => value.id === connectionId), oldConnection = beforeMeta.registry.connections.find(value => value.id === connectionId);
assert.equal(currentConnection.revision, oldConnection.revision + 2);
currentConnection.revision = oldConnection.revision; currentConnection.updatedAt = oldConnection.updatedAt;
const expected = structuredClone(beforeMeta.registry); Object.assign(expected.connections.find(value => value.id === connectionId), {
modelIdentityPolicy: 'record', modelIdentityAliases: [], modelIdentityAliasSource: '' });
for (const value of [expected, registry]) value.connections.sort((a, b) => a.id.localeCompare(b.id));
assert.deepEqual(canonical(registry), canonical(expected));
for (const name of ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']) assert.deepEqual(fs.readFileSync(file('run/node/source-before/' + name)), fs.readFileSync(file('run/node/source-after/' + name)));
const git = owned('run/node/git-final.json'), baseline = owned('run/node/source-identity.json');
assert.equal(git.head, baseline.head); assert.equal(git.stagedAfter, baseline.stagedBefore); assert.equal(git.status, baseline.status); assert.equal(git.diff, '');
assert.equal(manifest.releaseSourceCommit, owned('provenance/release-1.24.5.json').sourceCommit);
assert.equal(final.productSourceChanged, false); assert.equal(final.nodeDelivered, false); assert.equal(final.purgeBenefitTested, false);
let gitIndex;
if (process.argv.includes('--index')) {
const evidenceRoot = path.relative(process.cwd(), root).replaceAll('\\', '/');
const manifestBytes = fs.readFileSync(file('manifest.json'));
const entries = [...manifest.files, { path: 'manifest.json', bytes: manifestBytes.length, sha256: sha(manifestBytes) }];
const output = execFileSync('git', ['cat-file', '--batch'], { input: entries.map(row => ':' + evidenceRoot + '/' + row.path).join('\n') + '\n',
maxBuffer: 100 * 1024 * 1024, windowsHide: true });
let offset = 0, indexBytes = 0;
for (const entry of entries) {
const end = output.indexOf(10, offset), header = output.subarray(offset, end).toString('utf8');
const match = header.match(/^[a-f0-9]{40} blob (\d+)$/); assert.ok(match, header);
const size = Number(match[1]); offset = end + 1;
const content = output.subarray(offset, offset + size); offset += size;
assert.equal(output[offset++], 10); assert.equal(size, entry.bytes, entry.path); assert.equal(sha(content), entry.sha256, entry.path); indexBytes += size;
}
assert.equal(offset, output.length); gitIndex = { passed: true, files: entries.length, bytes: indexBytes, exactOriginalBytes: true };
}
console.log(JSON.stringify({ success: true, files: manifest.files.length, bytes, manifestSha256: sha(fs.readFileSync(file('manifest.json'))),
requests: 2, reportedTokens: 112, unknownRequests: 1, estimatedUnknownTokens: 10141, nodeDelivered: false,
tlsHandshakes: trials.length, tlsPassed: 11, tlsTimedOut: 1, originalDataAndDefaultsVerified: true, productVersion: '1.24.5',
originalPurgeBenefitTested: false, ...(gitIndex ? { gitIndex } : {}),
scope: 'Offline original bytes, frozen configuration, accounting and declared original-state restoration; no network, model or credential access.' }));