133 lines
12 KiB
JavaScript
133 lines
12 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
const root = path.resolve('docs/live-delivery-p-evidence');
|
|
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
const file = name => {
|
|
assert.equal(typeof name, 'string'); assert.ok(!/[\\\x00\r\n]/.test(name) && !name.split('/').includes('..'));
|
|
const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target;
|
|
};
|
|
const read = name => JSON.parse(fs.readFileSync(file(name), 'utf8'));
|
|
const manifest = read('manifest.json'); assert.equal(manifest.kind, 'owned-live-delivery-p-evidence');
|
|
const names = new Set(); let bytes = 0;
|
|
for (const row of manifest.files) {
|
|
assert.ok(!names.has(row.path)); names.add(row.path);
|
|
const content = fs.readFileSync(file(row.path)); assert.equal(content.length, row.bytes); assert.equal(sha(content), row.sha256, row.path); bytes += content.length;
|
|
}
|
|
const list = (folder = root) => fs.readdirSync(folder, { withFileTypes: true }).flatMap(entry => {
|
|
assert.equal(entry.isSymbolicLink(), false); const target = path.join(folder, entry.name);
|
|
return entry.isDirectory() ? list(target) : [path.relative(root, target).replaceAll('\\', '/')];
|
|
});
|
|
assert.deepEqual(list().sort(), ['manifest.json', ...names].sort());
|
|
const owned = name => { assert.ok(names.has(name)); return read(name); };
|
|
const initial = owned(manifest.references.initial), preflight = owned(manifest.references.preflight), node = owned(manifest.references.node);
|
|
const final = owned(manifest.references.final), network = owned(manifest.references.network);
|
|
const connectionId = '5b4daed9-b2f7-4de0-b949-573eba70c7b3';
|
|
for (const [folder, outcome] of [['preflight', preflight], ['node', node]]) {
|
|
const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, 1);
|
|
const attempt = attempts[0]; assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
|
|
assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions'); assert.equal(attempt.streaming, false);
|
|
assert.equal(attempt.requestUrl, 'https://jojocode.com/v1/chat/completions'); assert.equal(attempt.credentialRevision, 1);
|
|
assert.equal(attempt.modelIdentity.policy, 'require-match');
|
|
const request = outcome.captures.find(value => value.kind === 'request'); assert.ok(request);
|
|
assert.equal(request.attemptId, attempt.id); assert.ok(names.has('run/' + request.path));
|
|
const requestBytes = fs.readFileSync(file('run/' + request.path));
|
|
assert.equal(sha(requestBytes), request.sha256); assert.equal(request.sha256, attempt.requestSha256); assert.equal(requestBytes.length, attempt.requestBytes);
|
|
const body = JSON.parse(requestBytes); assert.equal(body.model, 'gpt-6.1-sol'); assert.notEqual(body.stream, true);
|
|
const restoration = owned('run/' + folder + '/restoration.json'); assert.equal(restoration.passed, true);
|
|
if (folder === 'preflight') {
|
|
assert.equal(preflight.passed, true); assert.equal(body.tools?.length ?? 0, 0); assert.equal(body.max_tokens, 256);
|
|
assert.equal(preflight.actualTools, 0); assert.equal(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported');
|
|
assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true);
|
|
const response = preflight.captures.find(value => value.kind === 'response'); assert.ok(response);
|
|
const responseBytes = fs.readFileSync(file('run/' + response.path)); assert.equal(sha(responseBytes), response.sha256);
|
|
assert.equal(response.sha256, attempt.responseSha256); assert.equal(response.sha256, preflight.identity.responseSource.sha256);
|
|
const original = JSON.parse(responseBytes); assert.equal(original.model, 'gpt-6.1-sol'); assert.deepEqual(original.usage, attempt.usage);
|
|
assert.equal(original.usage.total_tokens, 112); assert.deepEqual(preflight.usage, original.usage);
|
|
assert.equal(original.choices[0].message.tool_calls?.length ?? 0, 0);
|
|
assert.ok(original.choices[0].message.content.includes(owned('run/preflight/contract.json').nonce));
|
|
assert.equal(restoration.originalConfigRegistryAndPoolExact, true); assert.equal(restoration.inspectorClosed, true);
|
|
} else {
|
|
assert.equal(body.max_tokens, 2048); assert.ok(body.tools.length > 0); assert.equal(node.success, false);
|
|
assert.equal(attempt.status, 'failed'); assert.equal(attempt.httpStatus, null); assert.equal(attempt.usage, null); assert.equal(attempt.usageStatus, 'unknown');
|
|
assert.equal(attempt.responseArtifactId, undefined); assert.equal(node.captures.length, 1);
|
|
assert.equal(attempt.modelIdentity.complete, false); assert.equal(attempt.diagnostic.code, 'UND_ERR_CONNECT_TIMEOUT');
|
|
assert.equal(attempt.diagnostic.network.stage, 'connect'); assert.equal(attempt.diagnostic.network.replaySafety, 'before_model_send');
|
|
assert.equal(attempt.diagnostic.network.responseReceived, false); assert.equal(attempt.diagnostic.retryDecision, 'attempt_limit');
|
|
assert.equal(attempt.diagnostic.network.transport.connectTimeoutMs, 10000);
|
|
assert.equal(node.usage.requests, 1); assert.equal(node.usage.totalTokens, 0); assert.equal(node.usage.unknownRequests, 1); assert.equal(node.usage.estimatedUnknownTokens, 10141);
|
|
const session = owned('run/node/session.json'), plan = session.execution.contextPlans[0];
|
|
assert.equal(plan.tokenReservation.reservedTokens, node.usage.estimatedUnknownTokens);
|
|
assert.equal(plan.budget.contextLimit, 16000); assert.equal(plan.budget.outputReservation, 2048); assert.equal(plan.budget.safetyTokens, 1600);
|
|
assert.equal(session.execution.run.budget.maxSteps, 8); assert.equal(session.execution.run.budget.maxTokens, 90000);
|
|
assert.equal(session.execution.run.toolCalls, 0); assert.deepEqual(session.execution.checks, []); assert.deepEqual(session.changes, []);
|
|
assert.deepEqual(node.approvals, []); assert.deepEqual(node.applications, []); assert.equal(node.modelImplementationSuppliedBySupervisor, false);
|
|
assert.equal(restoration.budgetsAndSelectionRestored, true); assert.equal(restoration.poolsUnchanged, true);
|
|
assert.equal(restoration.originalCredentialRevision, 1); assert.equal(restoration.finalCredentialRevision, 1);
|
|
assert.equal(restoration.registryExceptDeclaredRevisionTimestampOrderAndIdentityDefaultsExact, true);
|
|
assert.deepEqual(restoration.identityDefaultNormalization, { modelIdentityPolicy: 'record', modelIdentityAliases: [], modelIdentityAliasSource: '' });
|
|
}
|
|
}
|
|
assert.equal(network.realModelRequests, 0); assert.equal(network.httpRequests, 0); assert.equal(network.keyValuesReadOrSent, false);
|
|
assert.equal(network.tlsCertificateValidationEnabled, true); assert.equal(network.productConfigurationChanged, false);
|
|
assert.equal(network.results.length, 4);
|
|
const trials = network.results.flatMap(value => value.trials); assert.equal(trials.length, 12); assert.equal(trials.filter(value => value.success).length, 11);
|
|
const failure = trials.find(value => !value.success); assert.equal(failure.mode, 'ipv6-only-diagnostic'); assert.equal(failure.code, 'QA_TLS_TIMEOUT');
|
|
assert.ok(trials.filter(value => value.success).every(value => value.certificateAuthorized === true));
|
|
assert.equal(final.passed, true); assert.equal(final.pid, 36552); assert.equal(final.version, '1.24.5');
|
|
assert.equal(final.originalProjects, 59); assert.equal(final.originalSessions, 75); assert.equal(final.originalArtifacts, 3605); assert.equal(final.originalAttempts, 207);
|
|
assert.equal(final.finalProjects, 61); assert.equal(final.finalSessions, 77); assert.equal(final.addedAttempts, 2);
|
|
assert.equal(final.originalBudgetsSelectionAndPoolRestored, true); assert.equal(final.originalSessionExportsRowsAndArtifactsExact, true);
|
|
assert.equal(final.exactSinceNodeRestoration, true); assert.equal(final.inspectorClosed, true); assert.equal(final.keyReadOrResaved, false);
|
|
const snapshot = owned('run/final/snapshot.json');
|
|
for (const row of initial.baseline.exports) assert.deepEqual(snapshot.exports.find(value => value.id === row.id), row);
|
|
for (const table of ['sessions', 'records', 'events', 'outbox']) {
|
|
const rows = new Map(snapshot.database.tables[table].rows.map(value => [value.id, value]));
|
|
for (const row of initial.baseline.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row);
|
|
}
|
|
const artifacts = new Map(snapshot.persistent.artifacts.map(value => [value.path, value]));
|
|
for (const row of initial.baseline.persistent.artifacts) assert.deepEqual(artifacts.get(row.path), row);
|
|
assert.deepEqual(snapshot.modelCenter.selected, initial.center.selected); assert.deepEqual(snapshot.modelCenter.pools, initial.center.pools);
|
|
assert.deepEqual(final.connectionRevision, [3, 5]); assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision, [61, 65]);
|
|
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
|
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
|
const beforeMeta = owned('run/node/original-public-metadata.json'), afterMeta = owned('run/node/final-public-metadata.json');
|
|
assert.deepEqual({ ...afterMeta.config, configRevision: beforeMeta.config.configRevision }, beforeMeta.config);
|
|
const registry = structuredClone(afterMeta.registry); registry.revision = beforeMeta.registry.revision;
|
|
const currentConnection = registry.connections.find(value => value.id === connectionId), oldConnection = beforeMeta.registry.connections.find(value => value.id === connectionId);
|
|
assert.equal(currentConnection.revision, oldConnection.revision + 2);
|
|
currentConnection.revision = oldConnection.revision; currentConnection.updatedAt = oldConnection.updatedAt;
|
|
const expected = structuredClone(beforeMeta.registry); Object.assign(expected.connections.find(value => value.id === connectionId), {
|
|
modelIdentityPolicy: 'record', modelIdentityAliases: [], modelIdentityAliasSource: '' });
|
|
for (const value of [expected, registry]) value.connections.sort((a, b) => a.id.localeCompare(b.id));
|
|
assert.deepEqual(canonical(registry), canonical(expected));
|
|
for (const name of ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']) assert.deepEqual(fs.readFileSync(file('run/node/source-before/' + name)), fs.readFileSync(file('run/node/source-after/' + name)));
|
|
const git = owned('run/node/git-final.json'), baseline = owned('run/node/source-identity.json');
|
|
assert.equal(git.head, baseline.head); assert.equal(git.stagedAfter, baseline.stagedBefore); assert.equal(git.status, baseline.status); assert.equal(git.diff, '');
|
|
assert.equal(manifest.releaseSourceCommit, owned('provenance/release-1.24.5.json').sourceCommit);
|
|
assert.equal(final.productSourceChanged, false); assert.equal(final.nodeDelivered, false); assert.equal(final.purgeBenefitTested, false);
|
|
let gitIndex;
|
|
if (process.argv.includes('--index')) {
|
|
const evidenceRoot = path.relative(process.cwd(), root).replaceAll('\\', '/');
|
|
const manifestBytes = fs.readFileSync(file('manifest.json'));
|
|
const entries = [...manifest.files, { path: 'manifest.json', bytes: manifestBytes.length, sha256: sha(manifestBytes) }];
|
|
const output = execFileSync('git', ['cat-file', '--batch'], { input: entries.map(row => ':' + evidenceRoot + '/' + row.path).join('\n') + '\n',
|
|
maxBuffer: 100 * 1024 * 1024, windowsHide: true });
|
|
let offset = 0, indexBytes = 0;
|
|
for (const entry of entries) {
|
|
const end = output.indexOf(10, offset), header = output.subarray(offset, end).toString('utf8');
|
|
const match = header.match(/^[a-f0-9]{40} blob (\d+)$/); assert.ok(match, header);
|
|
const size = Number(match[1]); offset = end + 1;
|
|
const content = output.subarray(offset, offset + size); offset += size;
|
|
assert.equal(output[offset++], 10); assert.equal(size, entry.bytes, entry.path); assert.equal(sha(content), entry.sha256, entry.path); indexBytes += size;
|
|
}
|
|
assert.equal(offset, output.length); gitIndex = { passed: true, files: entries.length, bytes: indexBytes, exactOriginalBytes: true };
|
|
}
|
|
console.log(JSON.stringify({ success: true, files: manifest.files.length, bytes, manifestSha256: sha(fs.readFileSync(file('manifest.json'))),
|
|
requests: 2, reportedTokens: 112, unknownRequests: 1, estimatedUnknownTokens: 10141, nodeDelivered: false,
|
|
tlsHandshakes: trials.length, tlsPassed: 11, tlsTimedOut: 1, originalDataAndDefaultsVerified: true, productVersion: '1.24.5',
|
|
originalPurgeBenefitTested: false, ...(gitIndex ? { gitIndex } : {}),
|
|
scope: 'Offline original bytes, frozen configuration, accounting and declared original-state restoration; no network, model or credential access.' }));
|