Files
Agent-Workbench/scripts/live-ae/seal.mjs
T
2026-11-06 13:08:37 +08:00

92 lines
5.9 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { engineeringEvidence, root, read, save, sha } from '../live-ad/common.mjs';
engineeringEvidence();
const auditPath = 'audits/react-python-3f300ee4-539a-46aa-94d9-e00b5aa8aef6/result.json';
const audit = read(auditPath);
assert.equal(audit.success, true);
assert.deepEqual(audit.completed, ['react', 'python']);
const samples = [], proofs = [];
for (const kind of ['react', 'python']) {
const result = read(kind + '/result.json'), state = read(kind + '/session.json'), attempts = read(kind + '/attempts.json');
assert.equal(result.success, true);
assert.equal(result.ownerBytesAndIndexPreserved, true);
assert.equal(result.requests, attempts.length);
assert.equal(result.usage.unknownRequests, 0);
for (const attempt of attempts) {
assert.equal(attempt.status, 'completed');
assert.equal(attempt.modelIdentity.complete, true);
assert.equal(attempt.modelIdentity.status, 'matched');
for (const type of ['request', 'response']) {
const capture = result.captures.filter(row => row.attemptId === attempt.id && row.kind === type);
assert.equal(capture.length, 1);
const bytes = fs.readFileSync(path.join(root, capture[0].path));
assert.equal(bytes.length, capture[0].bytes);
assert.equal(sha(bytes), attempt[type + 'Sha256']);
assert.equal(sha(bytes), capture[0].sha256);
if (type === 'request') {
const body = JSON.parse(bytes);
assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
const plan = state.execution.contextPlans.find(row => row.id === attempt.requestId);
const variant = plan.prefixAudit.variants.find(row => row.connectionId === attempt.connectionId);
assert.equal(plan.prefixAudit.schemaVersion, 2);
assert.equal(sha(JSON.stringify(body.tools)), variant.toolsSha256);
assert.equal(Buffer.byteLength(JSON.stringify(body.tools)), variant.toolBytes);
proofs.push({ kind, attemptId: attempt.id, planId: plan.id, toolCount: body.tools.length,
toolSha256: variant.toolsSha256, toolBytes: variant.toolBytes,
commonPrefixBytes: variant.commonPrefixBytes, firstChangedPath: variant.firstChange?.path ?? null });
}
}
}
const reports = state.events.filter(row => row.tool === 'delivery_report' && row.phase === 'completed')
.map(row => JSON.parse(row.output));
const reads = state.events.filter(row => row.tool === 'read_file' && row.phase === 'completed').map(row => row.input);
samples.push({ kind, success: result.success, commit: result.commit, committedPaths: result.committedPaths,
checks: result.final, browser: result.browser, requests: result.requests, usage: result.usage,
reportedCachedInputTokens: attempts.reduce((sum, row) => sum + (row.usage?.prompt_tokens_details?.cached_tokens ?? 0), 0),
elapsedMs: Date.parse(state.execution.run.finishedAt) - Date.parse(state.execution.run.startedAt),
reads, repeatedReadInputCount: reads.length - new Set(reads.map(row => JSON.stringify(row))).size,
internalChecks: state.execution.checks.map(row => ({ id: row.id, status: row.status,
results: row.results.map(check => ({ name: check.name, status: check.status, exitCode: check.exitCode,
executable: check.python?.executable ?? null, logExcerpt: check.status === 'failed' ? check.logExcerpt : null })) })),
internalDeliveryJobs: state.execution.deliveries.length, reports, terminalGit: true,
ownerBytesAndIndexPreserved: true, supervisorApprovals: 0, supervisorContinuations: 0 });
}
const summary = { schema: 1, driverCommit: read('current.json').driverCommit, version: '1.25.8',
realModelRequests: samples.reduce((sum, row) => sum + row.requests, 0),
reportedTokens: samples.reduce((sum, row) => sum + row.usage.totalTokens, 0), unknownRequests: 0,
samples, prefixProofs: proofs, originalStateAudit: auditPath, originalStateAuditResult: audit,
purgeExecuted: false, cacheCausalBenefit: 'unknown', historicalSamplesReplayed: false, keysReadOrResaved: false,
limitations: ['Terminal Git is independently verified; no internal delivery_job was registered.',
'Python built-in check selected a different interpreter without pytest; its original failure remains.',
'Reported cached tokens and repeated reads describe these samples only.'] };
save('summary.json', summary);
const files = [], excluded = [];
function walk(directory) {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const file = path.join(directory, entry.name), relative = path.relative(root, file).replaceAll('\\', '/');
if (entry.isDirectory() && entry.name === 'node_modules') { excluded.push(relative); continue; }
if (entry.isDirectory()) walk(file);
else {
assert.ok(entry.isFile(), 'Only regular evidence entries are accepted.');
const bytes = fs.readFileSync(file);
files.push({ path: relative, bytes: bytes.length, sha256: sha(bytes) });
}
}
}
walk(root);
files.sort((a, b) => a.path.localeCompare(b.path));
const bytes = files.reduce((sum, row) => sum + row.bytes, 0);
save('FINAL-MANIFEST.json', { schema: 1, files, bytes, excluded,
scope: 'All AE requests/responses, source/Git, checks, screenshots and audits, including original failures. Reproducible node_modules excluded; previous evidence is not repackaged.' });
for (const row of files) {
const value = fs.readFileSync(path.join(root, row.path));
assert.equal(value.length, row.bytes); assert.equal(sha(value), row.sha256);
}
fs.writeFileSync('docs/live-engineering-ae-evidence.json', JSON.stringify({ summary,
evidence: { root, files: files.length, bytes, excluded, manifestSha256: sha(fs.readFileSync(path.join(root, 'FINAL-MANIFEST.json'))) } }, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify({ success: true, requests: summary.realModelRequests, reportedTokens: summary.reportedTokens,
unknown: 0, files: files.length, bytes, excluded }));