Files
Agent-Workbench/scripts/verify-live-context-retention.mjs
2026-11-03 07:58:13 +08:00

97 lines
6.5 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { Store } from '../src/server/store.mjs';
import { RequestPlanner, historyGroups, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
if (process.argv.includes('--help')) {
console.log('node scripts/verify-live-context-retention.mjs OUTPUT [retained|original] — offline replay of sealed E source/context, no model calls. OUTPUT must be a new directory.');
process.exit(0);
}
assert.ok(process.argv[2], 'Choose a new output directory.');
const destination = path.resolve(process.argv[2]);
assert.ok(!fs.existsSync(destination), 'Keep prior evidence; choose a new directory.');
fs.mkdirSync(destination, { recursive: true });
const root = path.resolve('docs/live-original-purge-e-evidence');
const sha = value => createHash('sha256').update(value).digest('hex');
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
const manifest = read('manifest.json');
for (const entry of manifest.entries) {
const raw = fs.readFileSync(path.join(root, entry.path));
assert.equal(raw.length, entry.bytes); assert.equal(sha(raw), entry.sha256);
}
const sample = manifest.samples.find(sample => sample.name === 'react');
const exported = read('owned-exports/' + sample.sessionId + '.json');
const original = exported.session;
const rows = read('receipts/owned-records.json');
const originalCore = {};
for (const row of rows) (originalCore[row.kind] ??= new Map()).set(row.id, JSON.parse(row.payload));
const artifacts = new Map(manifest.artifacts.map(artifact => [artifact.id, artifact]));
function rawArtifact(id) { return fs.readFileSync(path.join(root, artifacts.get(id).file), 'utf8'); }
const plans = [...originalCore.context_plan.values()].filter(plan => plan.sessionId === sample.sessionId).sort((a, b) => a.requestIndex - b.requestIndex);
const reports = [];
for (const prior of plans.filter(plan => plan.requestIndex >= 7)) {
const store = new Store(path.join(destination, 'request-' + prior.requestIndex));
try {
for (const [kind, map] of Object.entries(originalCore)) store.core[kind] = structuredClone(map);
const readNew = store.artifactStore.read.bind(store.artifactStore);
store.artifactStore.read = artifact => artifacts.has(artifact.id) ? rawArtifact(artifact.id) : readNew(artifact);
const session = structuredClone(original);
store.sessions.set(session.id, session);
store.save = () => {}; // Scratch planner records only; never persist original sessions.
const project = store.project.bind(store);
store.project = id => id === exported.project.id ? exported.project : project(id);
session.providerMessages = session.providerMessages.slice(0, prior.originalMessages);
session.changes = session.changes.filter(change => change.createdAt <= prior.createdAt);
const run = store.core.run.get(prior.runId), task = store.core.task.get(prior.taskId);
// Restore this request's exact remaining credits, not the final run's usage.
for (const [record, remaining] of [[run, prior.budget.runRemainingTokens], [task, prior.budget.taskRemainingTokens]]) {
record.usage = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, estimatedUnknownTokens: 0 };
record.budget.maxTokens = remaining ?? 0;
}
run.consumption = []; // Context selection only; no fabricated upstream usage.
const request = JSON.parse(rawArtifact(prior.requestArtifactId));
const wire = request.variants?.[0].body ?? request;
const config = { ...run.model, modelRouting: run.model.routing, streaming: wire.stream, nativeParameters: run.model.nativeParameters ?? {} };
const planner = new RequestPlanner(store);
const prepared = planner.prepare(session, run, config, { system: request.messages[0].content, tools: wire.tools });
assert.equal(prepared.record.budget.inputBudget, prior.budget.inputBudget);
assert.ok(prepared.record.estimatedInputTokens <= prepared.record.budget.inputBudget);
verifyPlanEvidence(store, session, prepared.record);
const sources = [];
for (const group of historyGroups(session.providerMessages)) for (const message of group.messages.filter(message => message.role === 'tool')) {
const call = group.messages[0].tool_calls.find(call => call.id === message.tool_call_id);
const value = JSON.parse(message.content);
if (call.function.name === 'read_file' && !value.truncated && value.start_line === 1 && value.end_line === value.total_lines) {
sources.push({ path: value.path, content: value.content });
}
}
function strings(value, out = []) {
if (typeof value === 'string') {
out.push(value);
for (const line of [value, ...value.split('\n').filter(line => /^[\[{]/.test(line))]) { try { strings(JSON.parse(line), out); } catch {} }
} else if (value && typeof value === 'object') for (const item of Object.values(value)) strings(item, out);
return out;
}
const sent = strings(prepared.messages);
const report = { request: prior.requestIndex, originalInputTokens: prior.estimatedInputTokens,
inputTokens: prepared.record.estimatedInputTokens, inputBudget: prior.budget.inputBudget,
softTarget: prior.budget.targetInputTokens,
sources: [...new Map(sources.map(source => [source.path, source])).values()].map(source => ({ path: source.path,
characters: source.content.length, retained: sent.some(text => text.includes(source.content)) })) };
reports.push(report);
fs.writeFileSync(path.join(destination, 'request-' + prior.requestIndex + '.json'), JSON.stringify({ report, plan: prepared.record, body: prepared.body }, null, 2), { flag: 'wx' });
} finally { store.close(); }
}
const result = { schemaVersion: 1, manifestSha256: sha(fs.readFileSync(path.join(root, 'manifest.json'))),
scope: 'Offline context-selection counterfactual using original E system/tools/history/source bytes and remaining credits. No model requests or task execution. Historical outcomes unchanged.',
realModelRequests: 0, reports };
if (process.argv[3] === 'retained') assert.ok(reports.every(report => report.sources.length === 6 && report.sources.every(source => source.retained)));
if (process.argv[3] === 'original') {
assert.ok(reports.every(report => report.inputTokens === report.originalInputTokens));
assert.deepEqual(reports.map(report => report.sources.filter(source => source.retained).length), [6, 5, 5, 4, 2, 2]);
}
fs.writeFileSync(path.join(destination, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify(result));