97 lines
6.5 KiB
JavaScript
97 lines
6.5 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { Store } from '../src/server/store.mjs';
|
|
import { RequestPlanner, historyGroups, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
|
|
|
if (process.argv.includes('--help')) {
|
|
console.log('node scripts/verify-live-context-retention.mjs OUTPUT [retained|original] — offline replay of sealed E source/context, no model calls. OUTPUT must be a new directory.');
|
|
process.exit(0);
|
|
}
|
|
assert.ok(process.argv[2], 'Choose a new output directory.');
|
|
const destination = path.resolve(process.argv[2]);
|
|
assert.ok(!fs.existsSync(destination), 'Keep prior evidence; choose a new directory.');
|
|
fs.mkdirSync(destination, { recursive: true });
|
|
const root = path.resolve('docs/live-original-purge-e-evidence');
|
|
const sha = value => createHash('sha256').update(value).digest('hex');
|
|
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
|
const manifest = read('manifest.json');
|
|
for (const entry of manifest.entries) {
|
|
const raw = fs.readFileSync(path.join(root, entry.path));
|
|
assert.equal(raw.length, entry.bytes); assert.equal(sha(raw), entry.sha256);
|
|
}
|
|
const sample = manifest.samples.find(sample => sample.name === 'react');
|
|
const exported = read('owned-exports/' + sample.sessionId + '.json');
|
|
const original = exported.session;
|
|
const rows = read('receipts/owned-records.json');
|
|
const originalCore = {};
|
|
for (const row of rows) (originalCore[row.kind] ??= new Map()).set(row.id, JSON.parse(row.payload));
|
|
const artifacts = new Map(manifest.artifacts.map(artifact => [artifact.id, artifact]));
|
|
function rawArtifact(id) { return fs.readFileSync(path.join(root, artifacts.get(id).file), 'utf8'); }
|
|
const plans = [...originalCore.context_plan.values()].filter(plan => plan.sessionId === sample.sessionId).sort((a, b) => a.requestIndex - b.requestIndex);
|
|
const reports = [];
|
|
for (const prior of plans.filter(plan => plan.requestIndex >= 7)) {
|
|
const store = new Store(path.join(destination, 'request-' + prior.requestIndex));
|
|
try {
|
|
for (const [kind, map] of Object.entries(originalCore)) store.core[kind] = structuredClone(map);
|
|
const readNew = store.artifactStore.read.bind(store.artifactStore);
|
|
store.artifactStore.read = artifact => artifacts.has(artifact.id) ? rawArtifact(artifact.id) : readNew(artifact);
|
|
const session = structuredClone(original);
|
|
store.sessions.set(session.id, session);
|
|
store.save = () => {}; // Scratch planner records only; never persist original sessions.
|
|
const project = store.project.bind(store);
|
|
store.project = id => id === exported.project.id ? exported.project : project(id);
|
|
session.providerMessages = session.providerMessages.slice(0, prior.originalMessages);
|
|
session.changes = session.changes.filter(change => change.createdAt <= prior.createdAt);
|
|
const run = store.core.run.get(prior.runId), task = store.core.task.get(prior.taskId);
|
|
// Restore this request's exact remaining credits, not the final run's usage.
|
|
for (const [record, remaining] of [[run, prior.budget.runRemainingTokens], [task, prior.budget.taskRemainingTokens]]) {
|
|
record.usage = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, estimatedUnknownTokens: 0 };
|
|
record.budget.maxTokens = remaining ?? 0;
|
|
}
|
|
run.consumption = []; // Context selection only; no fabricated upstream usage.
|
|
const request = JSON.parse(rawArtifact(prior.requestArtifactId));
|
|
const wire = request.variants?.[0].body ?? request;
|
|
const config = { ...run.model, modelRouting: run.model.routing, streaming: wire.stream, nativeParameters: run.model.nativeParameters ?? {} };
|
|
const planner = new RequestPlanner(store);
|
|
const prepared = planner.prepare(session, run, config, { system: request.messages[0].content, tools: wire.tools });
|
|
assert.equal(prepared.record.budget.inputBudget, prior.budget.inputBudget);
|
|
assert.ok(prepared.record.estimatedInputTokens <= prepared.record.budget.inputBudget);
|
|
verifyPlanEvidence(store, session, prepared.record);
|
|
const sources = [];
|
|
for (const group of historyGroups(session.providerMessages)) for (const message of group.messages.filter(message => message.role === 'tool')) {
|
|
const call = group.messages[0].tool_calls.find(call => call.id === message.tool_call_id);
|
|
const value = JSON.parse(message.content);
|
|
if (call.function.name === 'read_file' && !value.truncated && value.start_line === 1 && value.end_line === value.total_lines) {
|
|
sources.push({ path: value.path, content: value.content });
|
|
}
|
|
}
|
|
function strings(value, out = []) {
|
|
if (typeof value === 'string') {
|
|
out.push(value);
|
|
for (const line of [value, ...value.split('\n').filter(line => /^[\[{]/.test(line))]) { try { strings(JSON.parse(line), out); } catch {} }
|
|
} else if (value && typeof value === 'object') for (const item of Object.values(value)) strings(item, out);
|
|
return out;
|
|
}
|
|
const sent = strings(prepared.messages);
|
|
const report = { request: prior.requestIndex, originalInputTokens: prior.estimatedInputTokens,
|
|
inputTokens: prepared.record.estimatedInputTokens, inputBudget: prior.budget.inputBudget,
|
|
softTarget: prior.budget.targetInputTokens,
|
|
sources: [...new Map(sources.map(source => [source.path, source])).values()].map(source => ({ path: source.path,
|
|
characters: source.content.length, retained: sent.some(text => text.includes(source.content)) })) };
|
|
reports.push(report);
|
|
fs.writeFileSync(path.join(destination, 'request-' + prior.requestIndex + '.json'), JSON.stringify({ report, plan: prepared.record, body: prepared.body }, null, 2), { flag: 'wx' });
|
|
} finally { store.close(); }
|
|
}
|
|
const result = { schemaVersion: 1, manifestSha256: sha(fs.readFileSync(path.join(root, 'manifest.json'))),
|
|
scope: 'Offline context-selection counterfactual using original E system/tools/history/source bytes and remaining credits. No model requests or task execution. Historical outcomes unchanged.',
|
|
realModelRequests: 0, reports };
|
|
if (process.argv[3] === 'retained') assert.ok(reports.every(report => report.sources.length === 6 && report.sources.every(source => source.retained)));
|
|
if (process.argv[3] === 'original') {
|
|
assert.ok(reports.every(report => report.inputTokens === report.originalInputTokens));
|
|
assert.deepEqual(reports.map(report => report.sources.filter(source => source.retained).length), [6, 5, 5, 4, 2, 2]);
|
|
}
|
|
fs.writeFileSync(path.join(destination, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
|
|
console.log(JSON.stringify(result));
|