test: seal AB delivery failure and preserve validation boundaries
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
// Analyze observed wire shape, public tools and source coverage. Do not inspect
|
||||
// or evaluate private reasoning semantics; only count its returned characters.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { root, save, sha, dataRoot } from './common.mjs';
|
||||
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
||||
const state = read('node/session.json'), outcome = read('node/model-outcome.json'), attempts = read('node/attempts.json');
|
||||
const readCapture = (attempt, kind) => fs.readFileSync(path.join(root, outcome.captures.find(c => c.attemptId === attempt.id && c.kind === kind).path));
|
||||
const requests = attempts.map((attempt, index) => {
|
||||
const request = JSON.parse(readCapture(attempt, 'request')), calls = new Map(), finishes = [];
|
||||
let terminalSeen = false, textCharacters = 0, reasoningCharacters = 0;
|
||||
for (const line of readCapture(attempt, 'response').toString('utf8').split(/\r?\n/)) {
|
||||
if (!line.startsWith('data: ')) continue;
|
||||
const data = line.slice(6); if (data === '[DONE]') { terminalSeen = true; continue; }
|
||||
const event = JSON.parse(data);
|
||||
for (const choice of event.choices ?? []) {
|
||||
if (choice.finish_reason) finishes.push(choice.finish_reason);
|
||||
textCharacters += (choice.delta?.content ?? '').length; reasoningCharacters += (choice.delta?.reasoning_content ?? '').length;
|
||||
for (const fragment of choice.delta?.tool_calls ?? []) {
|
||||
const call = calls.get(fragment.index) ?? { name: '', arguments: '' };
|
||||
call.name += fragment.function?.name ?? ''; call.arguments += fragment.function?.arguments ?? ''; calls.set(fragment.index, call);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.deepEqual(request.thinking, { type: 'enabled' }); assert.equal(request.reasoning_effort, 'low'); assert.equal(request.max_tokens, 2048);
|
||||
return { ordinal: index + 1, attemptId: attempt.id, status: attempt.status, identity: attempt.modelIdentity, usage: attempt.usage,
|
||||
parameters: { thinking: request.thinking, reasoning_effort: request.reasoning_effort, max_tokens: request.max_tokens },
|
||||
finishes, terminalSeen, textCharacters, reasoningCharacters,
|
||||
calls: [...calls.values()].map(call => { let input, argumentsComplete = false; try { input = JSON.parse(call.arguments); argumentsComplete = true; } catch {}
|
||||
return { name: call.name, argumentCharacters: call.arguments.length, argumentsComplete, ...(call.name === 'read_file' ? { input } : {}) }; }) };
|
||||
});
|
||||
const fileReads = state.events.filter(e => e.kind === 'tool' && e.tool === 'read_file').map(event => {
|
||||
const result = JSON.parse(event.output), source = fs.readFileSync(path.join(root, 'node/source-before', result.path), 'utf8');
|
||||
assert.equal(sha(source), result.sha256); assert.equal(source.slice(result.offset, result.next_offset), result.content);
|
||||
return { input: event.input, artifactId: event.artifactId, path: result.path, sha256: result.sha256, range: [result.offset, result.next_offset],
|
||||
characters: result.returned_characters, total: result.total_characters, truncated: result.truncated };
|
||||
});
|
||||
const duplicateRanges = fileReads.filter((row, index) => fileReads.slice(0, index).some(old => old.path === row.path && old.sha256 === row.sha256 && JSON.stringify(old.range) === JSON.stringify(row.range)));
|
||||
const lastRequest = JSON.parse(readCapture(attempts.at(-1), 'request'));
|
||||
const workingMessage = lastRequest.messages.find(message => typeof message.content === 'string' && message.content.includes('"kind":"task-working-set"'));
|
||||
assert.ok(workingMessage); const working = JSON.parse(workingMessage.content.slice(workingMessage.content.indexOf('{')));
|
||||
const retainedFiles = working.data.entries.filter(entry => entry.observation?.kind === 'file').map(entry => {
|
||||
const source = fs.readFileSync(path.join(root, 'node/source-before', entry.observation.path), 'utf8');
|
||||
assert.equal(sha(entry.content), entry.content_sha256); assert.equal(entry.content, source.slice(...entry.sent_range));
|
||||
return { path: entry.observation.path, sentRange: entry.sent_range, totalSourceCharacters: source.length,
|
||||
retainedCharacters: entry.content.length, sourceSha256: entry.observation.sha256, rangeComplete: entry.file_read?.range_complete ?? !entry.truncated };
|
||||
});
|
||||
const before = read('node/before.json'), after = read('node/after.json'), oldArtifacts = new Set(before.persistent.artifacts.map(a => a.path));
|
||||
const added = after.persistent.artifacts.filter(a => !oldArtifacts.has(a.path));
|
||||
for (const row of added) {
|
||||
const bytes = fs.readFileSync(path.join(dataRoot(), 'artifacts', row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
|
||||
save('node/artifacts/' + row.path, bytes);
|
||||
}
|
||||
save('node/owned-artifacts.json', { files: added, originalPhysicalFiles: before.persistent.artifacts.length, allNewPhysicalFilesCaptured: true });
|
||||
const checks = state.execution.checks.map(job => ({ id: job.id, status: job.status, results: job.results.map(result => ({ name: result.name, status: result.status, tests: result.tests, durationMs: result.durationMs, logArtifactId: result.logArtifactId })) }));
|
||||
const analysis = { passed: true, purpose: 'Preservation and failure analysis; engineering did not pass', requests, fileReads,
|
||||
exactDuplicateReadRanges: duplicateRanges.length, retainedFilesInLastActualRequest: retainedFiles,
|
||||
currentCheckJobPresentInLastRequest: checks.every(job => JSON.stringify(lastRequest).includes(job.id)),
|
||||
sourceFilesChanged: state.changes.length, checks, actualToolCalls: state.events.filter(e => e.kind === 'tool').length,
|
||||
supervision: { checkApprovals: outcome.approvals.length, applications: outcome.applications.length, reviewContinuations: Math.max(0, outcome.continuations.length - 1), budgetAllocations: outcome.continuations.length },
|
||||
elapsedSeconds: outcome.elapsedSeconds, reportedCacheHitTokens: attempts.reduce((sum, a) => sum + (a.usage?.prompt_cache_hit_tokens ?? 0), 0),
|
||||
finalReservation: state.execution.contextPlans.at(-1)?.tokenReservation, ownedPhysicalArtifacts: added.length,
|
||||
privateReasoningSemanticsReviewed: false, modelRequestsInAnalysis: 0, automaticReplay: false,
|
||||
conclusions: { stoppedBy: 'upstream finish_reason=length at the sealed 2048 output ceiling; DONE was seen',
|
||||
modelStartedImplementation: true, acceptedProposals: 0, lastResponseToolCalls: 'one complete propose_file followed by a second incomplete propose_file; the response was rejected as a whole',
|
||||
all2048TokensWereReasoning: 'not supported: the last usage lacks a reasoning-token split and the stream contains tool arguments',
|
||||
repeatReadingResolvedGenerally: false, reactPythonExecuted: false, originalPurgeBenefitTested: false } };
|
||||
assert.equal(analysis.currentCheckJobPresentInLastRequest, true); assert.equal(analysis.sourceFilesChanged, 0);
|
||||
save('node/analysis.json', analysis);
|
||||
console.log(JSON.stringify({ passed: true, requests: requests.length, exactDuplicateReadRanges: duplicateRanges.length, retainedFiles, checks,
|
||||
lastResponse: requests.at(-1), ownedPhysicalArtifacts: added.length, reportedCacheHitTokens: analysis.reportedCacheHitTokens, realModelRequests: 0 }));
|
||||
@@ -0,0 +1,79 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { batch, client, save, readRoot, dataRoot, meta, batchUsage } from './common.mjs';
|
||||
import { publicSnapshot, digest } from './state.mjs';
|
||||
import { inspectorClosed } from './inspector.mjs';
|
||||
const read = name => JSON.parse(fs.readFileSync(path.join(batch, name)));
|
||||
const current = readRoot(), before = current.baseline, owned = [read('preflight/owned.json'), read('node/owned.json')];
|
||||
const { api, boot, close } = await client();
|
||||
try {
|
||||
const after = fs.existsSync(path.join(batch, 'final-state.json')) ? read('final-state.json') : await publicSnapshot('http://127.0.0.1:4318', dataRoot());
|
||||
if (!fs.existsSync(path.join(batch, 'final-state.json'))) save('final-state.json', after);
|
||||
for (const field of ['pid', 'version', 'runtimeRoot', 'hasKey']) assert.equal(after[field], before[field]);
|
||||
assert.deepEqual(after.active, []); assert.equal(after.projects, before.projects + 2); assert.equal(after.exports.length, before.exports.length + 2);
|
||||
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item);
|
||||
for (const [table, previous] of Object.entries(before.database.tables)) {
|
||||
assert.equal(after.database.tables[table].schemaSha256, previous.schemaSha256);
|
||||
if (['projects', 'metadata', 'sqlite_sequence'].includes(table)) continue;
|
||||
const key = row => table === 'records' ? row.kind + '/' + row.id : row.id;
|
||||
const rows = new Map(after.database.tables[table].rows.map(row => [key(row), row]));
|
||||
for (const row of previous.rows) assert.deepEqual(rows.get(key(row)), row, table + '/' + key(row));
|
||||
}
|
||||
const physical = new Map(after.persistent.artifacts.map(row => [row.path, row]));
|
||||
for (const row of before.persistent.artifacts) assert.deepEqual(physical.get(row.path), row);
|
||||
const db = new DatabaseSync(path.join(dataRoot(), 'workbench.sqlite'), { readOnly: true });
|
||||
const originalProjects = new Map(before.database.tables.projects.rows.map(row => [row.id, row]));
|
||||
const ownedProjects = new Set(owned.map(row => row.project.id)), ownedSessions = new Set(owned.map(row => row.session.id));
|
||||
let sequence, attribution = {};
|
||||
try {
|
||||
const projects = db.prepare('SELECT * FROM projects ORDER BY rowid').all();
|
||||
assert.deepEqual(projects.filter(row => !originalProjects.has(row.id)).map(row => row.id).sort(), [...ownedProjects].sort());
|
||||
for (const row of projects.filter(row => originalProjects.has(row.id))) assert.equal(digest({ ...row, position: row.position - 2 }), originalProjects.get(row.id).sha256);
|
||||
for (const table of ['sessions', 'records', 'events']) {
|
||||
const key = row => table === 'records' ? row.kind + '/' + row.id : row.id;
|
||||
const previousIds = new Set(before.database.tables[table].rows.map(key));
|
||||
const rows = db.prepare('SELECT * FROM ' + table + ' ORDER BY rowid').all().filter(row => !previousIds.has(key(row)));
|
||||
for (const row of rows) assert.ok(ownedSessions.has(table === 'sessions' ? row.id : row.session_id), 'Every new row belongs to the two AB sessions.');
|
||||
attribution[table] = rows.length;
|
||||
}
|
||||
sequence = db.prepare('SELECT * FROM sqlite_sequence').all();
|
||||
assert.equal(sequence.length, 1); assert.equal(sequence[0].name, 'events');
|
||||
assert.equal(sequence[0].seq, Math.max(...after.database.tables.outbox.rows.map(row => row.id)));
|
||||
} finally { db.close(); }
|
||||
const restoration = read('node/restoration.json'); assert.equal(restoration.passed, true);
|
||||
assert.equal(read('preflight/verification.json').passed, true); assert.equal(read('node/analysis.json').passed, true);
|
||||
const metadata = new Map(after.database.tables.metadata.rows.map(row => [row.id, row]));
|
||||
for (const row of before.database.tables.metadata.rows.filter(row => !['config', 'modelRegistry', 'modelAttempts', 'modelNetworkDiagnostics'].includes(row.id))) assert.deepEqual(metadata.get(row.id), row);
|
||||
const originalRaw = read('node/original-public-metadata.json'), finalRaw = read('node/final-public-metadata.json');
|
||||
assert.deepEqual(meta('config'), finalRaw.config); assert.deepEqual(meta('modelRegistry'), finalRaw.registry);
|
||||
assert.deepEqual({ ...boot.config, configRevision: current.config.configRevision }, current.config);
|
||||
assert.deepEqual(after.modelCenter.selected, current.center.selected); assert.deepEqual(after.modelCenter.pools, current.center.pools);
|
||||
const attempts = meta('modelAttempts') ?? [], originalAttempts = read('original-attempts.json');
|
||||
for (const row of originalAttempts) assert.deepEqual(attempts.find(value => value.id === row.id), row);
|
||||
const expectedNew = [...read('preflight/attempts.json'), ...read('node/attempts.json')];
|
||||
assert.deepEqual(attempts.filter(row => !originalAttempts.some(old => old.id === row.id)), expectedNew);
|
||||
const diagnostics = meta('modelNetworkDiagnostics') ?? [], originalDiagnostics = read('original-diagnostics.json');
|
||||
for (const row of originalDiagnostics) assert.deepEqual(diagnostics.find(value => value.id === row.id), row);
|
||||
const { stale, currentConnectionRevision, ...networkRaw } = read('network.json');
|
||||
assert.equal(stale, false); assert.equal(currentConnectionRevision, current.connectionRevision);
|
||||
assert.deepEqual(diagnostics.filter(row => !originalDiagnostics.some(old => old.id === row.id)), [networkRaw]);
|
||||
const host = await api('/host/status'); assert.equal(host.identity.pid, current.host.identity.pid); assert.equal(host.identity.instanceId, current.host.identity.instanceId);
|
||||
assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0);
|
||||
const summary = { passed: true, purpose: 'AB evidence/restoration audit; autonomous engineering failed',
|
||||
node: { success: false, requests: 7, reportedTokens: 65440, stop: 'protocol / upstream finish_reason=length', modelBeganProposeFile: true, acceptedProposals: 0, delivery: false },
|
||||
preflight: read('preflight/result.json'), combinedAaAbUsage: batchUsage(),
|
||||
originalExportsExact: before.exports.length, originalArtifactFilesExact: before.persistent.artifacts.length, originalAttemptsExact: originalAttempts.length,
|
||||
originalBusinessRowsExact: true, originalProjectPayloadsExact: true, projectPositionNormalization: { ownedPrepends: 2 },
|
||||
eventSequenceFrom: before.database.tables.sqlite_sequence.rows[0].id, eventSequenceTo: sequence[0].seq, newRowsAttribution: attribution,
|
||||
originalDiagnosticsExact: true, originalConfigExceptMonotonicRevisionExact: true, registryRestoration: restoration.identityDefaultNormalization,
|
||||
originalConfigRevision: originalRaw.config.configRevision, finalConfigRevision: finalRaw.config.configRevision,
|
||||
originalCredentialRevision: restoration.originalCredentialRevision, finalCredentialRevision: restoration.finalCredentialRevision,
|
||||
finalCounts: Object.fromEntries(Object.entries(after.database.tables).map(([name, table]) => [name, table.count])),
|
||||
reactPython: { executed: false, reason: 'Node did not close the original tests/build/review/Git delivery loop.' },
|
||||
purge: { executed: false, benefit: 'unknown', reason: 'Node/React/Python prerequisites not met; no AB/BA instances dispatched.' },
|
||||
pid: host.identity.pid, instanceId: host.identity.instanceId, version: boot.version, windows: 0, inspectorClosed: await inspectorClosed(), noCredentialReadOrResave: true,
|
||||
replayedFailedSamples: false, pushed: false };
|
||||
save('final-summary.json', summary); console.log(JSON.stringify({ ...summary, preflight: { passed: summary.preflight.passed, requests: summary.preflight.requests, tokens: summary.preflight.usage.total_tokens } }));
|
||||
} finally { await close(); }
|
||||
@@ -0,0 +1,33 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { batch, sha } from './common.mjs';
|
||||
const destination = path.resolve('docs/live-delivery-ab-evidence'); assert.equal(fs.existsSync(destination), false);
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(batch, 'final-summary.json'))).passed, true);
|
||||
const sourceRoot = path.join(batch, 'node/expense-project'), bundleFile = path.join(batch, 'node/baseline.bundle');
|
||||
assert.equal(fs.existsSync(bundleFile), false);
|
||||
execFileSync('git', ['-C', sourceRoot, 'bundle', 'create', bundleFile, '--all'], { windowsHide: true });
|
||||
const verifiedBundle = execFileSync('git', ['-C', sourceRoot, 'bundle', 'verify', bundleFile], { windowsHide: true, encoding: 'utf8' });
|
||||
fs.writeFileSync(path.join(batch, 'node/bundle-verification.txt'), verifiedBundle, { flag: 'wx' });
|
||||
fs.cpSync(batch, destination, { recursive: true, errorOnExist: true, filter: file => file !== sourceRoot && !file.startsWith(sourceRoot + path.sep) });
|
||||
fs.cpSync('scripts/live-ab', path.join(destination, 'drivers'), { recursive: true, errorOnExist: true });
|
||||
for (const name of ['payload.mjs', 'inspector.mjs', 'state.mjs']) {
|
||||
const file = path.join(destination, 'shared-aa-drivers', name); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.copyFileSync('scripts/live-aa/' + name, file, fs.constants.COPYFILE_EXCL);
|
||||
}
|
||||
const files = [];
|
||||
function walk(relative) {
|
||||
for (const entry of fs.readdirSync(path.join(destination, relative), { withFileTypes: true })) {
|
||||
const name = path.posix.join(relative, entry.name), file = path.join(destination, name), stat = fs.lstatSync(file);
|
||||
assert.equal(stat.isSymbolicLink(), false); assert.equal(stat.nlink, 1);
|
||||
if (entry.isDirectory()) { walk(name); continue; }
|
||||
assert.equal(entry.isFile(), true); const bytes = fs.readFileSync(file); files.push({ path: name, bytes: bytes.length, sha256: sha(bytes) });
|
||||
}
|
||||
}
|
||||
walk(''); files.sort((a, b) => a.path.localeCompare(b.path));
|
||||
const manifest = { schemaVersion: 1, sample: 'AB-1.25.3-DeepSeek-low-engineering', files, filesCount: files.length,
|
||||
totalBytes: files.reduce((sum, row) => sum + row.bytes, 0),
|
||||
excluded: ['node/expense-project live Git checkout and generated dist: original files, exact baseline commit bundle, original index/working-copy states, after-source and checks are archived separately.'],
|
||||
note: 'Preserves original length failure and unexecuted React/Python/Purge. Inventory completeness is not engineering success.' };
|
||||
fs.writeFileSync(path.join(destination, 'MANIFEST.json'), JSON.stringify(manifest, null, 2) + '\n', { flag: 'wx' });
|
||||
console.log(JSON.stringify({ success: true, directory: destination, files: files.length, bytes: manifest.totalBytes, manifestSha256: sha(fs.readFileSync(path.join(destination, 'MANIFEST.json'))) }));
|
||||
@@ -0,0 +1,24 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
const directory = path.resolve('docs/live-delivery-ab-evidence');
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'MANIFEST.json')));
|
||||
const session = JSON.parse(fs.readFileSync(path.join(directory, 'node/session.json'))), observations = [];
|
||||
for (const row of manifest.files.filter(row => row.path.startsWith('node/artifacts/') && row.path.endsWith('.blob'))) {
|
||||
const bytes = fs.readFileSync(path.join(directory, row.path)); assert.equal(createHash('sha256').update(bytes).digest('hex'), row.sha256);
|
||||
let value; try { value = JSON.parse(bytes); } catch { continue; }
|
||||
if (!value.classification || value.mode !== 'shadow') continue;
|
||||
assert.ok(value.stepId || value.phase === 'baseline' && value.stepId === null && value.stepIndex === null);
|
||||
assert.equal(value.taskId, session.execution.task.id); assert.equal(value.runId, session.execution.run.id);
|
||||
assert.equal(value.delta.changed_files, 0); assert.equal(value.delta.repeated_reads, 0);
|
||||
observations.push({ path: row.path, sha256: row.sha256, stepIndex: value.stepIndex, phase: value.phase, classification: value.classification, delta: value.delta, reasons: value.reasons, waiting: value.waiting });
|
||||
}
|
||||
assert.equal(observations.length, 10);
|
||||
const classifications = {};
|
||||
for (const row of observations) classifications[row.classification] = (classifications[row.classification] ?? 0) + 1;
|
||||
assert.equal(classifications.evidence_growth, 6); assert.equal(classifications.waiting, 2); assert.equal(classifications.new_failure_evidence, 1); assert.equal(classifications.baseline, 1);
|
||||
const result = { passed: true, sample: 'AB', observations: observations.sort((a, b) => a.stepIndex - b.stepIndex || a.phase.localeCompare(b.phase)), classifications,
|
||||
scope: 'Ten preserved public-fact shadow records only; no prompt, stop, permission or budget intervention and no engineering-success claim.', realModelRequests: 0 };
|
||||
fs.writeFileSync('docs/live-delivery-ab-shadow.json', JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
|
||||
console.log(JSON.stringify({ passed: true, observations: observations.length, classifications, realModelRequests: 0 }));
|
||||
Reference in New Issue
Block a user