test: seal AB delivery failure and preserve validation boundaries

This commit is contained in:
Agent Workbench Dev
2026-11-05 03:35:06 +08:00
parent 658a5c4e37
commit 7ab242dfec
173 changed files with 1401499 additions and 6 deletions
+72
View File
@@ -0,0 +1,72 @@
// Analyze observed wire shape, public tools and source coverage. Do not inspect
// or evaluate private reasoning semantics; only count its returned characters.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { root, save, sha, dataRoot } from './common.mjs';
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
const state = read('node/session.json'), outcome = read('node/model-outcome.json'), attempts = read('node/attempts.json');
const readCapture = (attempt, kind) => fs.readFileSync(path.join(root, outcome.captures.find(c => c.attemptId === attempt.id && c.kind === kind).path));
const requests = attempts.map((attempt, index) => {
const request = JSON.parse(readCapture(attempt, 'request')), calls = new Map(), finishes = [];
let terminalSeen = false, textCharacters = 0, reasoningCharacters = 0;
for (const line of readCapture(attempt, 'response').toString('utf8').split(/\r?\n/)) {
if (!line.startsWith('data: ')) continue;
const data = line.slice(6); if (data === '[DONE]') { terminalSeen = true; continue; }
const event = JSON.parse(data);
for (const choice of event.choices ?? []) {
if (choice.finish_reason) finishes.push(choice.finish_reason);
textCharacters += (choice.delta?.content ?? '').length; reasoningCharacters += (choice.delta?.reasoning_content ?? '').length;
for (const fragment of choice.delta?.tool_calls ?? []) {
const call = calls.get(fragment.index) ?? { name: '', arguments: '' };
call.name += fragment.function?.name ?? ''; call.arguments += fragment.function?.arguments ?? ''; calls.set(fragment.index, call);
}
}
}
assert.deepEqual(request.thinking, { type: 'enabled' }); assert.equal(request.reasoning_effort, 'low'); assert.equal(request.max_tokens, 2048);
return { ordinal: index + 1, attemptId: attempt.id, status: attempt.status, identity: attempt.modelIdentity, usage: attempt.usage,
parameters: { thinking: request.thinking, reasoning_effort: request.reasoning_effort, max_tokens: request.max_tokens },
finishes, terminalSeen, textCharacters, reasoningCharacters,
calls: [...calls.values()].map(call => { let input, argumentsComplete = false; try { input = JSON.parse(call.arguments); argumentsComplete = true; } catch {}
return { name: call.name, argumentCharacters: call.arguments.length, argumentsComplete, ...(call.name === 'read_file' ? { input } : {}) }; }) };
});
const fileReads = state.events.filter(e => e.kind === 'tool' && e.tool === 'read_file').map(event => {
const result = JSON.parse(event.output), source = fs.readFileSync(path.join(root, 'node/source-before', result.path), 'utf8');
assert.equal(sha(source), result.sha256); assert.equal(source.slice(result.offset, result.next_offset), result.content);
return { input: event.input, artifactId: event.artifactId, path: result.path, sha256: result.sha256, range: [result.offset, result.next_offset],
characters: result.returned_characters, total: result.total_characters, truncated: result.truncated };
});
const duplicateRanges = fileReads.filter((row, index) => fileReads.slice(0, index).some(old => old.path === row.path && old.sha256 === row.sha256 && JSON.stringify(old.range) === JSON.stringify(row.range)));
const lastRequest = JSON.parse(readCapture(attempts.at(-1), 'request'));
const workingMessage = lastRequest.messages.find(message => typeof message.content === 'string' && message.content.includes('"kind":"task-working-set"'));
assert.ok(workingMessage); const working = JSON.parse(workingMessage.content.slice(workingMessage.content.indexOf('{')));
const retainedFiles = working.data.entries.filter(entry => entry.observation?.kind === 'file').map(entry => {
const source = fs.readFileSync(path.join(root, 'node/source-before', entry.observation.path), 'utf8');
assert.equal(sha(entry.content), entry.content_sha256); assert.equal(entry.content, source.slice(...entry.sent_range));
return { path: entry.observation.path, sentRange: entry.sent_range, totalSourceCharacters: source.length,
retainedCharacters: entry.content.length, sourceSha256: entry.observation.sha256, rangeComplete: entry.file_read?.range_complete ?? !entry.truncated };
});
const before = read('node/before.json'), after = read('node/after.json'), oldArtifacts = new Set(before.persistent.artifacts.map(a => a.path));
const added = after.persistent.artifacts.filter(a => !oldArtifacts.has(a.path));
for (const row of added) {
const bytes = fs.readFileSync(path.join(dataRoot(), 'artifacts', row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
save('node/artifacts/' + row.path, bytes);
}
save('node/owned-artifacts.json', { files: added, originalPhysicalFiles: before.persistent.artifacts.length, allNewPhysicalFilesCaptured: true });
const checks = state.execution.checks.map(job => ({ id: job.id, status: job.status, results: job.results.map(result => ({ name: result.name, status: result.status, tests: result.tests, durationMs: result.durationMs, logArtifactId: result.logArtifactId })) }));
const analysis = { passed: true, purpose: 'Preservation and failure analysis; engineering did not pass', requests, fileReads,
exactDuplicateReadRanges: duplicateRanges.length, retainedFilesInLastActualRequest: retainedFiles,
currentCheckJobPresentInLastRequest: checks.every(job => JSON.stringify(lastRequest).includes(job.id)),
sourceFilesChanged: state.changes.length, checks, actualToolCalls: state.events.filter(e => e.kind === 'tool').length,
supervision: { checkApprovals: outcome.approvals.length, applications: outcome.applications.length, reviewContinuations: Math.max(0, outcome.continuations.length - 1), budgetAllocations: outcome.continuations.length },
elapsedSeconds: outcome.elapsedSeconds, reportedCacheHitTokens: attempts.reduce((sum, a) => sum + (a.usage?.prompt_cache_hit_tokens ?? 0), 0),
finalReservation: state.execution.contextPlans.at(-1)?.tokenReservation, ownedPhysicalArtifacts: added.length,
privateReasoningSemanticsReviewed: false, modelRequestsInAnalysis: 0, automaticReplay: false,
conclusions: { stoppedBy: 'upstream finish_reason=length at the sealed 2048 output ceiling; DONE was seen',
modelStartedImplementation: true, acceptedProposals: 0, lastResponseToolCalls: 'one complete propose_file followed by a second incomplete propose_file; the response was rejected as a whole',
all2048TokensWereReasoning: 'not supported: the last usage lacks a reasoning-token split and the stream contains tool arguments',
repeatReadingResolvedGenerally: false, reactPythonExecuted: false, originalPurgeBenefitTested: false } };
assert.equal(analysis.currentCheckJobPresentInLastRequest, true); assert.equal(analysis.sourceFilesChanged, 0);
save('node/analysis.json', analysis);
console.log(JSON.stringify({ passed: true, requests: requests.length, exactDuplicateReadRanges: duplicateRanges.length, retainedFiles, checks,
lastResponse: requests.at(-1), ownedPhysicalArtifacts: added.length, reportedCacheHitTokens: analysis.reportedCacheHitTokens, realModelRequests: 0 }));
+79
View File
@@ -0,0 +1,79 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { DatabaseSync } from 'node:sqlite';
import { batch, client, save, readRoot, dataRoot, meta, batchUsage } from './common.mjs';
import { publicSnapshot, digest } from './state.mjs';
import { inspectorClosed } from './inspector.mjs';
const read = name => JSON.parse(fs.readFileSync(path.join(batch, name)));
const current = readRoot(), before = current.baseline, owned = [read('preflight/owned.json'), read('node/owned.json')];
const { api, boot, close } = await client();
try {
const after = fs.existsSync(path.join(batch, 'final-state.json')) ? read('final-state.json') : await publicSnapshot('http://127.0.0.1:4318', dataRoot());
if (!fs.existsSync(path.join(batch, 'final-state.json'))) save('final-state.json', after);
for (const field of ['pid', 'version', 'runtimeRoot', 'hasKey']) assert.equal(after[field], before[field]);
assert.deepEqual(after.active, []); assert.equal(after.projects, before.projects + 2); assert.equal(after.exports.length, before.exports.length + 2);
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item);
for (const [table, previous] of Object.entries(before.database.tables)) {
assert.equal(after.database.tables[table].schemaSha256, previous.schemaSha256);
if (['projects', 'metadata', 'sqlite_sequence'].includes(table)) continue;
const key = row => table === 'records' ? row.kind + '/' + row.id : row.id;
const rows = new Map(after.database.tables[table].rows.map(row => [key(row), row]));
for (const row of previous.rows) assert.deepEqual(rows.get(key(row)), row, table + '/' + key(row));
}
const physical = new Map(after.persistent.artifacts.map(row => [row.path, row]));
for (const row of before.persistent.artifacts) assert.deepEqual(physical.get(row.path), row);
const db = new DatabaseSync(path.join(dataRoot(), 'workbench.sqlite'), { readOnly: true });
const originalProjects = new Map(before.database.tables.projects.rows.map(row => [row.id, row]));
const ownedProjects = new Set(owned.map(row => row.project.id)), ownedSessions = new Set(owned.map(row => row.session.id));
let sequence, attribution = {};
try {
const projects = db.prepare('SELECT * FROM projects ORDER BY rowid').all();
assert.deepEqual(projects.filter(row => !originalProjects.has(row.id)).map(row => row.id).sort(), [...ownedProjects].sort());
for (const row of projects.filter(row => originalProjects.has(row.id))) assert.equal(digest({ ...row, position: row.position - 2 }), originalProjects.get(row.id).sha256);
for (const table of ['sessions', 'records', 'events']) {
const key = row => table === 'records' ? row.kind + '/' + row.id : row.id;
const previousIds = new Set(before.database.tables[table].rows.map(key));
const rows = db.prepare('SELECT * FROM ' + table + ' ORDER BY rowid').all().filter(row => !previousIds.has(key(row)));
for (const row of rows) assert.ok(ownedSessions.has(table === 'sessions' ? row.id : row.session_id), 'Every new row belongs to the two AB sessions.');
attribution[table] = rows.length;
}
sequence = db.prepare('SELECT * FROM sqlite_sequence').all();
assert.equal(sequence.length, 1); assert.equal(sequence[0].name, 'events');
assert.equal(sequence[0].seq, Math.max(...after.database.tables.outbox.rows.map(row => row.id)));
} finally { db.close(); }
const restoration = read('node/restoration.json'); assert.equal(restoration.passed, true);
assert.equal(read('preflight/verification.json').passed, true); assert.equal(read('node/analysis.json').passed, true);
const metadata = new Map(after.database.tables.metadata.rows.map(row => [row.id, row]));
for (const row of before.database.tables.metadata.rows.filter(row => !['config', 'modelRegistry', 'modelAttempts', 'modelNetworkDiagnostics'].includes(row.id))) assert.deepEqual(metadata.get(row.id), row);
const originalRaw = read('node/original-public-metadata.json'), finalRaw = read('node/final-public-metadata.json');
assert.deepEqual(meta('config'), finalRaw.config); assert.deepEqual(meta('modelRegistry'), finalRaw.registry);
assert.deepEqual({ ...boot.config, configRevision: current.config.configRevision }, current.config);
assert.deepEqual(after.modelCenter.selected, current.center.selected); assert.deepEqual(after.modelCenter.pools, current.center.pools);
const attempts = meta('modelAttempts') ?? [], originalAttempts = read('original-attempts.json');
for (const row of originalAttempts) assert.deepEqual(attempts.find(value => value.id === row.id), row);
const expectedNew = [...read('preflight/attempts.json'), ...read('node/attempts.json')];
assert.deepEqual(attempts.filter(row => !originalAttempts.some(old => old.id === row.id)), expectedNew);
const diagnostics = meta('modelNetworkDiagnostics') ?? [], originalDiagnostics = read('original-diagnostics.json');
for (const row of originalDiagnostics) assert.deepEqual(diagnostics.find(value => value.id === row.id), row);
const { stale, currentConnectionRevision, ...networkRaw } = read('network.json');
assert.equal(stale, false); assert.equal(currentConnectionRevision, current.connectionRevision);
assert.deepEqual(diagnostics.filter(row => !originalDiagnostics.some(old => old.id === row.id)), [networkRaw]);
const host = await api('/host/status'); assert.equal(host.identity.pid, current.host.identity.pid); assert.equal(host.identity.instanceId, current.host.identity.instanceId);
assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0);
const summary = { passed: true, purpose: 'AB evidence/restoration audit; autonomous engineering failed',
node: { success: false, requests: 7, reportedTokens: 65440, stop: 'protocol / upstream finish_reason=length', modelBeganProposeFile: true, acceptedProposals: 0, delivery: false },
preflight: read('preflight/result.json'), combinedAaAbUsage: batchUsage(),
originalExportsExact: before.exports.length, originalArtifactFilesExact: before.persistent.artifacts.length, originalAttemptsExact: originalAttempts.length,
originalBusinessRowsExact: true, originalProjectPayloadsExact: true, projectPositionNormalization: { ownedPrepends: 2 },
eventSequenceFrom: before.database.tables.sqlite_sequence.rows[0].id, eventSequenceTo: sequence[0].seq, newRowsAttribution: attribution,
originalDiagnosticsExact: true, originalConfigExceptMonotonicRevisionExact: true, registryRestoration: restoration.identityDefaultNormalization,
originalConfigRevision: originalRaw.config.configRevision, finalConfigRevision: finalRaw.config.configRevision,
originalCredentialRevision: restoration.originalCredentialRevision, finalCredentialRevision: restoration.finalCredentialRevision,
finalCounts: Object.fromEntries(Object.entries(after.database.tables).map(([name, table]) => [name, table.count])),
reactPython: { executed: false, reason: 'Node did not close the original tests/build/review/Git delivery loop.' },
purge: { executed: false, benefit: 'unknown', reason: 'Node/React/Python prerequisites not met; no AB/BA instances dispatched.' },
pid: host.identity.pid, instanceId: host.identity.instanceId, version: boot.version, windows: 0, inspectorClosed: await inspectorClosed(), noCredentialReadOrResave: true,
replayedFailedSamples: false, pushed: false };
save('final-summary.json', summary); console.log(JSON.stringify({ ...summary, preflight: { passed: summary.preflight.passed, requests: summary.preflight.requests, tokens: summary.preflight.usage.total_tokens } }));
} finally { await close(); }
+33
View File
@@ -0,0 +1,33 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { batch, sha } from './common.mjs';
const destination = path.resolve('docs/live-delivery-ab-evidence'); assert.equal(fs.existsSync(destination), false);
assert.equal(JSON.parse(fs.readFileSync(path.join(batch, 'final-summary.json'))).passed, true);
const sourceRoot = path.join(batch, 'node/expense-project'), bundleFile = path.join(batch, 'node/baseline.bundle');
assert.equal(fs.existsSync(bundleFile), false);
execFileSync('git', ['-C', sourceRoot, 'bundle', 'create', bundleFile, '--all'], { windowsHide: true });
const verifiedBundle = execFileSync('git', ['-C', sourceRoot, 'bundle', 'verify', bundleFile], { windowsHide: true, encoding: 'utf8' });
fs.writeFileSync(path.join(batch, 'node/bundle-verification.txt'), verifiedBundle, { flag: 'wx' });
fs.cpSync(batch, destination, { recursive: true, errorOnExist: true, filter: file => file !== sourceRoot && !file.startsWith(sourceRoot + path.sep) });
fs.cpSync('scripts/live-ab', path.join(destination, 'drivers'), { recursive: true, errorOnExist: true });
for (const name of ['payload.mjs', 'inspector.mjs', 'state.mjs']) {
const file = path.join(destination, 'shared-aa-drivers', name); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.copyFileSync('scripts/live-aa/' + name, file, fs.constants.COPYFILE_EXCL);
}
const files = [];
function walk(relative) {
for (const entry of fs.readdirSync(path.join(destination, relative), { withFileTypes: true })) {
const name = path.posix.join(relative, entry.name), file = path.join(destination, name), stat = fs.lstatSync(file);
assert.equal(stat.isSymbolicLink(), false); assert.equal(stat.nlink, 1);
if (entry.isDirectory()) { walk(name); continue; }
assert.equal(entry.isFile(), true); const bytes = fs.readFileSync(file); files.push({ path: name, bytes: bytes.length, sha256: sha(bytes) });
}
}
walk(''); files.sort((a, b) => a.path.localeCompare(b.path));
const manifest = { schemaVersion: 1, sample: 'AB-1.25.3-DeepSeek-low-engineering', files, filesCount: files.length,
totalBytes: files.reduce((sum, row) => sum + row.bytes, 0),
excluded: ['node/expense-project live Git checkout and generated dist: original files, exact baseline commit bundle, original index/working-copy states, after-source and checks are archived separately.'],
note: 'Preserves original length failure and unexecuted React/Python/Purge. Inventory completeness is not engineering success.' };
fs.writeFileSync(path.join(destination, 'MANIFEST.json'), JSON.stringify(manifest, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify({ success: true, directory: destination, files: files.length, bytes: manifest.totalBytes, manifestSha256: sha(fs.readFileSync(path.join(destination, 'MANIFEST.json'))) }));
+24
View File
@@ -0,0 +1,24 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
const directory = path.resolve('docs/live-delivery-ab-evidence');
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'MANIFEST.json')));
const session = JSON.parse(fs.readFileSync(path.join(directory, 'node/session.json'))), observations = [];
for (const row of manifest.files.filter(row => row.path.startsWith('node/artifacts/') && row.path.endsWith('.blob'))) {
const bytes = fs.readFileSync(path.join(directory, row.path)); assert.equal(createHash('sha256').update(bytes).digest('hex'), row.sha256);
let value; try { value = JSON.parse(bytes); } catch { continue; }
if (!value.classification || value.mode !== 'shadow') continue;
assert.ok(value.stepId || value.phase === 'baseline' && value.stepId === null && value.stepIndex === null);
assert.equal(value.taskId, session.execution.task.id); assert.equal(value.runId, session.execution.run.id);
assert.equal(value.delta.changed_files, 0); assert.equal(value.delta.repeated_reads, 0);
observations.push({ path: row.path, sha256: row.sha256, stepIndex: value.stepIndex, phase: value.phase, classification: value.classification, delta: value.delta, reasons: value.reasons, waiting: value.waiting });
}
assert.equal(observations.length, 10);
const classifications = {};
for (const row of observations) classifications[row.classification] = (classifications[row.classification] ?? 0) + 1;
assert.equal(classifications.evidence_growth, 6); assert.equal(classifications.waiting, 2); assert.equal(classifications.new_failure_evidence, 1); assert.equal(classifications.baseline, 1);
const result = { passed: true, sample: 'AB', observations: observations.sort((a, b) => a.stepIndex - b.stepIndex || a.phase.localeCompare(b.phase)), classifications,
scope: 'Ten preserved public-fact shadow records only; no prompt, stop, permission or budget intervention and no engineering-success claim.', realModelRequests: 0 };
fs.writeFileSync('docs/live-delivery-ab-shadow.json', JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify({ passed: true, observations: observations.length, classifications, realModelRequests: 0 }));