test: preserve AA live provider compatibility results

This commit is contained in:
Agent Workbench Dev
2026-11-05 03:21:49 +08:00
parent 4a7af1a6af
commit 6726ce1448
173 changed files with 3880608 additions and 5 deletions
+63
View File
@@ -0,0 +1,63 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { DatabaseSync } from 'node:sqlite';
import { batch, routes, client, saveAt, dataRoot, meta, batchUsage } from './common.mjs';
import { publicSnapshot, digest } from './state.mjs';
import { inspectorClosed } from './inspector.mjs';
const read = file => JSON.parse(fs.readFileSync(path.join(batch, file)));
const before = read('current.json').baseline, { api, close } = await client();
try {
const snapshotFile = 'compatibility-final-state.json';
const after = fs.existsSync(path.join(batch, snapshotFile)) ? read(snapshotFile) : await publicSnapshot('http://127.0.0.1:4318', dataRoot());
if (!fs.existsSync(path.join(batch, snapshotFile))) saveAt(batch, snapshotFile, after);
for (const field of ['pid', 'version', 'runtimeRoot', 'hasKey', 'configSha256']) assert.equal(after[field], before[field], field);
assert.deepEqual(after.active, []); assert.equal(after.exports.length, before.exports.length + 6);
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item);
for (const table of Object.keys(before.database.tables).filter(table => !['projects', 'metadata', 'sqlite_sequence'].includes(table))) {
const rows = new Map(after.database.tables[table].rows.map(row => [row.id, row]));
for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, table + '/' + row.id);
}
// The project picker prepends each new owned project. Reverse exactly these
// six known insertions for the physical position only, preserving all payload bytes.
const db = new DatabaseSync(path.join(dataRoot(), 'workbench.sqlite'), { readOnly: true });
let projectRows, sequence;
try { projectRows = db.prepare('SELECT * FROM projects ORDER BY rowid').all(); sequence = db.prepare('SELECT * FROM sqlite_sequence').all(); } finally { db.close(); }
assert.equal(sequence.length, 1); assert.equal(sequence[0].name, 'events');
assert.equal(sequence[0].seq, Math.max(...after.database.tables.outbox.rows.map(row => row.id)));
const originalSequence = before.database.tables.sqlite_sequence.rows;
assert.equal(originalSequence.length, 1); assert.equal(digest({ name: 'events', seq: originalSequence[0].id }), originalSequence[0].sha256);
assert.ok(sequence[0].seq > originalSequence[0].id);
const originalProjectIds = new Set(before.database.tables.projects.rows.map(row => row.id));
const newIds = routes.flatMap(route => ['off', 'on'].map(mode => read(route.key + '/preflight-' + mode + '/owned.json').project.id));
assert.deepEqual(projectRows.filter(row => !originalProjectIds.has(row.id)).map(row => row.id).sort(), newIds.slice().sort());
for (const row of projectRows.filter(row => originalProjectIds.has(row.id))) {
assert.equal(digest({ ...row, position: row.position - 6 }), before.database.tables.projects.rows.find(value => value.id === row.id).sha256, 'Original project payload and position before six owned prepends: ' + row.id);
}
const metadata = new Map(after.database.tables.metadata.rows.map(row => [row.id, row]));
for (const row of before.database.tables.metadata.rows.filter(row => !['modelAttempts', 'modelNetworkDiagnostics'].includes(row.id))) assert.deepEqual(metadata.get(row.id), row);
const artifacts = new Map(after.persistent.artifacts.map(row => [row.path, row]));
for (const row of before.persistent.artifacts) assert.deepEqual(artifacts.get(row.path), row);
for (const row of read('original-attempts.json')) assert.deepEqual((meta('modelAttempts') ?? []).find(value => value.id === row.id), row);
for (const row of read('original-diagnostics.json')) assert.deepEqual((meta('modelNetworkDiagnostics') ?? []).find(value => value.id === row.id), row);
const observations = routes.map(route => ({ key: route.key, catalog: read(route.key + '/catalog/result.json'),
network: read(route.key + '/catalog/network.json').status,
modes: Object.fromEntries(['off', 'on'].map(mode => {
const result = read(route.key + '/preflight-' + mode + '/result.json');
assert.equal(read(route.key + '/preflight-' + mode + '/verification.json').passed, true);
return [mode, result];
})) }));
const eligible = observations.find(row => row.modes.off.passed)?.key ?? null;
const host = await api('/host/status'); assert.equal(host.activity.active, false); assert.equal(host.desktopWindows.count, 0);
const summary = { passed: true, purpose: 'AA compatibility and preservation audit, not a claim all providers passed',
observations, usage: batchUsage(), engineeringRoute: eligible, nodeExecuted: false, reactPythonExecuted: false, purgeExecuted: false,
blockedReason: eligible ? null : 'No off-mode passed; AA engineering gate not satisfied. No replay or change of mode within AA.',
originalExportsExact: before.exports.length, originalArtifactFilesExact: before.persistent.artifacts.length,
originalRowsExceptSixOwnedProjectPositionPrependsAndEventSequenceExact: true, eventSequenceFrom: originalSequence[0].id, eventSequenceTo: sequence[0].seq,
originalProjectPayloadsExact: true, originalAttemptsExact: read('original-attempts.json').length, originalDiagnosticsExact: true,
originalConfigAndRegistryExact: true, credentialsReadOrResaved: false, pid: host.identity.pid,
instanceId: host.identity.instanceId, windows: host.desktopWindows.count, inspectorClosed: await inspectorClosed() };
saveAt(batch, 'compatibility-summary.json', summary);
console.log(JSON.stringify({ ...summary, observations: observations.map(row => ({ key: row.key, catalogModels: row.catalog.models, network: row.network,
off: row.modes.off.passed, on: row.modes.on.passed })) }));
} finally { await close(); }
+25
View File
@@ -0,0 +1,25 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { batch, sha } from './common.mjs';
const destination = path.resolve('docs/live-validation-aa-evidence'); assert.equal(fs.existsSync(destination), false);
assert.equal(JSON.parse(fs.readFileSync(path.join(batch, 'compatibility-summary.json'))).passed, true);
fs.cpSync(batch, destination, { recursive: true, errorOnExist: true });
fs.cpSync('scripts/live-aa', path.join(destination, 'drivers'), { recursive: true, errorOnExist: true });
for (const id of ['5e95f790-38fe-4afc-a834-41e994c48173', '4e646483-343f-4cf6-a906-c32a32fbd80f', '973a49f4-f5e8-432b-bc51-f6eaaefcfc33']) {
fs.cpSync('.qa/aa-driver-check-' + id, path.join(destination, 'driver-fixtures', id), { recursive: true, errorOnExist: true });
}
const files = [];
function walk(relative) {
for (const entry of fs.readdirSync(path.join(destination, relative), { withFileTypes: true })) {
const name = path.posix.join(relative, entry.name), absolute = path.join(destination, name), stat = fs.lstatSync(absolute);
assert.equal(stat.isSymbolicLink(), false); assert.equal(stat.nlink, 1);
if (entry.isDirectory()) { walk(name); continue; }
assert.equal(entry.isFile(), true); const bytes = fs.readFileSync(absolute); files.push({ path: name, bytes: bytes.length, sha256: sha(bytes) });
}
}
walk(''); files.sort((a, b) => a.path.localeCompare(b.path));
const manifest = { schemaVersion: 1, sample: 'AA-1.25.3-provider-compatibility', files, filesCount: files.length,
totalBytes: files.reduce((sum, row) => sum + row.bytes, 0), note: 'Preserves failures and untested engineering/Purge. A complete evidence inventory is not a claim all compatibility tests passed.' };
fs.writeFileSync(path.join(destination, 'MANIFEST.json'), JSON.stringify(manifest, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify({ success: true, directory: destination, files: files.length, bytes: manifest.totalBytes, manifestSha256: sha(fs.readFileSync(path.join(destination, 'MANIFEST.json'))) }));
+24
View File
@@ -0,0 +1,24 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
const directory = path.resolve(process.argv[2] ?? 'docs/live-validation-aa-evidence');
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const manifestBytes = fs.readFileSync(path.join(directory, 'MANIFEST.json')), manifest = JSON.parse(manifestBytes);
for (const row of manifest.files) {
assert.ok(!row.path.split(/[\\/]/).includes('..') && !path.isAbsolute(row.path));
const bytes = fs.readFileSync(path.join(directory, row.path)); assert.equal(bytes.length, row.bytes); assert.equal(hash(bytes), row.sha256);
}
assert.equal(manifest.filesCount, manifest.files.length); assert.equal(manifest.totalBytes, manifest.files.reduce((sum, row) => sum + row.bytes, 0));
const files = []; const walk = relative => { for (const entry of fs.readdirSync(path.join(directory, relative), { withFileTypes: true })) { const file = path.posix.join(relative, entry.name); entry.isDirectory() ? walk(file) : files.push(file); } }; walk('');
assert.deepEqual(files.filter(file => file !== 'MANIFEST.json').sort(), manifest.files.map(row => row.path).sort());
if (process.argv.includes('--index')) for (const row of [...manifest.files, { path: 'MANIFEST.json', bytes: manifestBytes.length, sha256: hash(manifestBytes) }]) {
const relative = path.relative(process.cwd(), path.join(directory, row.path)).replaceAll('\\', '/');
const bytes = execFileSync('git', ['show', ':' + relative], { windowsHide: true, maxBuffer: Math.max(1024 * 1024, row.bytes + 1024) });
assert.equal(bytes.length, row.bytes, relative); assert.equal(hash(bytes), row.sha256, relative);
}
const result = { passed: true, files: manifest.filesCount, bytes: manifest.totalBytes, manifestSha256: hash(manifestBytes), gitIndex: process.argv.includes('--index'), modelRequests: 0 };
const receiptIndex = process.argv.indexOf('--receipt');
if (receiptIndex >= 0) fs.writeFileSync(process.argv[receiptIndex + 1], JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify(result));