test(m4): prepare fresh provider and engineering validation

This commit is contained in:
Agent Workbench Dev
2026-11-11 22:58:21 +08:00
parent 0054ea2821
commit 7152d654e3
17 changed files with 1717 additions and 0 deletions
+119
View File
@@ -0,0 +1,119 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { Agent, fetch } from 'undici';
import { messagesOutputSetting } from '../../src/shared/contracts/model-capacity.mjs';
export let root = path.resolve('.qa/m4-live-baseline-2026-11-11');
export let origin = 'http://127.0.0.1:4318';
export function configureFixture(options) {
const target = path.resolve(options.root), url = new URL(options.origin);
assert.equal(path.dirname(target), path.resolve('.qa'));
assert.match(path.basename(target), /^m4-driver-[a-f0-9-]{36}$/);
assert.equal(url.hostname, '127.0.0.1'); assert.ok(url.port && url.port !== '4318');
root = target; origin = url.origin;
}
export const sha = bytes => createHash('sha256').update(bytes).digest('hex');
export const read = relative => JSON.parse(fs.readFileSync(path.join(root, relative), 'utf8'));
export function save(relative, value) {
const file = path.resolve(root, relative);
assert.ok(file.startsWith(root + path.sep), 'Evidence must stay inside its new root.');
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, typeof value === 'string' || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
}
export function connectionDefinition(value) {
const { health, status, activeRequests, ...definition } = value;
return definition;
}
export function compareConfig(before, after) {
assert.ok(Number.isInteger(after.configRevision) && after.configRevision >= before.configRevision);
assert.deepEqual({ ...after, configRevision: before.configRevision }, before);
return { originalRevision: before.configRevision, currentRevision: after.configRevision, otherFields: 'exact' };
}
export function compareConnectionDefinitions(before, after, beforeConfig, afterConfig) {
assert.deepEqual(after.connections.map(value => value.id), before.connections.map(value => value.id));
const changes = [];
for (let index = 0; index < before.connections.length; index++) {
const original = connectionDefinition(before.connections[index]), current = connectionDefinition(after.connections[index]);
if (original.id === 'legacy-default') {
// The legacy public projection uses configRevision, rather than a saved
// connection revision. Both deliberate select/restore saves advance it.
const expected = original.revision + afterConfig.configRevision - beforeConfig.configRevision;
assert.equal(current.revision, expected, 'Legacy projection tracks the exact declared config revision increment.');
changes.push({ id: original.id, originalRevision: original.revision, currentRevision: current.revision, reason: 'public-legacy-projection-of-configRevision' });
current.revision = original.revision;
}
assert.deepEqual(current, original, 'All original connection fields: ' + original.id);
}
return { originalFields: 'exact', declaredPublicProjectionChanges: changes };
}
export function expectedOutputFields(connection) {
const fields = Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens']
.filter(key => Object.hasOwn(connection.nativeParameters ?? {}, key)).map(key => [key, connection.nativeParameters[key]]));
if (connection.protocol !== 'anthropic-messages') return { fields, source: Object.keys(fields).length ? 'explicit_native' : 'passthrough_no_local_output_cap' };
const setting = messagesOutputSetting(connection);
if (setting.value !== undefined) fields.max_tokens = setting.value;
return { fields, source: setting.source, declaration: setting };
}
export async function client() {
const dispatcher = new Agent({ connect: { autoSelectFamily: true, timeout: 5000 } }); let token;
const api = async (route, input, timeout = 30000) => {
const response = await fetch(origin + '/api' + route, { dispatcher, method: input === undefined ? 'GET' : 'POST', redirect: 'error', signal: AbortSignal.timeout(timeout),
headers: { ...(token ? { 'X-Workbench-Token': token } : {}), 'Content-Type': 'application/json', Connection: 'close' },
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const value = await response.json();
if (!response.ok) throw Object.assign(new Error(route + ': ' + (value.error ?? response.status)), { status: response.status, response: value });
return value;
};
const boot = await api('/bootstrap'); token = boot.token;
api.download = async (route, timeout = 30000) => {
const response = await fetch(origin + '/api' + route, { dispatcher, method: 'GET', redirect: 'error', signal: AbortSignal.timeout(timeout),
headers: { 'X-Workbench-Token': token, Connection: 'close' } });
if (!response.ok) throw Object.assign(new Error(route + ': artifact download HTTP ' + response.status), { status: response.status });
return Buffer.from(await response.arrayBuffer());
};
// The bootstrap authentication token is only kept in this client's closure.
delete boot.token;
return { api, boot, close: () => dispatcher.close() };
}
export async function sessionAttempts(api, sessionId) {
let cursor, total, snapshot; const rows = [], references = [];
do {
const page = await api('/model-center/attempts?sessionId=' + sessionId + '&limit=50' + (cursor ? '&cursor=' + encodeURIComponent(cursor) : ''));
if (total === undefined) { total = page.total; snapshot = page.snapshotSequence; }
assert.equal(page.total, total); assert.equal(page.snapshotSequence, snapshot);
rows.push(...page.items); references.push(...page.references); cursor = page.nextCursor;
assert.equal(page.hasMore, !!cursor);
} while (cursor);
assert.equal(rows.length, total); assert.equal(new Set(rows.map(value => value.id)).size, total);
// Public pages are newest first. Store experiment receipts in dispatch order.
return { rows: rows.reverse(), references: references.reverse(), total, snapshot };
}
export async function artifactBytes(api, sessionId, artifact) {
assert.equal(typeof api.download, 'function', 'Original artifacts need the byte-preserving download path.');
const bytes = await api.download('/artifacts/' + artifact.id + '?session=' + sessionId + '&download=1');
assert.equal(sha(bytes), artifact.sha256); if (artifact.bytes !== undefined) assert.equal(bytes.length, artifact.bytes);
return bytes;
}
export async function captureSession(api, sessionId, prefix) {
const state = await api('/sessions/' + sessionId), exported = await api('/sessions/' + sessionId + '/export');
save(prefix + '/session.json', state); save(prefix + '/export.json', exported);
const ledger = await sessionAttempts(api, sessionId); save(prefix + '/attempts.json', ledger.rows); save(prefix + '/ledger-references.json', ledger);
const captures = [], capturedIds = new Set();
for (const row of ledger.rows) for (const kind of ['request', 'response']) {
const id = row[kind + 'ArtifactId']; if (!id) continue;
const artifact = state.execution.artifacts.find(value => value.id === id) ?? { id, sha256: row[kind + 'Sha256'] };
const bytes = await artifactBytes(api, sessionId, artifact);
assert.equal(sha(bytes), row[kind + 'Sha256']);
const relative = prefix + '/captures/' + row.id + '-' + kind + (kind === 'request' ? '.json' : row.streaming ? '.sse' : '.txt');
save(relative, bytes); captures.push({ attemptId: row.id, artifactId: id, kind, path: relative, sha256: sha(bytes), bytes: bytes.length }); capturedIds.add(id);
}
for (const artifact of state.execution.artifacts) {
if (capturedIds.has(artifact.id) || !/^(text\/|application\/(json|x-ndjson))/.test(artifact.mediaType ?? '')) continue;
const bytes = await artifactBytes(api, sessionId, artifact), relative = prefix + '/artifacts/' + artifact.id + (artifact.mediaType === 'application/json' ? '.json' : '.txt');
save(relative, bytes); captures.push({ artifactId: artifact.id, kind: artifact.kind, path: relative, sha256: sha(bytes), bytes: bytes.length });
}
save(prefix + '/artifact-sources.json', captures);
return { state, exported, ...ledger, captures };
}
+79
View File
@@ -0,0 +1,79 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { root, save, read, client, captureSession, connectionDefinition, expectedOutputFields } from './common.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
const frozen = read('frozen.json'); assert.equal(fs.existsSync(path.join(root, 'compatibility')), false, 'Never replay a compatibility batch.');
save('compatibility/claim.json', { id: randomUUID(), at: new Date().toISOString(), scopes: ['text', 'tools'], streaming: 'Separately assessed from each configured streaming request; no redundant real request.', noRetry: true });
const { api, boot, close } = await client(), results = [];
try {
assert.equal(boot.version, '1.25.13'); await inspectorClosed();
assert.equal((await api('/host/status')).identity.pid, frozen.host.identity.pid);
for (const connection of frozen.connections) {
const prefix = 'compatibility/' + connection.id;
assert.deepEqual(connectionDefinition((await api('/model-center')).connections.find(value => value.id === connection.id)), connectionDefinition(connection), 'Frozen connection changed.');
const result = { connectionId: connection.id, name: connection.name, model: connection.model, protocol: connection.protocol,
endpointKind: new URL(connection.baseUrl).hostname === 'api.deepseek.com' ? 'official-deepseek-endpoint' : 'compatible-proxy-endpoint', preflights: [], media: 'not_exercised_in_this_slice' };
console.log(JSON.stringify({ phase: 'compatibility', connection: connection.name, protocol: connection.protocol }));
let started;
try {
started = await api('/model-center/network-diagnostics', { id: connection.id, revision: connection.revision }); save(prefix + '/network-start.json', started);
const deadline = Date.now() + 30000; let network;
while (true) {
network = (await api('/model-center/network-diagnostics')).jobs.find(value => value.id === started.job.id);
if (network?.finishedAt) break;
if (Date.now() >= deadline) { network = (await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {})).job; result.networkObservationEnded = true; break; }
await delay(250);
}
save(prefix + '/network.json', network); result.network = network.status;
assert.equal(network.httpRequests, 0); assert.equal(network.modelRequests, 0); assert.equal(network.credentialAccess, false);
} catch (error) {
result.network = 'error'; save(prefix + '/network-error.json', { error: error.stack, response: error.response, noRetry: true });
if (started?.job?.id) save(prefix + '/network-owned-stop.json', await api('/model-center/network-diagnostics/' + started.job.id + '/stop', {}));
}
const requestId = randomUUID();
try {
const catalog = await api('/model-center/catalog', { id: connection.id, revision: connection.revision, requestId }, 90000);
save(prefix + '/catalog.json', catalog); result.catalog = { complete: catalog.complete, models: catalog.entries.length, selectedListed: catalog.entries.some(value => value.id === connection.model),
evidenceScope: 'Product catalog preview including its supplied source metadata; no invented raw HTTP artifact.' };
} catch (error) { result.catalog = { complete: false, error: error.message, response: error.response, noRetry: true }; save(prefix + '/catalog-error.json', result.catalog); }
finally { save(prefix + '/catalog-cancel.json', await api('/model-center/catalog/cancel', { id: requestId })); }
const projectRoot = path.join(root, prefix, 'project'); fs.mkdirSync(projectRoot, { recursive: true });
fs.writeFileSync(path.join(projectRoot, 'README.md'), 'M4 independent compatibility observations only. No engineering implementation.\n', { flag: 'wx' });
const project = await api('/projects', { root: projectRoot }); result.projectId = project.id;
for (const scope of ['text', 'tools']) {
const session = await api('/sessions', { projectId: project.id });
const input = { pid: frozen.host.identity.pid, port: 4318, runtimeRoot: boot.runtimeRoot, projectId: project.id, sessionId: session.id, connection,
scope, nonce: randomUUID(), expectedOutput: expectedOutputFields(connection), observationMs: 180000 };
save(prefix + '/' + scope + '/input.json', input); let invoked;
try { invoked = await invokeProbe(input); save(prefix + '/' + scope + '/invocation.json', invoked); }
catch (error) { save(prefix + '/' + scope + '/invocation-error.json', { error: error.stack, noRetry: true }); }
const captured = await captureSession(api, session.id, prefix + '/' + scope), attempt = captured.rows[0];
const request = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'request');
const response = captured.captures.find(value => value.attemptId === attempt?.id && value.kind === 'response');
const body = request ? JSON.parse(fs.readFileSync(path.join(root, request.path), 'utf8')) : null;
const raw = response ? fs.readFileSync(path.join(root, response.path), 'utf8') : '';
const outputFields = body ? Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]])) : null;
const streaming = { requested: body?.stream === true, sseObserved: /^(?:event:|data:)/m.test(raw), complete: attempt?.status === 'completed' && attempt?.modelIdentity?.complete === true };
const wirePassed = !!body && JSON.stringify(outputFields) === JSON.stringify(input.expectedOutput.fields) && (!body.stream || streaming.sseObserved);
const passed = !!invoked?.result.contentPassed && invoked.result.actualToolsExecuted === 0 && captured.total === 1 && wirePassed
&& attempt.status === 'completed' && attempt.modelIdentity?.status === 'matched' && attempt.modelIdentity.complete === true;
const entry = { scope, passed, sessionId: session.id, taskId: invoked?.result.taskId, runId: invoked?.result.runId, identity: attempt?.modelIdentity ?? null,
usage: attempt?.usage ?? null, usageStatus: attempt?.usageStatus ?? 'not_dispatched', status: attempt?.status ?? 'not_dispatched',
diagnostic: invoked?.result.diagnostic ?? null, actualToolsExecuted: invoked?.result.actualToolsExecuted ?? null, requests: captured.total,
streaming, wirePassed, outputFields, expectedOutput: input.expectedOutput, captures: captured.captures };
save(prefix + '/' + scope + '/result.json', entry); result.preflights.push(entry);
console.log(JSON.stringify({ connection: connection.name, scope, passed, identity: entry.identity?.status, usageStatus: entry.usageStatus, streaming, outputFields, diagnostic: entry.diagnostic?.kind }));
assert.ok(captured.total <= 1); await inspectorClosed();
}
result.engineeringAllowed = result.catalog.complete && result.catalog.selectedListed && result.preflights.every(value => value.passed && value.usageStatus === 'reported');
save(prefix + '/result.json', result); results.push(result);
}
const center = await api('/model-center');
assert.deepEqual(center.selected, frozen.center.selected); assert.deepEqual(center.connections.map(connectionDefinition), frozen.center.connections.map(connectionDefinition)); assert.deepEqual(center.pools, frozen.center.pools);
save('compatibility/result.json', { completed: true, results, requests: results.reduce((sum, row) => sum + row.preflights.reduce((n, value) => n + value.requests, 0), 0),
connectionDefinitionsPoolsSelection: 'exact', healthChangesRetained: true, keysReadOrResaved: false, historicalSamplesReplayed: false });
} finally { await inspectorClosed(); await close(); }
+25
View File
@@ -0,0 +1,25 @@
import fs from 'node:fs';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { root, origin, save, client, expectedOutputFields } from './common.mjs';
import { publicSnapshot } from '../retained-host-state.mjs';
import { inspectorClosed } from './inspector.mjs';
assert.ok(fs.existsSync(root)); assert.equal(fs.existsSync(root + '/frozen.json'), false, 'Freeze this batch once.');
assert.equal(execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8', windowsHide: true }).trim(), '', 'Commit and verify the new driver before real requests.');
const { api, boot, close } = await client();
try {
await inspectorClosed(); const host = await api('/host/status'), center = await api('/model-center');
assert.equal(boot.version, '1.25.13'); assert.equal(host.activity.active, false);
for (const [key, expected] of Object.entries({ executionMode: 'unrestricted', capacityMode: 'passthrough', approvalMode: 'full', maxSteps: 0, maxRunTokens: 0, maxTaskTokens: 0 })) assert.equal(boot.config[key], expected, key);
const connections = center.connections.filter(value => value.enabled && value.hasKey); assert.ok(connections.length > 0, 'Restore credentials in Model Center before freezing.');
const priority = connections.slice().sort((a, b) => {
const rank = value => new URL(value.baseUrl).hostname === 'api.deepseek.com' ? 0 : value.protocol === 'responses' ? 1 : value.protocol === 'chat-completions' ? 2 : 3;
return rank(a) - rank(b) || a.id.localeCompare(b.id);
}).map(value => value.id);
const baseline = await publicSnapshot(origin, host.identity.dataRoot);
save('frozen.json', { observedAt: new Date().toISOString(), host, config: boot.config, center, connections, baseline, originalProjects: boot.projects,
engineeringPriority: priority, outputFields: Object.fromEntries(connections.map(value => [value.id, expectedOutputFields(value)])),
driverCommit: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(), modelRequests: 0, historicalSamplesReplayed: false, keysReadOrResaved: false });
console.log(JSON.stringify({ frozen: true, version: boot.version, pid: host.identity.pid, connections: connections.map(value => ({ id: value.id, name: value.name, protocol: value.protocol, model: value.model, revision: value.revision })), priority, modelRequests: 0 }));
} finally { await close(); }
+56
View File
@@ -0,0 +1,56 @@
import assert from 'node:assert/strict';
import net from 'node:net';
import { setTimeout as delay } from 'node:timers/promises';
import { providerPreflight } from './payload.mjs';
export async function inspectorClosed() {
return new Promise((resolve, reject) => {
const socket = net.createConnection({ host: '127.0.0.1', port: 9229 }); socket.setTimeout(1500);
socket.once('connect', () => { socket.destroy(); reject(Error('Inspector already open.')); });
socket.once('error', error => { socket.destroy(); error.code === 'ECONNREFUSED' ? resolve(true) : reject(error); });
socket.once('timeout', () => { socket.destroy(); reject(Error('Inspector state unknown.')); });
});
}
class Protocol {
constructor(socket) {
this.socket = socket; this.nextId = 1; this.pending = new Map();
socket.addEventListener('message', event => {
const message = JSON.parse(String(event.data)); if (!message.id) return;
const pending = this.pending.get(message.id); if (!pending) return;
this.pending.delete(message.id); clearTimeout(pending.timer);
message.error ? pending.reject(Error(message.error.message)) : pending.resolve(message.result);
});
socket.addEventListener('close', () => { for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(Error('Inspector closed; outcome unknown.')); } this.pending.clear(); });
}
send(method, params, timeout = 10000) {
return new Promise((resolve, reject) => {
const id = this.nextId++, timer = setTimeout(() => { this.pending.delete(id); reject(Error('Inspector timeout; do not replay.')); }, timeout);
this.pending.set(id, { resolve, reject, timer }); this.socket.send(JSON.stringify({ id, method, params }));
});
}
}
export async function invokeProbe(input) {
await inspectorClosed(); let socket, protocol, enabled = false;
try {
process._debugProcess(input.pid); enabled = true; let targets;
for (let index = 0; index < 40 && !targets; index++) {
await delay(100);
try { targets = await (await fetch('http://127.0.0.1:9229/json/list', { signal: AbortSignal.timeout(700) })).json(); } catch {}
}
assert.equal(targets?.length, 1); assert.ok(targets[0].webSocketDebuggerUrl.startsWith('ws://127.0.0.1:9229/'));
socket = new WebSocket(targets[0].webSocketDebuggerUrl);
await new Promise((resolve, reject) => { socket.addEventListener('open', resolve, { once: true }); socket.addEventListener('error', reject, { once: true }); });
protocol = new Protocol(socket);
const identity = await protocol.send('Runtime.evaluate', { expression: 'process.pid', returnByValue: true }); assert.equal(identity.result.value, input.pid);
const expression = '(' + providerPreflight.toString() + ')(' + JSON.stringify(input) + ')';
const result = await protocol.send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true }, input.observationMs + 30000);
if (result.exceptionDetails) throw Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
return { result: result.result.value, expression };
} finally {
if (protocol && enabled) {
await protocol.send('Runtime.evaluate', { expression: "setTimeout(() => process.getBuiltinModule('inspector').close(), 25); 'closing'", returnByValue: true });
await delay(200);
}
socket?.close(); await inspectorClosed();
}
}
+66
View File
@@ -0,0 +1,66 @@
// A one-shot compatibility observation in a new owned session. This function
// calls the existing Provider; it never reads or returns any credential value.
export async function providerPreflight(input) {
const ensure = (value, message) => { if (!value) throw Error(message); };
ensure(process.pid === input.pid, 'Host PID changed.');
const servers = process._getActiveHandles().filter(h => h?.listening && h.address?.()?.address === '127.0.0.1' && h.address?.()?.port === input.port);
ensure(servers.length === 1, 'Expected one loopback host.');
const { store, provider, runtime } = servers[0].workbenchHostAuthority.context;
ensure(runtime.appDirectory === input.runtimeRoot, 'Runtime changed.'); ensure(!runtime.active.size, 'Host is busy.');
const session = store.session(input.sessionId), project = store.project(session.projectId), hub = provider.modelHub(store);
ensure(session.projectId === input.projectId && !session.runId && !session.messages.length, 'Only a fresh owned session may run.');
const connection = hub.view().connections.find(value => value.id === input.connection.id);
ensure(connection?.enabled && connection.hasKey && connection.revision === input.connection.revision
&& connection.credentialRevision === input.connection.credentialRevision && connection.model === input.connection.model
&& connection.protocol === input.connection.protocol && connection.baseUrl === input.connection.baseUrl, 'Frozen connection changed.');
globalThis.__workbenchM4Preflights ??= new Set();
ensure(!globalThis.__workbenchM4Preflights.has(session.id), 'A preflight cannot be replayed.'); globalThis.__workbenchM4Preflights.add(session.id);
const config = hub.prepare({ ...store.config, mode: 'compatible' }, { type: 'connection', id: connection.id });
ensure(config.capacityMode === 'passthrough' && config.executionMode === 'unrestricted', 'Keep default passthrough.');
// A single physical request is this experiment's design, not a saved quota.
config.modelRouting.maxAttempts = 1;
config.modelRouting.members = config.modelRouting.members.map(value => ({ ...value, modelIdentityPolicy: 'require-match',
...(input.scope === 'streaming' ? { streaming: true } : {}) }));
const tools = input.scope === 'tools' ? [{ type: 'function', function: { name: 'workbench_m4_echo_nonce', description: 'Echo the supplied compatibility nonce. No file or command operation.',
parameters: { type: 'object', properties: { nonce: { type: 'string' } }, required: ['nonce'], additionalProperties: false } } }] : [];
const prompt = input.scope === 'tools' ? 'Call workbench_m4_echo_nonce exactly once with nonce ' + input.nonce + '. No other task.' : 'Reply exactly: OK ' + input.nonce;
const run = store.beginRun(session, prompt, false, config, { forceNewTask: true }), user = store.message(session, 'user', prompt);
run.inputMessage = { id: user.id, providerIndex: 0, origin: 'user' }; session.providerMessages.push({ role: 'user', content: prompt });
session.status = 'running'; const step = store.beginStep(session, run), planner = runtime.requestPlanner();
const controller = new AbortController(), timer = setTimeout(() => controller.abort(Error('Predeclared M4 observation ended; do not replay.')), input.observationMs);
let plan, response, diagnostic = null, event, receipts = 0; const wireIds = new Set();
try {
plan = planner.prepare(session, run, config, { system: 'Connection compatibility preflight only. Follow the user exactly. Do not perform project work.', tools, stepId: step.id });
const variants = plan.body.kind === 'model-route-plan' ? plan.body.variants.map(value => value.body) : [plan.body];
ensure(variants.length === 1, 'A frozen preflight must have one route.');
for (const body of variants) {
const actual = Object.fromEntries(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].filter(key => Object.hasOwn(body, key)).map(key => [key, body[key]]));
ensure(JSON.stringify(actual) === JSON.stringify(input.expectedOutput.fields), 'Output fields must equal explicit settings or the Messages protocol requirement.');
}
response = await provider.complete(config, plan.messages, project, controller.signal, progress => {
if (progress.consumption) { receipts += Number(store.recordConsumption(session, run, { ...progress.consumption, requestId: plan.record.id }, plan.record.tokenReservation?.reservedTokens ?? plan.record.estimatedInputTokens)); return; }
if (progress.phase === 'running' && progress.wireAttemptId) { wireIds.add(progress.wireAttemptId); ensure(wireIds.size === 1, 'No retry permitted.'); }
event ??= store.event(session, { kind: 'model', source: 'model_connection', label: 'M4 单次兼容预检', phase: 'running', runId: run.id, contextPlanId: plan.record.id, startedAt: new Date().toISOString() });
for (const key of ['modelAttemptId', 'connectionId', 'modelIdentity', 'requestUrl', 'diagnostic', 'stream', 'phase']) if (progress[key] !== undefined) event[key] = progress[key];
plan.record.requestAttempts = wireIds.size; step.modelAttempts = wireIds.size; store.save(session);
}, plan.record, tools, { attempts: 1, maxBudgetAttempts: 1 });
planner.finish(session, plan.record, { response, attempts: wireIds.size });
// An observed tool call is not executed or inserted as an unpaired call.
if (!response.message.tool_calls?.length) session.providerMessages.push(response.message);
store.message(session, 'assistant', response.message.content || '已收到原工具调用,预检不执行工具。'); session.status = 'completed';
} catch (cause) {
diagnostic = cause.diagnostic ?? { source: 'workbench', origin: 'local', stage: 'preflight', kind: controller.signal.aborted ? 'observation_stop' : 'runtime', message: provider.redact(cause.message) };
session.status = 'failed'; session.lastFailure = { ...diagnostic, runId: run.id }; store.message(session, 'system', diagnostic.message);
if (plan) planner.finish(session, plan.record, { error: new Error(diagnostic.message), attempts: wireIds.size });
} finally {
clearTimeout(timer); if (event) { event.phase = diagnostic ? 'failed' : 'completed'; event.finishedAt = new Date().toISOString(); }
store.finishStep(session, step, diagnostic ? 'failed' : 'completed'); store.finishRun(session, run, diagnostic ? diagnostic.kind : 'm4_compatibility_preflight');
}
const ledger = hub.ledger.page({ filters: { sessionId: session.id }, limit: 2 }); ensure(ledger.total <= 1 && ledger.items.length === ledger.total, 'Single-request contract violated.');
const calls = response?.message?.tool_calls ?? []; let contentPassed = false;
try { contentPassed = input.scope === 'tools' ? calls.length === 1 && calls[0].function.name === 'workbench_m4_echo_nonce'
&& JSON.parse(calls[0].function.arguments).nonce === input.nonce : !calls.length && response?.message?.content?.trim() === 'OK ' + input.nonce; } catch {}
return { sessionId: session.id, taskId: run.taskId, runId: run.id, scope: input.scope, status: session.status, diagnostic,
attempts: ledger.total, wireStarts: wireIds.size, receipts, actualToolsExecuted: run.toolCalls, contentPassed,
outputFieldsSource: input.expectedOutput.source, savedConfigurationChanged: false, message: response?.message ?? null, usage: run.usage };
}
+42
View File
@@ -0,0 +1,42 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { execFileSync, spawnSync } from 'node:child_process';
import { gitEnvironment } from '../../src/core/git-process.mjs';
import { prepareProject as prepareSecondary, python, specs as secondarySpecs } from '../live-ae/project.mjs';
import { sha } from './common.mjs';
// The old project module supplies only preparation/check helpers. It does not
// load an old claim, connection, session, model result or real-batch entry point.
export { python };
export const specs = { node: { baseline: 'benchmarks/live-delivery-v1/node', allowed: ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs'], tests: 9 }, ...secondarySpecs };
export function prepareProject(kind, output, save) {
if (kind !== 'node') return prepareSecondary(kind, output, save);
const spec = specs.node, sourceRoot = path.join(output, 'project'); assert.equal(fs.existsSync(sourceRoot), false);
fs.mkdirSync(output, { recursive: true }); fs.cpSync(spec.baseline, sourceRoot, { recursive: true, errorOnExist: true });
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', sourceRoot, ...args], { encoding: 'utf8', windowsHide: true, env: gitEnvironment() });
fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# M4 independent Node baseline\nRepair the three src modules using the original nine tests. Preserve owner bytes and their staging.\n', { flag: 'wx' });
fs.writeFileSync(path.join(sourceRoot, '.gitignore'), 'dist/\n.awb-*\nnode_modules/\n', { flag: 'wx' });
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) fs.writeFileSync(path.join(sourceRoot, name), 'Owner baseline\n', { flag: 'wx' });
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); git(['config', 'user.name', 'Workbench QA']); git(['config', 'user.email', 'qa@workbench.invalid']);
git(['add', '--', '.']); git(['commit', '--no-gpg-sign', '-m', 'Seal M4 independent Node original baseline']); const initialHead = git(['rev-parse', 'HEAD']).trim();
fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner staged change\n'); git(['add', '--', 'owner-staged.txt']);
fs.writeFileSync(path.join(sourceRoot, 'owner-staged.txt'), 'Owner unstaged over original staging\n');
fs.writeFileSync(path.join(sourceRoot, 'owner-unstaged.txt'), 'Owner unstaged change\n'); fs.writeFileSync(path.join(sourceRoot, 'owner-untracked.txt'), 'Owner untracked change\n', { flag: 'wx' });
const all = git(['ls-files', '-z']).split('\0').filter(Boolean), protectedNames = all.filter(name => !spec.allowed.includes(name)).concat('owner-untracked.txt');
const protectedBytes = Object.fromEntries(protectedNames.map(name => [name, fs.readFileSync(path.join(sourceRoot, name))]));
const index = git(['ls-files', '--stage', '--', ...protectedNames]);
const protect = () => {
for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(sourceRoot, name)), bytes, 'Protected file: ' + name);
assert.equal(git(['ls-files', '--stage', '--', ...protectedNames]), index, 'All protected original index entries');
};
const check = (name, args, prefix) => {
const execution = spawnSync(process.execPath, args, { cwd: sourceRoot, windowsHide: true, encoding: 'utf8', env: gitEnvironment(), maxBuffer: 16 * 1024 * 1024 });
save(prefix + '/' + name + '.txt', (execution.stdout ?? '') + (execution.stderr ?? ''));
const stats = name === 'test' ? Object.fromEntries(['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo'].map(key => [key, Number(execution.stdout?.match(new RegExp('^# ' + key + ' (\\d+)$', 'm'))?.[1] ?? NaN)])) : null;
return { status: execution.status, signal: execution.signal, error: execution.error?.message ?? null, stats, stdoutSha256: sha(execution.stdout ?? ''), stderrSha256: sha(execution.stderr ?? '') };
};
const checks = prefix => { const result = { test: check('test', ['--test', '--test-reporter=tap', 'test/expense.test.mjs'], prefix), build: check('build', ['scripts/build.mjs'], prefix) }; save(prefix + '/result.json', result); protect(); return result; };
save('source-before.json', { initialHead, status: git(['status', '--porcelain=v1']), protectedIndex: index, files: Object.fromEntries([...all, 'owner-untracked.txt'].map(name => [name, sha(fs.readFileSync(path.join(sourceRoot, name)))])) });
return { spec, sourceRoot, git, initialHead, protect, checks };
}
+98
View File
@@ -0,0 +1,98 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { root, save, read, client, sha, captureSession, connectionDefinition, compareConfig, compareConnectionDefinitions } from './common.mjs';
import { prepareProject, python, specs } from './project.mjs';
import { verifyLiveReactClient } from '../live-react-client.mjs';
const kind = process.argv[2]; assert.ok(Object.hasOwn(specs, kind));
const frozen = read('frozen.json'), compatibility = read('compatibility/result.json');
if (kind !== 'node') assert.equal(read((kind === 'react' ? 'node' : 'react') + '/result.json').success, true, 'The previous engineering baseline must fully pass.');
const connectionId = kind === 'node' ? frozen.engineeringPriority.find(id => compatibility.results.some(value => value.connectionId === id && value.engineeringAllowed)) : read('node/result.json').connectionId;
assert.equal(fs.existsSync(path.join(root, kind)), false, 'Do not replay an engineering sample.');
if (!connectionId) { save(kind + '/result.json', { success: false, executed: false, reason: 'No connection passed the predeclared compatibility gate.', purgeExecuted: false }); process.exit(0); }
const { api, boot, close } = await client(); let session, route, selected = false, original, own, failure, result, state, selectionRevision;
const observations = [], facts = new Set(), output = path.join(root, kind), store = (file, value) => save(kind + '/' + file, value);
const settle = async () => {
if (!route) return; state = await api(route);
if (['running', 'stopping', 'awaiting_approval'].includes(state.status)) { await api(route + '/stop', {}, 60000); state = await api(route); }
assert.ok(!['running', 'stopping', 'awaiting_approval'].includes(state.status), 'Only a quiescent owned task can be independently checked.');
};
try {
const host = await api('/host/status'); assert.equal(host.activity.active, false); assert.equal(host.identity.pid, frozen.host.identity.pid); assert.equal(boot.version, '1.25.13');
compareConfig(frozen.config, boot.config); const center = await api('/model-center'); assert.deepEqual(center.selected, frozen.center.selected); assert.deepEqual(center.pools, frozen.center.pools);
original = center.connections.find(value => value.id === connectionId);
assert.deepEqual(connectionDefinition(original), connectionDefinition(frozen.connections.find(value => value.id === connectionId)));
store('claim.json', { id: randomUUID(), kind, connection: original, config: boot.config, source: specs[kind].baseline, allowed: specs[kind].allowed,
observationMs: 600000, noNewFactMs: 180000, implementationSupplied: false, supervisorContinuation: false, localTokenCaps: false, identityPolicySaved: false });
own = prepareProject(kind, output, store); const baseline = own.checks('baseline'); assert.equal(baseline.test.stats.tests, own.spec.tests); assert.ok(baseline.test.stats.fail > 0); assert.equal(baseline.build.status, 0);
const project = await api('/projects', { root: own.sourceRoot }); session = await api('/sessions', { projectId: project.id }); route = '/sessions/' + session.id; store('owned.json', { project, session });
const selection = await api('/model-center/select', { type: 'connection', id: connectionId }); selected = true; selectionRevision = selection.config.configRevision;
const goals = {
node: '请独立修复这个Node费用报表工程,使原九项测试和build通过,并完成本地Git交付。只修改src/amount.mjs、src/summary.mjs、src/report.mjs;按原测试理解金额、币种汇总和报表要求,不修改原测试、package配置、构建脚本、README或owner文件,保留原有暂存/未暂存/未跟踪状态。请自行读取、修改、执行检查并提交。',
react: '独立完成这个React/TypeScript看板工程README要求,修复状态筛选、去空白搜索、实际统计、按目标任务切换与空态,保留既有样式和布局。只修改src/query.ts、src/summary.ts、src/transition.ts、src/App.tsx。自行读取源码及原九项测试、运行test/typecheck/build并完成仅这四文件的本地Git提交。依赖已按原锁离线准备;不可改原测试、配置、锁、其它源码/样式、README和owner文件,保留原暂存/未暂存/未跟踪状态。',
python: '独立完成这个Python任务报表工程README的规范化、汇总与渲染契约。只修改normalize.py、summary.py、render.py。自行读取源码及原十项测试,运行pytest与源码编译并完成仅这三文件的本地Git提交。现成解释器为 ' + python + ',已装pytest,无需安装依赖。不可改原测试、配置、README和owner文件,保留原暂存/未暂存/未跟踪状态。',
};
const prompt = goals[kind] + ' 当前为完全访问权限,可用工程工具或终端自动执行,无需独立审查,不要只解释方案或口头完成,不推送远端。';
store('goal.txt', prompt); await api(route + '/messages', { content: prompt }); const started = Date.now(); let novel = started, signature = '';
while (true) {
state = await api(route); own.protect();
for (const event of state.events.filter(value => value.kind === 'tool' && value.phase === 'completed' && ['read_file', 'read_artifact', 'list_files', 'query_code', 'search_files'].includes(value.tool))) {
const key = event.tool + JSON.stringify(event.input) + sha(event.output ?? ''); if (!facts.has(key)) { facts.add(key); novel = Date.now(); }
}
const current = JSON.stringify({ files: own.spec.allowed.map(name => sha(fs.readFileSync(path.join(own.sourceRoot, name)))), checks: state.execution.checks.map(value => ({ status: value.status, source: value.sourceFingerprint })) });
if (current !== signature) { signature = current; novel = Date.now(); }
const observation = { kind, status: state.status, requests: state.execution.task?.usage.requests, tokens: state.execution.task?.usage.totalTokens, facts: facts.size,
changes: state.changes.length, checks: state.execution.checks.map(value => value.status), lastTool: state.events.filter(value => value.kind === 'tool').at(-1)?.tool };
if (JSON.stringify(observation) !== JSON.stringify(observations.at(-1))) { observations.push(observation); console.log(JSON.stringify(observation)); }
if (!['running', 'stopping', 'awaiting_approval'].includes(state.status)) break;
if (Date.now() - started > 600000 || Date.now() - novel > 180000) { failure = { reason: 'experiment_observation_ended', elapsedMs: Date.now() - started, noNewFactMs: Date.now() - novel, productBudgetChanged: false }; break; }
await delay(750);
}
await settle(); own.protect(); const final = own.checks('independent-final'); let browser = null;
if (kind === 'react' && Object.values(final).every(value => value.status === 0)) {
try { browser = await verifyLiveReactClient(own.sourceRoot, path.join(output, 'browser')); } catch (error) { browser = { success: false, error: error.stack }; } store('browser/result.json', browser);
}
const commit = own.git(['rev-parse', 'HEAD']).trim(), paths = own.git(['diff', '--name-only', own.initialHead, commit]).trim().split(/\r?\n/).filter(Boolean).sort();
const committedCurrent = own.spec.allowed.every(name => sha(Buffer.from(own.git(['show', commit + ':' + name]))) === sha(fs.readFileSync(path.join(own.sourceRoot, name))));
const otherUntracked = own.git(['ls-files', '--others', '--exclude-standard']).trim().split(/\r?\n/).filter(name => name && name !== 'owner-untracked.txt');
result = { success: !failure && state.status === 'completed' && Object.values(final).every(value => value.status === 0) && final.test.stats.tests === own.spec.tests && final.test.stats.pass === own.spec.tests
&& commit !== own.initialHead && JSON.stringify(paths) === JSON.stringify(own.spec.allowed.slice().sort()) && committedCurrent && !otherUntracked.length && (kind !== 'react' || browser?.success === true),
kind, executed: true, sessionId: session.id, projectId: project.id, connectionId, commit, initialHead: own.initialHead, committedPaths: paths, committedCurrent, otherUntracked,
final, browser, failure: failure ?? null, productStopReason: state.execution.run?.stopReason ?? null, ownerBytesAndIndexPreserved: true, supervisorContinuations: 0, localTokenCapsChanged: false };
} catch (error) { result = { success: false, kind, executed: !!session, failure: { error: error.stack, noRetry: true }, sessionId: session?.id, connectionId }; }
finally {
const errors = [], restoration = {}; const guard = async (label, fn) => { try { await fn(); } catch (error) { errors.push({ label, error: error.stack }); } };
if (route) {
await guard('settle-owned-task', settle);
await guard('capture-owned-task', async () => {
const captured = await captureSession(api, session.id, kind);
const identitiesPassed = captured.total > 0 && captured.rows.every(value => value.connectionId === connectionId && value.status === 'completed' && value.modelIdentity?.complete && value.modelIdentity.status === 'matched')
&& captured.state.execution.task.usage.unknownRequests === 0 && captured.state.execution.task.usage.requests === captured.total;
result = { ...result, success: result.success && identitiesPassed, identitiesPassed, requests: captured.total, usage: captured.state.execution.task.usage, taskId: captured.state.execution.task?.id,
runId: captured.state.execution.run?.id, captures: captured.captures, internalDeliveries: captured.state.execution.deliveries.map(value => ({ id: value.id, status: value.status, commit: value.commit })),
reportArtifacts: captured.state.execution.artifacts.filter(value => /^delivery-report/.test(value.kind)).map(value => ({ id: value.id, kind: value.kind, sha256: value.sha256 })) };
store('git-status.txt', own.git(['status', '--porcelain=v1'])); store('git-log.txt', own.git(['log', '--format=fuller', '--stat', own.initialHead + '..HEAD']));
});
}
await guard('restore-selection', async () => {
if (!selected) return;
const center = await api('/model-center'), now = await api('/bootstrap');
if (center.selected.type === 'connection' && center.selected.id === connectionId && now.config.configRevision === selectionRevision) {
await api('/model-center/select', frozen.center.selected); restoration.selection = frozen.center.selected;
} else restoration.externalSelectionOrConfigChangePreserved = true;
});
await guard('verify-config-and-definitions', async () => {
restoration.config = (await api('/bootstrap')).config;
if (!restoration.externalSelectionOrConfigChangePreserved) restoration.configComparison = compareConfig(frozen.config, restoration.config);
const center = await api('/model-center');
restoration.connectionDefinitions = compareConnectionDefinitions(frozen.center, center, frozen.config, restoration.config);
assert.deepEqual(center.pools, frozen.center.pools); restoration.pools = 'exact';
});
restoration.keysReadOrResaved = false; restoration.errors = errors; if (errors.length) result = { ...result, success: false, cleanupErrors: errors };
store('restoration.json', restoration); store('observations.json', observations); store('result.json', result); await close();
}
console.log(JSON.stringify({ kind, success: result.success, requests: result.requests, usage: result.usage, commit: result.commit, failure: result.failure, cleanupErrors: result.cleanupErrors }));
if (!result.success) process.exitCode = 1;
+76
View File
@@ -0,0 +1,76 @@
// Build a read-only source index from already sealed AD evidence. No model,
// task, command, approval, retry, Git write or native-v2 envelope is produced.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
const repository = process.cwd(), evidence = path.resolve('.qa/live-validation-ad');
const output = path.resolve('.qa/coordination-readonly-2026-11-11-engineering-ad-r2');
assert.equal(fs.existsSync(output), false, 'Do not overwrite a source handoff.');
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
const parse = file => JSON.parse(fs.readFileSync(path.join(evidence, file), 'utf8'));
const originalManifestBytes = fs.readFileSync(path.join(evidence, 'FINAL-MANIFEST.json'));
const originalManifest = JSON.parse(originalManifestBytes), members = new Map(originalManifest.files.map(value => [value.path, value]));
const result = parse('node/result.json'), state = parse('node/session.json'), frozen = parse('current.json');
const ids = { projectId: state.projectId, sessionId: state.id, taskId: state.execution.task.id, runId: state.execution.run.id };
assert.equal(ids.taskId, result.taskId); assert.equal(ids.sessionId, result.sessionId);
assert.equal(state.execution.run.taskId, ids.taskId);
fs.mkdirSync(output); const sources = [], missing = [];
const addSealed = relative => {
const member = members.get(relative); assert.ok(member, 'Original manifest member: ' + relative);
const file = path.join(evidence, relative), bytes = fs.readFileSync(file);
assert.equal(bytes.length, member.bytes); assert.equal(sha(bytes), member.sha256);
sources.push({ path: file, bytes: bytes.length, sha256: sha(bytes), origin: 'original-ad-seal', originalRelative: relative });
};
for (const file of ['node/goal.txt', 'node/contract.json', 'node/source-before.json', 'node/result.json', 'node/session.json', 'node/export.json', 'node/attempts.json', 'node/git-log.txt', 'node/git-status.txt',
'node/baseline/test.txt', 'node/baseline/build.txt', 'node/baseline/result.json', 'node/independent-final/test.txt', 'node/independent-final/build.txt', 'node/independent-final/result.json',
'node/expense-project/package.json', 'node/expense-project/test/expense.test.mjs', 'node/expense-project/scripts/build.mjs', ...result.committedPaths.map(file => 'node/expense-project/' + file), ...result.captures.map(value => value.path)]) addSealed(file);
const plan = execFileSync('git', ['show', frozen.driverCommit + ':docs/LIVE-VALIDATION-AD-PLAN.md'], { windowsHide: true });
const planPath = path.join(output, 'original-predeclared-plan.md'); fs.writeFileSync(planPath, plan, { flag: 'wx' });
sources.push({ path: planPath, bytes: plan.length, sha256: sha(plan), origin: 'original-git-blob', commit: frozen.driverCommit, repositoryPath: 'docs/LIVE-VALIDATION-AD-PLAN.md' });
const artifactIds = new Set(state.execution.checks.flatMap(value => (value.results ?? []).map(row => row.logArtifactId)).filter(Boolean));
for (const value of state.execution.artifacts) if (/^delivery-report/.test(value.kind)) artifactIds.add(value.id);
const originalArtifacts = [];
for (const id of artifactIds) {
const artifact = state.execution.artifacts.find(value => value.id === id);
if (!artifact) { missing.push({ artifactId: id, reason: 'Check log descriptor absent from the sealed session artifact preview; independent-final logs remain in the read scope.' }); continue; }
for (const [key, expected] of Object.entries(ids)) assert.equal(artifact[key], expected, 'Artifact ownership: ' + key);
assert.match(artifact.relative, /^[a-f0-9]{2}\/[a-f0-9]{64}\.blob$/);
const originalPath = path.join(frozen.host.identity.dataRoot, 'artifacts', artifact.relative);
if (!fs.existsSync(originalPath)) { missing.push({ artifactId: id, kind: artifact.kind, reason: 'Original artifact file unavailable', expectedSha256: artifact.sha256 }); continue; }
const bytes = fs.readFileSync(originalPath); assert.equal(bytes.length, artifact.bytes); assert.equal(sha(bytes), artifact.sha256);
const target = path.join(output, id + (artifact.mediaType === 'application/json' ? '.json' : '.txt'));
fs.writeFileSync(target, bytes, { flag: 'wx' });
sources.push({ path: target, bytes: bytes.length, sha256: sha(bytes), origin: 'byte-exact-copy-of-original-owned-artifact', artifactId: id, kind: artifact.kind });
originalArtifacts.push({ ...artifact, sharedPath: target, originalPath, copiedBytesExact: true });
}
const report = originalArtifacts.find(value => value.kind === 'delivery-report');
const reportBody = report ? JSON.parse(fs.readFileSync(report.sharedPath, 'utf8')) : null;
const metadata = {
schemaVersion: 1, kind: 'workbench-readonly-engineering-source-index', generatedAt: new Date().toISOString(),
sourceName: 'agent-workbench/ad/node/' + ids.taskId + '/' + ids.runId, ids, originalVersion: '1.25.8', originalDriverCommit: frozen.driverCommit,
purpose: 'Read-only acceptance of an existing independent Node task. No incoming restricted delegation or new M4 task is represented.',
goalPath: path.join(evidence, 'node/goal.txt'), predeclaredPlanPath: planPath, originalTaskContract: state.execution.task.contract,
originalStructuredAcceptance: state.execution.task.acceptance,
originalReportArtifact: report ? { id: report.id, path: report.sharedPath, sha256: report.sha256, bytes: report.bytes, status: reportBody?.status ?? null } : null,
reportMetadata: originalArtifacts.filter(value => /^delivery-report/.test(value.kind)), checkLogMetadata: originalArtifacts.filter(value => artifactIds.has(value.id) && !/^delivery-report/.test(value.kind)),
engineering: { sourceRoot: result.sourceRoot, initialHead: result.initialHead, commit: result.commit, committedPaths: result.committedPaths, committedCurrent: result.committedCurrent,
actualIndependentChecks: result.final, ownerBytesAndIndexPreserved: result.allOriginalOwnerBytesAndIndexPreserved, supervisorContinuations: result.supervisionContinuations },
originalOutcome: { independentEngineeringSuccess: result.success, sessionStatus: state.status, taskStatus: state.execution.task.status, runStatus: state.execution.run.status,
terminalGitObserved: true, nativeDeliveryRows: state.execution.deliveries.length, internalDeliveryReportStatus: reportBody?.status ?? 'missing',
taskVerification: state.execution.task.verification, productStopReason: result.productStopReason, failure: result.failure, usage: result.usage,
missing, batchNetworkAndClaudeErrors: 'Preserved separately in QA-LIVE-VALIDATION-AD.md; the Node subset does not replace the whole-batch outcome.' },
readiness: { originalTaskAndRunAvailable: true, originalReportsCopied: !!report, nativeEngineeringEvidenceV2Envelope: false, restrictedDelegationReceiver: 'not_established_by_this_source',
originalKeyLookup: 'not_established_by_this_source', currentM4Engineering: 'not_dispatched_at_source_index_creation', allConsumerContractFieldsSatisfied: false,
gaps: ['No producer-native mnemo.engineering-evidence.v2 envelope or receipt is claimed.', 'The original task structured acceptance array is empty; the raw goal, predeclared plan and original tests carry its conditions.', 'Original terminal Git success and internal in_progress report remain distinct.'] },
readScope: { mode: 'read-only-existing-evidence', allowedFiles: sources, originalManifestPath: path.join(evidence, 'FINAL-MANIFEST.json'), originalManifestSha256: sha(originalManifestBytes),
excludes: ['Any credential value or credential store', 'Whole formal data directory or SQLite database', 'Other user projects or unrelated sessions', 'Replay, retry, execution, write or native-v2 certification'] },
actions: { realModelRequests: 0, engineeringDispatches: 0, gitWrites: 0, originalArtifactsChanged: false, keysReadOrResaved: false, messagesSentToOtherThreads: 0 },
};
const metadataPath = path.join(repository, 'docs/integration/WORKBENCH-AD-NODE-READONLY-SOURCE-2026-11-11.json');
fs.writeFileSync(metadataPath, JSON.stringify(metadata, null, 2) + '\n', { flag: 'wx' });
const metadataBytes = fs.readFileSync(metadataPath);
fs.writeFileSync(path.join(output, 'SOURCE-MANIFEST.json'), JSON.stringify({ schemaVersion: 1, metadataPath, metadataSha256: sha(metadataBytes), sources, originalArtifacts, missing }, null, 2) + '\n', { flag: 'wx' });
console.log(JSON.stringify({ metadataPath, output, sourceName: metadata.sourceName, reportStatus: metadata.originalOutcome.internalDeliveryReportStatus, terminalGit: result.commit, nativeV2Envelope: false, allowedFiles: sources.length, missing: missing.length, realModelRequests: 0 }));
+155
View File
@@ -0,0 +1,155 @@
// Fixed local responses verify the new driver and real tool execution only.
// They are not evidence of a provider's intelligence or autonomous delivery.
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { fixture, launchWorkbench, completion, call, assertToolPairs } from '../../tests/helpers.mjs';
import { root, save, client, configureFixture, captureSession, artifactBytes, expectedOutputFields, connectionDefinition } from './common.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
const output = path.resolve('.qa/m4-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
fs.mkdirSync(output); const f = fixture(t, { modernDefaults: true }); f.store.saveConfig({ approvalMode: 'full', mode: 'compatible' });
const malformedBytes = Buffer.from([100, 97, 116, 97, 58, 32, 0xff, 10, 10]);
const malformedArtifact = f.store.artifactStore.write(f.session, null, malformedBytes, { kind: 'model-stream-response', mediaType: 'text/event-stream' });
f.store.core.artifact.set(malformedArtifact.id, malformedArtifact); f.store.save(f.session); f.store.close();
let host, captureClient, result, engineeringRequests = 0; const requests = [], cases = [], sockets = new Set();
const files = {
'src/amount.mjs': `export function amountToCents(input) {
if (!['string','number'].includes(typeof input) || typeof input === 'number' && !Number.isFinite(input)) throw new TypeError('amount');
const match = String(input).trim().match(/^([+-]?)([0-9]+)(?:\\.([0-9]{1,2}))?$/);
if (!match) throw new TypeError('amount');
const n = (BigInt(match[2]) * 100n + BigInt((match[3] || '').padEnd(2,'0'))) * (match[1] === '-' ? -1n : 1n);
if (n > BigInt(Number.MAX_SAFE_INTEGER) || n < BigInt(Number.MIN_SAFE_INTEGER)) throw new TypeError('unsafe');
return n === 0n ? 0 : Number(n);
}\n`,
'src/summary.mjs': `import {amountToCents} from './amount.mjs';
export function summarize(rows) {
if (!Array.isArray(rows)) throw new TypeError('rows'); const map = new Map();
for (const row of rows) {
if (!row || typeof row.currency !== 'string') throw new TypeError('row');
const currency = row.currency.trim().toUpperCase(); if (!/^[A-Z]{3}$/.test(currency)) throw new TypeError('currency');
const item = map.get(currency) || {currency,cents:0,count:0};
const cents = item.cents + amountToCents(row.amount); if (!Number.isSafeInteger(cents)) throw new TypeError('unsafe');
map.set(currency,{currency,cents,count:item.count+1});
}
return [...map.values()].sort((a,b)=>a.currency.localeCompare(b.currency));
}\n`,
'src/report.mjs': `export function formatReport(rows) {return rows.map(({currency,cents,count})=>currency+' '+(cents<0?'-':'')+Math.floor(Math.abs(cents)/100)+'.'+String(Math.abs(cents)%100).padStart(2,'0')+' ('+count+')').join('\\n');}\n`,
};
const data = value => 'data: ' + (typeof value === 'string' ? value : JSON.stringify(value)) + '\n\n';
function streamBody(protocol, model, nonce, tool) {
if (protocol === 'chat-completions') {
const delta = tool ? { tool_calls: [{ index: 0, ...call('call_' + nonce, 'workbench_m4_echo_nonce', { nonce }) }] } : { content: 'OK ' + nonce };
return data({ model, choices: [{ index: 0, delta, finish_reason: null }] })
+ data({ model, choices: [{ index: 0, delta: {}, finish_reason: tool ? 'tool_calls' : 'stop' }] })
+ data({ model, choices: [], usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25, prompt_tokens_details: { cached_tokens: 4 } } }) + data('[DONE]');
}
if (protocol === 'responses') {
let sequence = 0; const native = (type, value) => data({ type, sequence_number: sequence++, ...value });
let bytes = native('response.created', { response: { id: 'resp_' + nonce, model } }), output;
if (tool) {
const item = { type: 'function_call', id: 'fc_' + nonce, call_id: 'call_' + nonce, name: 'workbench_m4_echo_nonce', arguments: '', status: 'in_progress' };
bytes += native('response.output_item.added', { output_index: 0, item });
bytes += native('response.function_call_arguments.delta', { item_id: item.id, delta: JSON.stringify({ nonce }) });
output = [{ ...item, status: 'completed', arguments: JSON.stringify({ nonce }) }];
bytes += native('response.function_call_arguments.done', { item_id: item.id, arguments: output[0].arguments });
} else {
bytes += native('response.output_text.delta', { delta: 'OK ' + nonce });
output = [{ type: 'message', id: 'msg_' + nonce, role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'OK ' + nonce }] }];
}
return bytes + native('response.completed', { response: { id: 'resp_' + nonce, model, status: 'completed', output, usage: { input_tokens: 20, output_tokens: 5, total_tokens: 25, input_tokens_details: { cached_tokens: 4 } } } });
}
const event = (type, value = {}) => 'event: ' + type + '\n' + data({ type, ...value });
const initial = { id: 'msg_' + nonce, type: 'message', role: 'assistant', model, content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 16, cache_read_input_tokens: 4, cache_creation_input_tokens: 0, output_tokens: 0 } };
const block = tool ? { type: 'tool_use', id: 'call_' + nonce, name: 'workbench_m4_echo_nonce', input: {} } : { type: 'text', text: '' };
const delta = tool ? { type: 'input_json_delta', partial_json: JSON.stringify({ nonce }) } : { type: 'text_delta', text: 'OK ' + nonce };
return event('message_start', { message: initial }) + event('content_block_start', { index: 0, content_block: block })
+ event('content_block_delta', { index: 0, delta }) + event('content_block_stop', { index: 0 })
+ event('message_delta', { delta: { stop_reason: tool ? 'tool_use' : 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } }) + event('message_stop');
}
const server = http.createServer(async (request, response) => {
try {
const chunks = []; for await (const chunk of request) chunks.push(chunk);
const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: request.url, body });
const protocol = request.url.endsWith('/messages') ? 'anthropic-messages' : request.url.endsWith('/responses') ? 'responses' : 'chat-completions';
if (protocol === 'anthropic-messages') assert.equal(body.max_tokens, 128000, 'Messages uses the exact official specification, not the old 2048.');
else assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
if (body.model === 'm4-error') { response.writeHead(400, { 'Content-Type': 'application/json', 'x-request-id': 'owned-m4-error' }); response.end('{"error":{"message":"owned-m4-original-error","type":"invalid_request_error"}}'); return; }
if (body.model === 'm4-unknown') { response.writeHead(200, { 'Content-Type': 'text/event-stream' }); response.write(data({ model: body.model, choices: [{ index: 0, delta: { content: 'incomplete' }, finish_reason: null }] })); return; }
if (body.model === 'm4-node') {
let answer; engineeringRequests++;
if (engineeringRequests === 1) answer = completion(Object.entries(files).map(([name, content], index) => call('owned-file-' + index, 'propose_file', { path: name, content, reason: 'Fixed local fixture only.' })));
else if (engineeringRequests === 2) answer = completion([call('owned-plan', 'plan_checks', { names: ['test', 'build'] })]);
else if (engineeringRequests === 3) {
const sessions = await host.api('/sessions'), active = sessions.find(value => value.status === 'running'); assert.ok(active);
const state = await host.api('/sessions/' + active.id), plan = state.execution.plans.at(-1); assert.ok(plan);
answer = completion([call('owned-check', 'run_checks', { plan_id: plan.id })]);
} else if (engineeringRequests === 4) {
answer = completion([call('owned-terminal-git', 'run_command', { command: "git -c core.hooksPath=NUL commit --only --no-gpg-sign -m 'Fixed M4 local driver fixture' -- src/amount.mjs src/summary.mjs src/report.mjs\nif ($LASTEXITCODE -ne 0) { throw 'fixture commit failed' }", reason: 'Verify actual local Git with original owner staging preserved.' })]);
} else answer = completion([], 'Fixed local fixture finished.');
response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ ...answer, model: body.model })); return;
}
const nonce = JSON.stringify(body.messages ?? body.input).match(/(?:nonce |OK )([a-f0-9-]{36})/)?.[1]; assert.ok(nonce);
const tool = body.tools?.length > 0;
if (body.stream) { response.writeHead(200, { 'Content-Type': 'text/event-stream' }); response.end(streamBody(protocol, body.model, nonce, tool)); }
else { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ ...completion([], 'OK ' + nonce), model: body.model })); }
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
});
server.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
disposers.push(() => new Promise(resolve => { for (const socket of sockets) socket.destroy(); server.close(resolve); }));
try {
await inspectorClosed();
const observation = JSON.parse(fs.readFileSync('.qa/m4-live-baseline-2026-11-11/observation.json', 'utf8'));
host = await launchWorkbench(t, f.data, {}, { appRoot: observation.boot.runtimeRoot }); configureFixture({ root: output, origin: host.url });
captureClient = await client(); disposers.push(() => captureClient.close());
const downloaded = await artifactBytes(captureClient.api, f.session.id, malformedArtifact); assert.deepEqual(downloaded, malformedBytes);
save('byte-preservation/original.bin', downloaded); save('byte-preservation/metadata.json', malformedArtifact);
cases.push({ prefix: 'byte-preservation', expected: 'exact-invalid-utf8-bytes', requests: 0, passed: true });
const health = await host.api('/host/status'), beforeSelection = (await host.api('/model-center')).selected;
const runProbe = async (connection, scope, prefix, expected = 'success', observationMs = 30000) => {
const session = await host.api('/sessions', { projectId: f.project.id }), nonce = randomUUID();
const input = { pid: health.identity.pid, port: Number(new URL(host.url).port), runtimeRoot: host.bootstrap.runtimeRoot, projectId: f.project.id, sessionId: session.id,
connection, nonce, scope, expectedOutput: expectedOutputFields(connection), observationMs };
save(prefix + '/input.json', input); const invoked = await invokeProbe(input); save(prefix + '/invocation.json', invoked);
const captured = await captureSession(captureClient.api, session.id, prefix), attempt = captured.rows[0];
assert.equal(captured.total, 1); assert.equal(invoked.result.actualToolsExecuted, 0); assert.equal(invoked.result.wireStarts, 1);
assertToolPairs(captured.exported.session.providerMessages);
if (expected === 'success') { assert.equal(invoked.result.contentPassed, true, JSON.stringify(invoked.result)); assert.equal(attempt.status, 'completed'); assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true); assert.equal(attempt.usageStatus, 'reported'); }
else { assert.equal(attempt.status, expected === 'unknown' ? 'interrupted' : 'failed'); assert.equal(attempt.usageStatus, 'unknown'); assert.equal(invoked.result.contentPassed, false); }
if (expected === 'error') assert.ok(captured.captures.some(value => value.kind === 'response' && fs.readFileSync(path.join(root, value.path), 'utf8').includes('owned-m4-original-error')));
assert.deepEqual((await host.api('/model-center')).selected, beforeSelection);
assert.deepEqual(connectionDefinition((await host.api('/model-center')).connections.find(value => value.id === connection.id)), connectionDefinition(connection));
cases.push({ prefix, protocol: connection.protocol, scope, expected, requests: 1, passed: true });
};
const baseUrl = 'http://127.0.0.1:' + server.address().port + '/v1';
for (const [protocol, model] of [['chat-completions', 'm4-chat'], ['responses', 'm4-responses'], ['anthropic-messages', 'claude-opus-5-5']]) {
const connection = await host.api('/model-center/connections', { name: 'Owned M4 ' + protocol, model, protocol, baseUrl, streaming: true, apiKey: 'owned-m4-fixture-secret' });
for (const scope of ['text', 'tools']) await runProbe(connection, scope, 'preflight/' + protocol + '-' + scope);
}
const json = await host.api('/model-center/connections', { name: 'Owned M4 JSON', model: 'm4-json', protocol: 'chat-completions', baseUrl, streaming: false, apiKey: 'owned-m4-fixture-secret' });
await runProbe(json, 'text', 'preflight/json'); await runProbe(json, 'streaming', 'preflight/streaming-override');
for (const [model, expected] of [['m4-error', 'error'], ['m4-unknown', 'unknown']]) {
const connection = await host.api('/model-center/connections', { name: 'Owned ' + model, model, protocol: 'chat-completions', baseUrl, streaming: true, apiKey: 'owned-m4-fixture-secret' });
await runProbe(connection, 'text', 'preflight/' + model, expected, expected === 'unknown' ? 750 : 30000);
}
const connection = await host.api('/model-center/connections', { name: 'Owned M4 Node', model: 'm4-node', protocol: 'chat-completions', baseUrl, streaming: false, apiKey: 'owned-m4-fixture-secret' });
const c = await client();
try {
const center = await c.api('/model-center');
save('frozen.json', { fixture: true, host: await c.api('/host/status'), config: c.boot.config, center, connections: [connection], engineeringPriority: [connection.id] });
} finally { await c.close(); }
save('compatibility/result.json', { fixture: true, results: [{ connectionId: connection.id, engineeringAllowed: true }] });
const argv = process.argv[2]; process.argv[2] = 'node'; await import('./run.mjs'); process.argv[2] = argv;
const node = JSON.parse(fs.readFileSync(path.join(root, 'node/result.json'), 'utf8')); assert.equal(node.success, true, JSON.stringify({ failure: node.failure, final: node.final, cleanupErrors: node.cleanupErrors, requests: node.requests }));
assert.equal(node.requests, 5); assert.equal(node.final.test.stats.pass, 9); assert.equal(node.ownerBytesAndIndexPreserved, true); assert.equal(node.identitiesPassed, true);
assert.equal(requests.length, 15); result = { success: true, cases, fixtureRequests: requests.length, engineeringRequests, realModelRequests: 0,
installedAppRoot: observation.boot.runtimeRoot, version: host.bootstrap.version, driverRunsOriginalNodeChecksAndGit: true, fixedImplementationOnlyInFixture: true };
} catch (error) { result = { success: false, error: error.stack, cases, fixtureRequests: requests.length, engineeringRequests, realModelRequests: 0 }; process.exitCode = 1; }
finally {
await inspectorClosed(); for (const dispose of disposers.reverse()) await dispose();
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
}
console.log(JSON.stringify({ output, ...result }));