Files
Agent-Workbench/scripts/m4-live/verify-driver.mjs
T
2026-11-11 22:58:21 +08:00

156 lines
15 KiB
JavaScript

// Fixed local responses verify the new driver and real tool execution only.
// They are not evidence of a provider's intelligence or autonomous delivery.
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { fixture, launchWorkbench, completion, call, assertToolPairs } from '../../tests/helpers.mjs';
import { root, save, client, configureFixture, captureSession, artifactBytes, expectedOutputFields, connectionDefinition } from './common.mjs';
import { invokeProbe, inspectorClosed } from './inspector.mjs';
const output = path.resolve('.qa/m4-driver-' + randomUUID()), disposers = [], t = { after: fn => disposers.push(fn) };
fs.mkdirSync(output); const f = fixture(t, { modernDefaults: true }); f.store.saveConfig({ approvalMode: 'full', mode: 'compatible' });
const malformedBytes = Buffer.from([100, 97, 116, 97, 58, 32, 0xff, 10, 10]);
const malformedArtifact = f.store.artifactStore.write(f.session, null, malformedBytes, { kind: 'model-stream-response', mediaType: 'text/event-stream' });
f.store.core.artifact.set(malformedArtifact.id, malformedArtifact); f.store.save(f.session); f.store.close();
let host, captureClient, result, engineeringRequests = 0; const requests = [], cases = [], sockets = new Set();
const files = {
'src/amount.mjs': `export function amountToCents(input) {
if (!['string','number'].includes(typeof input) || typeof input === 'number' && !Number.isFinite(input)) throw new TypeError('amount');
const match = String(input).trim().match(/^([+-]?)([0-9]+)(?:\\.([0-9]{1,2}))?$/);
if (!match) throw new TypeError('amount');
const n = (BigInt(match[2]) * 100n + BigInt((match[3] || '').padEnd(2,'0'))) * (match[1] === '-' ? -1n : 1n);
if (n > BigInt(Number.MAX_SAFE_INTEGER) || n < BigInt(Number.MIN_SAFE_INTEGER)) throw new TypeError('unsafe');
return n === 0n ? 0 : Number(n);
}\n`,
'src/summary.mjs': `import {amountToCents} from './amount.mjs';
export function summarize(rows) {
if (!Array.isArray(rows)) throw new TypeError('rows'); const map = new Map();
for (const row of rows) {
if (!row || typeof row.currency !== 'string') throw new TypeError('row');
const currency = row.currency.trim().toUpperCase(); if (!/^[A-Z]{3}$/.test(currency)) throw new TypeError('currency');
const item = map.get(currency) || {currency,cents:0,count:0};
const cents = item.cents + amountToCents(row.amount); if (!Number.isSafeInteger(cents)) throw new TypeError('unsafe');
map.set(currency,{currency,cents,count:item.count+1});
}
return [...map.values()].sort((a,b)=>a.currency.localeCompare(b.currency));
}\n`,
'src/report.mjs': `export function formatReport(rows) {return rows.map(({currency,cents,count})=>currency+' '+(cents<0?'-':'')+Math.floor(Math.abs(cents)/100)+'.'+String(Math.abs(cents)%100).padStart(2,'0')+' ('+count+')').join('\\n');}\n`,
};
const data = value => 'data: ' + (typeof value === 'string' ? value : JSON.stringify(value)) + '\n\n';
function streamBody(protocol, model, nonce, tool) {
if (protocol === 'chat-completions') {
const delta = tool ? { tool_calls: [{ index: 0, ...call('call_' + nonce, 'workbench_m4_echo_nonce', { nonce }) }] } : { content: 'OK ' + nonce };
return data({ model, choices: [{ index: 0, delta, finish_reason: null }] })
+ data({ model, choices: [{ index: 0, delta: {}, finish_reason: tool ? 'tool_calls' : 'stop' }] })
+ data({ model, choices: [], usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25, prompt_tokens_details: { cached_tokens: 4 } } }) + data('[DONE]');
}
if (protocol === 'responses') {
let sequence = 0; const native = (type, value) => data({ type, sequence_number: sequence++, ...value });
let bytes = native('response.created', { response: { id: 'resp_' + nonce, model } }), output;
if (tool) {
const item = { type: 'function_call', id: 'fc_' + nonce, call_id: 'call_' + nonce, name: 'workbench_m4_echo_nonce', arguments: '', status: 'in_progress' };
bytes += native('response.output_item.added', { output_index: 0, item });
bytes += native('response.function_call_arguments.delta', { item_id: item.id, delta: JSON.stringify({ nonce }) });
output = [{ ...item, status: 'completed', arguments: JSON.stringify({ nonce }) }];
bytes += native('response.function_call_arguments.done', { item_id: item.id, arguments: output[0].arguments });
} else {
bytes += native('response.output_text.delta', { delta: 'OK ' + nonce });
output = [{ type: 'message', id: 'msg_' + nonce, role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'OK ' + nonce }] }];
}
return bytes + native('response.completed', { response: { id: 'resp_' + nonce, model, status: 'completed', output, usage: { input_tokens: 20, output_tokens: 5, total_tokens: 25, input_tokens_details: { cached_tokens: 4 } } } });
}
const event = (type, value = {}) => 'event: ' + type + '\n' + data({ type, ...value });
const initial = { id: 'msg_' + nonce, type: 'message', role: 'assistant', model, content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 16, cache_read_input_tokens: 4, cache_creation_input_tokens: 0, output_tokens: 0 } };
const block = tool ? { type: 'tool_use', id: 'call_' + nonce, name: 'workbench_m4_echo_nonce', input: {} } : { type: 'text', text: '' };
const delta = tool ? { type: 'input_json_delta', partial_json: JSON.stringify({ nonce }) } : { type: 'text_delta', text: 'OK ' + nonce };
return event('message_start', { message: initial }) + event('content_block_start', { index: 0, content_block: block })
+ event('content_block_delta', { index: 0, delta }) + event('content_block_stop', { index: 0 })
+ event('message_delta', { delta: { stop_reason: tool ? 'tool_use' : 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } }) + event('message_stop');
}
const server = http.createServer(async (request, response) => {
try {
const chunks = []; for await (const chunk of request) chunks.push(chunk);
const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: request.url, body });
const protocol = request.url.endsWith('/messages') ? 'anthropic-messages' : request.url.endsWith('/responses') ? 'responses' : 'chat-completions';
if (protocol === 'anthropic-messages') assert.equal(body.max_tokens, 128000, 'Messages uses the exact official specification, not the old 2048.');
else assert.ok(['max_tokens', 'max_output_tokens', 'max_completion_tokens'].every(key => !Object.hasOwn(body, key)));
if (body.model === 'm4-error') { response.writeHead(400, { 'Content-Type': 'application/json', 'x-request-id': 'owned-m4-error' }); response.end('{"error":{"message":"owned-m4-original-error","type":"invalid_request_error"}}'); return; }
if (body.model === 'm4-unknown') { response.writeHead(200, { 'Content-Type': 'text/event-stream' }); response.write(data({ model: body.model, choices: [{ index: 0, delta: { content: 'incomplete' }, finish_reason: null }] })); return; }
if (body.model === 'm4-node') {
let answer; engineeringRequests++;
if (engineeringRequests === 1) answer = completion(Object.entries(files).map(([name, content], index) => call('owned-file-' + index, 'propose_file', { path: name, content, reason: 'Fixed local fixture only.' })));
else if (engineeringRequests === 2) answer = completion([call('owned-plan', 'plan_checks', { names: ['test', 'build'] })]);
else if (engineeringRequests === 3) {
const sessions = await host.api('/sessions'), active = sessions.find(value => value.status === 'running'); assert.ok(active);
const state = await host.api('/sessions/' + active.id), plan = state.execution.plans.at(-1); assert.ok(plan);
answer = completion([call('owned-check', 'run_checks', { plan_id: plan.id })]);
} else if (engineeringRequests === 4) {
answer = completion([call('owned-terminal-git', 'run_command', { command: "git -c core.hooksPath=NUL commit --only --no-gpg-sign -m 'Fixed M4 local driver fixture' -- src/amount.mjs src/summary.mjs src/report.mjs\nif ($LASTEXITCODE -ne 0) { throw 'fixture commit failed' }", reason: 'Verify actual local Git with original owner staging preserved.' })]);
} else answer = completion([], 'Fixed local fixture finished.');
response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ ...answer, model: body.model })); return;
}
const nonce = JSON.stringify(body.messages ?? body.input).match(/(?:nonce |OK )([a-f0-9-]{36})/)?.[1]; assert.ok(nonce);
const tool = body.tools?.length > 0;
if (body.stream) { response.writeHead(200, { 'Content-Type': 'text/event-stream' }); response.end(streamBody(protocol, body.model, nonce, tool)); }
else { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ ...completion([], 'OK ' + nonce), model: body.model })); }
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
});
server.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
disposers.push(() => new Promise(resolve => { for (const socket of sockets) socket.destroy(); server.close(resolve); }));
try {
await inspectorClosed();
const observation = JSON.parse(fs.readFileSync('.qa/m4-live-baseline-2026-11-11/observation.json', 'utf8'));
host = await launchWorkbench(t, f.data, {}, { appRoot: observation.boot.runtimeRoot }); configureFixture({ root: output, origin: host.url });
captureClient = await client(); disposers.push(() => captureClient.close());
const downloaded = await artifactBytes(captureClient.api, f.session.id, malformedArtifact); assert.deepEqual(downloaded, malformedBytes);
save('byte-preservation/original.bin', downloaded); save('byte-preservation/metadata.json', malformedArtifact);
cases.push({ prefix: 'byte-preservation', expected: 'exact-invalid-utf8-bytes', requests: 0, passed: true });
const health = await host.api('/host/status'), beforeSelection = (await host.api('/model-center')).selected;
const runProbe = async (connection, scope, prefix, expected = 'success', observationMs = 30000) => {
const session = await host.api('/sessions', { projectId: f.project.id }), nonce = randomUUID();
const input = { pid: health.identity.pid, port: Number(new URL(host.url).port), runtimeRoot: host.bootstrap.runtimeRoot, projectId: f.project.id, sessionId: session.id,
connection, nonce, scope, expectedOutput: expectedOutputFields(connection), observationMs };
save(prefix + '/input.json', input); const invoked = await invokeProbe(input); save(prefix + '/invocation.json', invoked);
const captured = await captureSession(captureClient.api, session.id, prefix), attempt = captured.rows[0];
assert.equal(captured.total, 1); assert.equal(invoked.result.actualToolsExecuted, 0); assert.equal(invoked.result.wireStarts, 1);
assertToolPairs(captured.exported.session.providerMessages);
if (expected === 'success') { assert.equal(invoked.result.contentPassed, true, JSON.stringify(invoked.result)); assert.equal(attempt.status, 'completed'); assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true); assert.equal(attempt.usageStatus, 'reported'); }
else { assert.equal(attempt.status, expected === 'unknown' ? 'interrupted' : 'failed'); assert.equal(attempt.usageStatus, 'unknown'); assert.equal(invoked.result.contentPassed, false); }
if (expected === 'error') assert.ok(captured.captures.some(value => value.kind === 'response' && fs.readFileSync(path.join(root, value.path), 'utf8').includes('owned-m4-original-error')));
assert.deepEqual((await host.api('/model-center')).selected, beforeSelection);
assert.deepEqual(connectionDefinition((await host.api('/model-center')).connections.find(value => value.id === connection.id)), connectionDefinition(connection));
cases.push({ prefix, protocol: connection.protocol, scope, expected, requests: 1, passed: true });
};
const baseUrl = 'http://127.0.0.1:' + server.address().port + '/v1';
for (const [protocol, model] of [['chat-completions', 'm4-chat'], ['responses', 'm4-responses'], ['anthropic-messages', 'claude-opus-5-5']]) {
const connection = await host.api('/model-center/connections', { name: 'Owned M4 ' + protocol, model, protocol, baseUrl, streaming: true, apiKey: 'owned-m4-fixture-secret' });
for (const scope of ['text', 'tools']) await runProbe(connection, scope, 'preflight/' + protocol + '-' + scope);
}
const json = await host.api('/model-center/connections', { name: 'Owned M4 JSON', model: 'm4-json', protocol: 'chat-completions', baseUrl, streaming: false, apiKey: 'owned-m4-fixture-secret' });
await runProbe(json, 'text', 'preflight/json'); await runProbe(json, 'streaming', 'preflight/streaming-override');
for (const [model, expected] of [['m4-error', 'error'], ['m4-unknown', 'unknown']]) {
const connection = await host.api('/model-center/connections', { name: 'Owned ' + model, model, protocol: 'chat-completions', baseUrl, streaming: true, apiKey: 'owned-m4-fixture-secret' });
await runProbe(connection, 'text', 'preflight/' + model, expected, expected === 'unknown' ? 750 : 30000);
}
const connection = await host.api('/model-center/connections', { name: 'Owned M4 Node', model: 'm4-node', protocol: 'chat-completions', baseUrl, streaming: false, apiKey: 'owned-m4-fixture-secret' });
const c = await client();
try {
const center = await c.api('/model-center');
save('frozen.json', { fixture: true, host: await c.api('/host/status'), config: c.boot.config, center, connections: [connection], engineeringPriority: [connection.id] });
} finally { await c.close(); }
save('compatibility/result.json', { fixture: true, results: [{ connectionId: connection.id, engineeringAllowed: true }] });
const argv = process.argv[2]; process.argv[2] = 'node'; await import('./run.mjs'); process.argv[2] = argv;
const node = JSON.parse(fs.readFileSync(path.join(root, 'node/result.json'), 'utf8')); assert.equal(node.success, true, JSON.stringify({ failure: node.failure, final: node.final, cleanupErrors: node.cleanupErrors, requests: node.requests }));
assert.equal(node.requests, 5); assert.equal(node.final.test.stats.pass, 9); assert.equal(node.ownerBytesAndIndexPreserved, true); assert.equal(node.identitiesPassed, true);
assert.equal(requests.length, 15); result = { success: true, cases, fixtureRequests: requests.length, engineeringRequests, realModelRequests: 0,
installedAppRoot: observation.boot.runtimeRoot, version: host.bootstrap.version, driverRunsOriginalNodeChecksAndGit: true, fixedImplementationOnlyInFixture: true };
} catch (error) { result = { success: false, error: error.stack, cases, fixtureRequests: requests.length, engineeringRequests, realModelRequests: 0 }; process.exitCode = 1; }
finally {
await inspectorClosed(); for (const dispose of disposers.reverse()) await dispose();
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
}
console.log(JSON.stringify({ output, ...result }));