Files
Agent-Workbench/tests/diagnostics.test.mjs
T

213 lines
16 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { Provider, toolDefinitions, projectContext } from '../src/server/provider.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Store } from '../src/server/store.mjs';
import { sessionSnapshot, sessionContext } from '../src/server/diagnostics.mjs';
import { fixture, compatibleFixture, call, completion, until, assertToolPairs } from './helpers.mjs';
const policy = { attempts: 3, delays: [1, 2], timeoutMs: 1000, totalTimeoutMs: 3000 };
const response = result => new Response(JSON.stringify(result), { headers: { 'Content-Type': 'application/json' } });
const disconnected = () => new TypeError('fetch failed', { cause: Object.assign(new Error('connection reset'), { code: 'ECONNRESET' }) });
const refused = () => new TypeError('fetch failed', { cause: Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' }) });
const configure = f => f.store.saveConfig({ mode: 'compatible', baseUrl: 'http://127.0.0.1:1/v1', model: 'fixture-agent' });
test('routine engineering facts remove duplicate success excerpts while complete diagnostics and errors remain available', t => {
const f=fixture(t);
f.store.event(f.session,{kind:'tool',tool:'read_file',phase:'completed',input:{path:'src/a.mjs'},output:'successful source excerpt'});
f.store.event(f.session,{kind:'tool',tool:'read_file',phase:'failed',input:{path:'src/missing.mjs'},output:'original missing file error'});
f.store.event(f.session,{kind:'error',phase:'failed',label:'Original error',output:'fetch failed'});
const full=sessionSnapshot(f.session,f.project,f.store.config),context=sessionContext(f.session,f.project,f.store.config,String,{executionFactsIncluded:true});
assert.ok(full.recentTools.some(tool=>tool.outputExcerpt==='successful source excerpt'));
assert.ok(!context.includes('successful source excerpt'));assert.ok(context.includes('original missing file error'));
assert.ok(context.includes('model_connection'));assert.ok(context.includes('inspect_session'));assert.ok(context.includes('fetch failed'));
assert.deepEqual(sessionSnapshot(f.session,f.project,f.store.config),full);
});
test('a question about the previous workbench error receives local error context and current change states', async t => {
// This test requires the original inspection result in the next model turn.
// Exercise the new complete-history default; finite compaction is covered
// independently and may archive this growing runtime diagnostic under 16K.
const f = fixture(t, { modernDefaults: true });
f.session.changes.push({ id: 'impl', path: 'src/catalog.js', status: 'applied' }, { id: 'tests', path: 'tests/catalog.test.mjs', status: 'applied' });
f.store.message(f.session, 'system', 'fetch failed');
f.store.event(f.session, { kind: 'error', label: '任务需要处理', phase: 'failed', output: 'fetch failed' });
const provider = new Provider();
let round = 0;
provider.complete = async (_config, messages) => {
const sentContext = messages.map(message => typeof message.content === 'string' ? message.content : '').join('\n');
assert.match(sentContext, /model_connection/);
assert.match(sentContext, /inspect_session/);
assert.match(sentContext, /"status":"applied"/);
if (++round === 1) return { message: completion([call('inspect', 'inspect_session', {})]).choices[0].message };
const result = JSON.parse(messages.findLast(message => message.role === 'tool').content);
assert.equal(result.lastFailure.source, 'model_connection');
assert.equal(result.lastFailure.stage, 'model_request');
assert.equal(result.lastFailure.historical, true);
assert.deepEqual(result.changes.map(change => change.status), ['applied', 'applied']);
assert.ok(result.environment.shell); assert.equal(result.environment.encoding, 'UTF-8');
return { message: { role: 'assistant', content: 'The recorded failure is a model request failure.' } };
};
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, '刚才工作台的 fetch failed 怎么回事?'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'completed'); assert.equal(round, 2);
assert.equal(f.session.events.filter(event => event.tool === 'search_files').length, 0);
assertToolPairs(f.session.providerMessages);
assert.ok(toolDefinitions.some(tool => tool.function.name === 'inspect_session'));
});
test('an unknown model request stops, and explicit resume never replays the approved command', async t => {
const f = fixture(t); configure(f);
fs.writeFileSync(path.join(f.projectRoot, 'counter.mjs'), "import fs from 'node:fs'; const previous = fs.existsSync('count.txt') ? Number(fs.readFileSync('count.txt','utf8')) : 0; fs.writeFileSync('count.txt', String(previous + 1));\n");
let requests = 0;
const provider = new Provider({ retryPolicy: policy, fetch: async (_url, input) => {
const messages = JSON.parse(input.body).messages; assertToolPairs(messages);
if (++requests === 1) return response(completion([call('command', 'run_command', { command: 'node counter.mjs', reason: 'Controlled one-time command' })]));
if (requests === 2) throw disconnected();
return response(completion([], 'Done'));
} });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Run the controlled counter.'); const done = runtime.active.get(f.session.id).promise;
await until(() => f.session.status === 'awaiting_approval'); runtime.decide(f.session.id, f.session.approvals[0].id, true);
await done;
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'count.txt'), 'utf8'), '1');
assert.equal(requests, 2); assert.equal(f.session.status, 'failed');
assert.equal(f.session.failure.outcome, 'unknown');
assert.equal(f.session.failure.retryable, false);
assert.equal(f.session.failure.failoverAllowed, false);
assert.equal(f.session.failure.attempts, 1);
const failedEvent = f.session.events.filter(event => event.kind === 'model').at(-1);
assert.equal(failedEvent.phase, 'failed'); assert.equal(failedEvent.diagnostic.outcome, 'unknown');
const userTurns = f.session.providerMessages.filter(message => message.role === 'user').length;
runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
assert.equal(requests, 3); assert.equal(f.session.status, 'completed');
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'count.txt'), 'utf8'), '1');
assert.equal(f.session.providerMessages.filter(message => message.role === 'user').length, userTurns);
assert.equal(f.session.approvals.length, 1);
assert.equal(f.session.events.filter(event => event.tool === 'run_command').length, 1);
assert.equal(f.session.events.filter(event => event.kind === 'model').at(-1).attempts, 1);
assert.ok(f.session.events.includes(failedEvent));
assertToolPairs(f.session.providerMessages);
});
test('exhausted refused connections preserve tool results, and explicit resume adds no new user turn or replay', async t => {
const f = fixture(t); configure(f);
let requests = 0, fail = true;
const provider = new Provider({ retryPolicy: { ...policy, attempts: 2 }, fetch: async (_url, input) => {
const messages = JSON.parse(input.body).messages; assertToolPairs(messages);
if (++requests === 1) return response(completion([call('read', 'read_file', { path: 'README.md' })]));
if (fail) throw refused();
assert.equal(messages.at(-1).role, 'tool'); return response(completion([], 'Recovered'));
} });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Read the fixture.'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'failed');
assert.equal(f.session.failure.source, 'model_connection'); assert.equal(f.session.failure.code, 'ECONNREFUSED');
assert.equal(f.session.failure.attempts, 2);
const userTurns = f.session.providerMessages.filter(message => message.role === 'user').length;
fail = false; runtime.resume(f.session.id); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'completed');
assert.equal(f.session.providerMessages.filter(message => message.role === 'user').length, userTurns);
assert.equal(f.session.events.filter(event => event.tool === 'read_file').length, 1);
assertToolPairs(f.session.providerMessages);
});
test('stopping during model backoff cancels further requests and clears the running request state', async t => {
const f = fixture(t); configure(f); let requests = 0;
const provider = new Provider({ retryPolicy: { ...policy, delays: [500, 500] }, fetch: async () => { requests++; throw refused(); } });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'test'); await until(() => f.session.events.some(event => event.phase === 'retrying'));
await runtime.stop(f.session.id);
assert.equal(requests, 1); assert.equal(f.session.status, 'paused');
assert.equal(f.session.events.find(event => event.kind === 'model').phase, 'interrupted');
assert.equal(provider.lastRequest.status, 'interrupted');
assert.equal(provider.lastRequest.attempts, 1);
assert.equal(Object.hasOwn(provider.lastRequest, 'kind'), false);
});
test('HTTP authentication errors are classified, redacted and never automatically retried', async t => {
const model = await compatibleFixture(t, () => ({ status: 401, body: { error: { message: 'Rejected fixture-private-key' } } }));
const provider = new Provider({ retryPolicy: policy }); provider.setKey('fixture-private-key');
await assert.rejects(provider.complete({ mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture' }, [], null), error => {
assert.equal(error.diagnostic.kind, 'authentication'); assert.equal(error.diagnostic.httpStatus, 401);
assert.equal(error.diagnostic.attempts, 1); assert.equal(error.diagnostic.retryable, false);
assert.ok(error.message.includes('[已隐藏]')); assert.ok(!JSON.stringify(error.diagnostic).includes('fixture-private-key')); return true;
});
assert.equal(model.requests.length, 1);
});
test('temporary HTTP service errors retry and recover with recorded attempt counts', async t => {
const model = await compatibleFixture(t, (_url, _input, requests) => requests.length === 1
? { status: 503, body: { error: { message: 'Temporary fixture outage' } } } : completion([], 'Recovered'));
const provider = new Provider({ retryPolicy: policy }), progress = [];
const result = await provider.complete({ mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture' }, [], null, undefined, value => progress.push(value));
assert.equal(result.message.content, 'Recovered'); assert.equal(model.requests.length, 2);
assert.ok(progress.some(item => item.phase === 'retrying')); assert.equal(provider.lastRequest.attempts, 2);
});
test('invalid and oversized protocol responses fail once with a useful response-format diagnosis', async () => {
for (const content of ['not JSON', 'x'.repeat(4 * 1024 * 1024 + 1)]) {
let requests = 0;
const provider = new Provider({ retryPolicy: policy, fetch: async () => { requests++; return new Response(content); } });
await assert.rejects(provider.complete({ mode: 'compatible', baseUrl: 'http://127.0.0.1:1/v1', model: 'fixture' }, [], null), error => error.diagnostic.kind === 'protocol' && !error.diagnostic.retryable);
assert.equal(requests, 1);
}
});
test('HTTP rate limits expose their source and obey the service retry interval before recovery', async t => {
const timestamps = [];
const model = await compatibleFixture(t, () => {
timestamps.push(Date.now());
return timestamps.length === 1 ? { status: 429, headers: { 'Retry-After': '1' }, body: { error: { message: 'Controlled rate limit' } } }
: completion([], 'Limit recovered');
});
const provider = new Provider({ retryPolicy: policy }), progress = [];
const result = await provider.complete({ mode: 'compatible', baseUrl: model.baseUrl, model: 'fixture' }, [], null, undefined, value => progress.push(value));
assert.equal(result.message.content, 'Limit recovered'); assert.equal(model.requests.length, 2);
assert.ok(timestamps[1] - timestamps[0] >= 950, 'The service retry interval must be respected.');
const retry = progress.find(value => value.phase === 'retrying');
assert.equal(retry.diagnostic.kind, 'rate_limit'); assert.equal(retry.diagnostic.httpStatus, 429);
assert.equal(retry.waitMs, 1000); assert.equal(provider.lastRequest.status, 'completed');
});
test('Windows commands preserve Chinese through a PowerShell pipeline into Node', { skip: process.platform !== 'win32' }, async t => {
const f = fixture(t), runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
const command = "@'\nconsole.log(JSON.stringify({ message: '工具·中文完成', value: 2 + 3 }));\n'@ | node --input-type=module\nexit $LASTEXITCODE";
const result = await runtime.executeCommand(command, f.projectRoot, new AbortController().signal, () => {});
assert.equal(result.status, 'completed', result.stderr); assert.equal(result.exit_code, 0);
assert.deepEqual(JSON.parse(result.stdout), { message: '工具·中文完成', value: 5 });
assert.ok(!result.stdout.includes('�')); assert.match(projectContext(f.project), /PowerShell 5\.1/);
});
test('Windows shell syntax errors stay readable and native command exit codes are retained', { skip: process.platform !== 'win32' }, async t => {
const f = fixture(t), runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
const invalid = await runtime.executeCommand("Write-Output '未执行的中文' && Write-Output 'ok'", f.projectRoot, new AbortController().signal, () => {});
assert.equal(invalid.status, 'failed'); assert.equal(invalid.stdout, ''); assert.match(invalid.stderr, /&&/);
assert.ok(!invalid.stderr.includes('�'));
const failed = await runtime.executeCommand("node -e 'process.exit(7)'", f.projectRoot, new AbortController().signal, () => {});
assert.equal(failed.status, 'failed'); assert.equal(failed.exit_code, 7);
});
test('review and completed states do not turn a generic continue click into a new model request', async t => {
const f = fixture(t), provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let requests = 0; provider.complete = async () => { requests++; throw new Error('Should not be requested'); };
for (const status of ['needs_review', 'completed']) {
f.session.status = status; assert.throws(() => runtime.resume(f.session.id), { status: 409 });
}
assert.equal(requests, 0); assert.equal(f.session.providerMessages.length, 0);
});
test('old session data receives a truthful historical connection diagnosis without rewriting history', async t => {
const f = fixture(t);
f.store.message(f.session, 'system', 'fetch failed'); f.store.event(f.session, { kind: 'error', phase: 'failed', output: 'fetch failed' });
const originalMessages = structuredClone(f.session.messages), reloaded = new Store(f.data);
f.onCleanup(() => reloaded.close());
const session = reloaded.session(f.session.id), visible = reloaded.publicSession(session);
assert.equal(visible.lastFailure.source, 'model_connection'); assert.equal(visible.lastFailure.historical, true);
assert.deepEqual(session.messages, originalMessages); assert.equal(Object.hasOwn(visible, 'providerMessages'), false);
const snapshot = sessionSnapshot(session, f.project, f.store.config);
assert.match(snapshot.lastFailure.detail, /没有保存底层/);
});