Files
Agent-Workbench/tests/engine.test.mjs

187 lines
14 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { EngineManager } from '../src/core/engine-manager.mjs';
import { DeepSeekAdapter } from '../src/core/engines/deepseek-adapter.mjs';
import { resolveDeepSeekInstallation } from '../src/core/engines/deepseek-installation.mjs';
import { Store } from '../src/server/store.mjs';
import { Runtime, validateToolArgs } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { createEngineFixture } from '../scripts/engine-fixtures.mjs';
import { fixture, until } from './helpers.mjs';
function setup(t, options = {}) {
const f = fixture(t), root = createEngineFixture(path.join(f.root, 'harness 源码 #'), options);
const manager = new EngineManager(f.store, { adapterFactory: installation => new DeepSeekAdapter(installation, { timeoutMs: 3000 }) });
f.onCleanup(async () => {
await manager.stopAll();
for (const job of manager.jobs(f.session)) {
const directory = path.resolve(job.directory), base = path.resolve(os.tmpdir(), 'AgentWorkbench-Engines');
assert.equal(path.dirname(directory), base); assert.equal(path.basename(directory), job.id);
fs.rmSync(directory, { recursive: true, force: true, maxRetries: 3 });
}
});
const audit = () => fs.existsSync(path.join(root, 'wire.jsonl')) ? fs.readFileSync(path.join(root, 'wire.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line)) : [];
return { ...f, root, manager, audit };
}
test('engine detection seals the actual CLI, SDK, profile and Node and rejects mixed versions without launch', t => {
const f = setup(t), current = resolveDeepSeekInstallation(process.cwd(), f.root);
assert.equal(current.status, 'available'); assert.equal(current.version, '0.2.1-rc.2'); assert.equal(current.inputs.length, 13);
const pkg = path.join(f.root, 'packages/sdk/server/package.json');
fs.writeFileSync(pkg, JSON.stringify({ version: 'another-version' }));
assert.equal(f.manager.availability(f.root).status, 'unsupported'); assert.throws(() => f.manager.start(f.session, { root: f.root }), /版本不一致/);
assert.equal(f.audit().length, 0); assert.equal(f.store.core.engine_job.size, 0);
fs.writeFileSync(pkg, JSON.stringify({ version: '0.2.1-rc.2' }));
const detected = f.manager.availability(f.root); fs.appendFileSync(path.join(f.root, 'apps/cli/lib/bin.js'), '\n// changed after detection\n');
assert.throws(() => f.manager.start(f.session, { root: f.root, identitySha256: detected.identitySha256 }), /检测结果不一致/);
assert.equal(f.audit().length, 0); assert.equal(f.store.core.engine_job.size, 0);
});
test('owned engine probe waits for real ready identity, verifies six native responses and saves immutable task evidence', async t => {
const f = setup(t, { readyDelayMs: 150 });
const originalBytes = fs.readFileSync(path.join(f.projectRoot, 'README.md'));
const started = f.manager.start(f.session, { root: f.root });
assert.throws(() => f.manager.start(f.session, { root: f.root }), /在执行/);
const job = await started.promise;
assert.equal(job.status, 'completed'); assert.equal(job.verification.verified, true); assert.equal(job.report.responses.length, 6);
assert.equal(job.report.modelRequests, 0); assert.equal(job.report.cleanup.rootExited, true); assert.equal(job.report.cleanup.exit.code, 0);
assert.equal(job.report.readiness.pid, job.pid); assert.equal(job.sdkSessionId, job.report.sdkSessionId);
assert.equal(f.store.executionSnapshot(f.session).engineJobs[0].report, undefined);
assert.equal(f.store.executionSnapshot(f.session).engineJobs[0].reportSummary.observedResponses, 6);
const original = JSON.parse(f.manager.originalReport(f.session, job));
assert.equal(original.owner.taskId, f.session.taskId); assert.equal(original.inputSha256, job.contractSha256);
assert.ok(original.report.responses[2].error.message.includes(original.report.responses[2].parameters.sessionId));
assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, 'README.md')), originalBytes);
assert.equal(f.store.core.task.get(f.session.taskId).usage.requests, 0); assert.equal(f.session.status, 'paused');
assert.equal(f.audit().filter(value => value.type === 'wire' && value.value.method === 'session/prompt').length, 0);
});
test('an invalid startup nonce is rejected before initialize and preserves the process close and partial report', async t => {
const f = setup(t, { mode: 'bad-ready' }), job = await f.manager.start(f.session, { root: f.root }).promise;
assert.equal(job.status, 'failed'); assert.match(job.summary, /启动完成信号/); assert.equal(job.report.trace.length, 0);
assert.equal(job.report.cleanup.rootExited, true); assert.equal(f.audit().filter(value => value.type === 'wire').length, 0);
assert.equal(f.manager.summary(f.session, job).evidence_verified, false);
});
test('SDK identity and session directory mismatches retain raw responses and never become verified', async t => {
for (const mode of ['wrong-version', 'wrong-cwd', 'malformed']) {
const f = setup(t, { mode }), job = await f.manager.start(f.session, { root: f.root }).promise;
assert.equal(job.status, 'failed', mode); assert.equal(job.verification, null); assert.ok(job.reportArtifactId);
assert.equal(job.report.cleanup.rootExited, true); assert.equal(f.manager.summary(f.session, job).evidence_verified, false);
if (mode !== 'malformed') assert.ok(job.report.responses.length);
}
});
test('cancel during a pending initialize closes only the owned process without sending an unsupported MCP cancellation', async t => {
const f = setup(t, { mode: 'hang-init' }), started = f.manager.start(f.session, { root: f.root });
await until(() => started.job.status === 'initializing' && f.audit().some(value => value.value?.method === 'initialize'));
await f.manager.stop(f.session.id);
const job = await started.promise;
assert.equal(job.status, 'cancelled'); assert.equal(job.report.cleanup.rootExited, true);
assert.equal(job.report.cleanup.pid, job.pid); assert.equal(job.requiresVerification, false);
assert.equal(f.audit().filter(value => value.value?.method === 'notifications/cancelled').length, 0);
assert.throws(() => f.manager.start(f.session, { root: f.root }), /重新验证/);
});
test('cancel before readiness sends no protocol request and a user retry creates a different process and record', async t => {
const f = setup(t, { mode: 'no-ready' }), started = f.manager.start(f.session, { root: f.root });
await until(() => started.job.pid && f.audit().length);
await f.manager.stop(f.session.id); assert.equal((await started.promise).status, 'cancelled');
assert.equal(f.audit().filter(value => value.type === 'wire').length, 0);
fs.writeFileSync(path.join(f.root, 'fixture-options.json'), '{}');
const retry = await f.manager.start(f.session, { root: f.root, retry: true }).promise;
assert.equal(retry.status, 'completed'); assert.notEqual(retry.id, started.job.id); assert.notEqual(retry.pid, started.job.pid);
});
test('source or scratch changes and unexpected model traffic fail after preserving the actual observed result', async t => {
for (const mode of ['change-input', 'change-workspace', 'model-request']) {
const f = setup(t, { mode }), job = await f.manager.start(f.session, { root: f.root }).promise;
assert.equal(job.status, 'failed', mode); assert.equal(job.report.cleanup.rootExited, true); assert.equal(job.verification, null);
if (mode === 'model-request') { assert.equal(job.report.modelRequests, 1); assert.equal(job.report.providerRequests, 0); }
assert.equal(f.session.changes.length, 0);
}
});
test('unowned idle notifications stay quarantined and server requests receive a rejection without granting operations', async t => {
const f = setup(t, { mode: 'unowned-event' }), job = await f.manager.start(f.session, { root: f.root }).promise;
assert.equal(job.status, 'completed'); assert.equal(job.report.notifications.length, 0); assert.equal(job.report.unownedNotifications.length, 1);
assert.equal(f.store.core.task.get(f.session.taskId).status, 'paused');
const reverse = setup(t, { mode: 'reverse-request' }), response = await reverse.manager.start(reverse.session, { root: reverse.root }).promise;
assert.equal(response.status, 'completed'); assert.equal(response.report.rejectedServerRequests.length, 1);
assert.ok(reverse.audit().some(value => value.value?.id === 'reverse-control' && value.value?.error?.code === -32601));
assert.equal(reverse.session.approvals.length, 0); assert.equal(reverse.session.changes.length, 0);
});
test('engine probe environment excludes parent secrets and NODE_OPTIONS while preserving operating system paths', async t => {
const f = setup(t), previousSecret = process.env.OWNED_ENGINE_TEST_SECRET, previousOptions = process.env.NODE_OPTIONS;
process.env.OWNED_ENGINE_TEST_SECRET = 'owned-non-credential-test'; process.env.NODE_OPTIONS = '--no-warnings';
try {
const job = await f.manager.start(f.session, { root: f.root }).promise; assert.equal(job.status, 'completed');
const launch = f.audit()[0]; assert.equal(launch.secretInherited, false); assert.equal(launch.nodeOptionsInherited, false);
} finally {
if (previousSecret === undefined) delete process.env.OWNED_ENGINE_TEST_SECRET; else process.env.OWNED_ENGINE_TEST_SECRET = previousSecret;
if (previousOptions === undefined) delete process.env.NODE_OPTIONS; else process.env.NODE_OPTIONS = previousOptions;
}
});
test('foreign sessions and altered reports or task bindings cannot reuse verified engine evidence', async t => {
const f = setup(t), job = await f.manager.start(f.session, { root: f.root }).promise, other = f.store.createSession(f.project.id);
assert.throws(() => f.manager.originalReport(other, job), /不属于/);
const original = job.sdkSessionId; job.sdkSessionId = other.id;
assert.equal(f.manager.summary(f.session, job).evidence_verified, false); assert.throws(() => f.manager.originalReport(f.session, job), /不一致/); job.sdkSessionId = original;
const artifact = f.store.core.artifact.get(job.reportArtifactId), filename = path.join(f.store.artifactStore.root, artifact.relative);
fs.appendFileSync(filename, 'tampered'); assert.equal(f.manager.summary(f.session, job).evidence_verified, false);
assert.throws(() => f.manager.originalReport(f.session, job), /校验失败/);
});
test('restart never replays an active engine check or stops a historical PID, and keeps prior task state', async t => {
const f = setup(t), job = await f.manager.start(f.session, { root: f.root }).promise;
const calls = f.audit().length; job.status = 'starting'; job.pid = process.pid; job.verification = null;
f.store.putRecord('engine_job', f.session, job, 'fixture.crash_window'); f.store.close();
const reopened = new Store(f.data); f.onCleanup(() => reopened.close());
const saved = reopened.core.engine_job.get(job.id);
assert.equal(saved.status, 'unknown'); assert.equal(saved.requiresVerification, true); assert.equal(saved.pid, process.pid);
assert.equal(f.audit().length, calls); assert.equal(reopened.session(f.session.id).taskId, f.session.taskId);
});
test('model argument validation and runtime activity do not grant external engine prompt or project execution', t => {
const f = setup(t), runtime = new Runtime(f.store, new Provider());
assert.doesNotThrow(() => validateToolArgs('inspect_engine_checks', {}));
assert.throws(() => validateToolArgs('inspect_engine_checks', { root: f.root }), /当前任务/);
assert.throws(() => validateToolArgs('engine_prompt', {}), /没有工具/);
assert.throws(() => f.manager.start(f.session, { root: f.root, prompt: 'not accepted' }), /只接受/);
runtime.engines = { active: new Map([[f.session.id, {}]]), isActive: id => id === f.session.id, projectActive: id => id === f.project.id };
assert.equal(runtime.hasActivity(), true); assert.equal(runtime.isActive(f.session.id), true); assert.equal(runtime.projectHasActivity(f.project.id), true);
assert.throws(() => runtime.start(f.session.id, 'fixture request'), /在执行/);
});
test('a follow-up question receives the actual failed engine check from its original Task and keeps its report unchanged', async t => {
const f = setup(t, { mode: 'bad-ready' }), job = await f.manager.start(f.session, { root: f.root }).promise;
assert.equal(job.status, 'failed'); const originalReport = f.manager.originalReport(f.session, job), taskId = job.taskId;
const originalWireCount = f.audit().length, originalGoal = f.store.core.task.get(taskId).goal;
const provider = new Provider(); let requests = 0;
provider.complete = async (_config, messages, _project, _signal, _progress, _plan, tools) => {
requests++;
assert.ok(tools.some(tool => tool.function.name === 'inspect_engine_checks'));
if (requests === 1) return { message: { role: 'assistant', content: '', tool_calls: [
{ id: 'owned-engine-question', type: 'function', function: { name: 'inspect_engine_checks', arguments: '{}' } },
] }, usage: null };
const result = JSON.parse(messages.filter(message => message.role === 'tool').at(-1).content), check = result.engine_checks[0];
assert.equal(check.engine_job_id, job.id); assert.equal(check.task_id, taskId); assert.equal(check.engine_status, 'failed');
assert.match(check.message, /启动完成信号/); assert.equal(check.report_sha256, job.reportSha256);
return { message: { role: 'assistant', content: '当前任务的引擎启动完成信号与所属进程不一致,检查未通过;原进程已经关闭,原记录仍保留。' }, usage: null };
};
const runtime = new Runtime(f.store, provider); runtime.engines = f.manager;
f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, '刚才的引擎连接检查为什么报错?'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.taskId, taskId); assert.equal(f.session.status, 'paused'); assert.equal(requests, 2);
const diagnostic = f.store.core.run.get(f.session.runId);
assert.equal(diagnostic.purpose, 'diagnostic'); assert.equal(diagnostic.status, 'completed'); assert.equal(diagnostic.stopReason, 'diagnostic_answered');
assert.equal(f.store.core.task.get(taskId).status, 'paused'); assert.equal(f.store.core.task.get(taskId).goal, originalGoal);
assert.equal(f.audit().length, originalWireCount, 'Reading the failed original report must not start or prompt the external engine again.');
assert.equal(f.manager.originalReport(f.session, job), originalReport); assert.match(f.session.messages.at(-1).content, /当前任务/);
});