Files
Agent-Workbench/tests/unrestricted-execution.test.mjs

214 lines
17 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import { fixture, compatibleFixture, completion, call, until } from './helpers.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { completionBody } from '../src/shared/contracts/model-request.mjs';
import { openDefaults } from '../src/shared/contracts/execution-policy.mjs';
import { validateRuntimeBudgets } from '../src/core/models/runtime-settings.mjs';
import { budgetFor, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
import { readText, readLines } from '../src/server/files.mjs';
import { requestDeadline } from '../src/core/models/request-deadline.mjs';
import { createDeliveryProject, deliveryGit } from './fixtures/delivery-project.mjs';
import { launchWorkbench } from './helpers.mjs';
import { taskPlanInput } from '../src/shared/contracts/task-plans.mjs';
import { ChangeSetManager } from '../src/core/workspaces/change-sets.mjs';
import { proposeFile, rollbackChange } from '../src/server/files.mjs';
test('an unrestricted continuation releases an exhausted legacy Task allowance without rewriting its original Run or usage', async t => {
const f=fixture(t),first=f.store.beginRun(f.session,'Original bounded task',false,f.store.config),task=f.store.core.task.get(first.taskId);
task.budget.maxTokens=100;task.usage.totalTokens=200;
f.store.finishRun(f.session,first,'task_token_limit');
const original=structuredClone(first),usage=structuredClone(task.usage);
const server=await compatibleFixture(t,(_url,body)=>{for(const key of ['max_tokens','max_output_tokens','max_completion_tokens'])assert.equal(Object.hasOwn(body,key),false);return completion([],'Continued the original task.');});
f.store.saveConfig({...openDefaults,mode:'compatible',model:'owned',baseUrl:server.baseUrl});
const runtime=new Runtime(f.store,new Provider());f.onCleanup(()=>runtime.stopAll());
runtime.start(f.session.id,'Continue the saved task.',true);await until(()=>!runtime.isActive(f.session.id),30000);
assert.equal(f.session.status,'completed',JSON.stringify(f.session.failure));assert.equal(f.session.taskId,task.id);
assert.equal(task.budget.maxTokens,0);assert.equal(server.requests.length,1);
assert.ok(task.usage.totalTokens>=usage.totalTokens);assert.deepEqual(f.store.core.run.get(first.id),original);
});
test('open change sets apply and restore a file larger than the old four MiB batch ceiling', async t => {
const f = fixture(t, { modernDefaults: true }), before = '\uFEFFORIGINAL\r\n' + 'a'.repeat(5 * 1024 * 1024);
fs.writeFileSync(path.join(f.projectRoot, 'large-change.txt'), before);
const run = f.store.beginRun(f.session, 'Apply and restore the complete owned file.', false, f.store.config);
const proposal = proposeFile(f.store, f.session, { path: 'large-change.txt', content: 'COMPLETE-AFTER\n' + before, reason: 'Owned large-file regression.' });
const manager = new ChangeSetManager(f.store), set = await manager.prepare(f.session, [proposal.change_id]);
const operation = await manager.start(f.session, set.id, set.fingerprint); await operation.promise;
assert.equal(set.status, 'completed', set.summary);
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'large-change.txt'), 'utf8'), 'COMPLETE-AFTER\n' + before);
rollbackChange(f.store, f.session, proposal.change_id);
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'large-change.txt'), 'utf8'), before);
});
for (const protocol of ['chat-completions', 'responses', 'anthropic-messages']) test(protocol + ' passthrough preserves optional limits and Messages required max_tokens while manual settings remain exact', () => {
const config = { ...openDefaults, protocol, model: 'owned', contextLimit: 1000, maxOutputTokens: 256 };
const body = completionBody(config, [{ role: 'user', content: 'original'.repeat(2000) }], []);
for (const key of ['max_tokens', 'max_completion_tokens', 'max_output_tokens']) assert.equal(Object.hasOwn(body, key), protocol === 'anthropic-messages' && key === 'max_tokens');
if (protocol === 'anthropic-messages') assert.equal(body.max_tokens, config.maxOutputTokens);
const manual = completionBody({ ...config, capacityMode: 'manual', maxOutputTokens: 90000 }, [{ role: 'user', content: 'manual' }], []);
assert.equal(manual[protocol === 'responses' ? 'max_output_tokens' : 'max_tokens'], 90000);
const budget = budgetFor(config); assert.equal(budget.contextLimit, null); assert.equal(budget.inputBudget, null);
});
test('open budgets persist real zero values and an unlimited request wait remains user cancellable', async () => {
assert.deepEqual(validateRuntimeBudgets({ executionMode: 'unrestricted', maxSteps: 0, maxRunDurationMs: 0, maxRunTokens: 0, maxTaskTokens: 0 }),
{ ...openDefaults });
const controller = new AbortController(), deadline = requestDeadline({ signal: controller.signal, timeoutMs: 0, idleTimeoutMs: 0 });
deadline.beginStream(); await new Promise(resolve => setTimeout(resolve, 30)); assert.equal(deadline.signal.aborted, false);
controller.abort(); assert.equal(deadline.signal.aborted, true); deadline.close();
});
test('default runtime passes the old window and 30-turn/tool-count limits without compaction or no-progress pauses', async t => {
const f = fixture(t, { modernDefaults: true }), provider = new Provider(); let count = 0;
fs.writeFileSync(path.join(f.projectRoot, 'large.txt'), 'SOURCE-END\n'.repeat(130000));
assert.ok(readText(f.store.project(f.project.id), 'large.txt').bytes > 1024 * 1024);
assert.equal(readLines(f.store.project(f.project.id), { path: 'large.txt' }).total_lines, 130001);
const server = await compatibleFixture(t, (_url, body) => {
assert.ok(body.messages.some(row => row.role === 'user' && row.content.includes('FULL-GOAL-END')));
assert.equal(Object.hasOwn(body, 'max_tokens'), false);
count++; return count <= 31 ? completion(Array.from({ length: count === 1 ? 13 : 1 }, (_, index) => call('read-' + count + '-' + index, 'list_files', { path: '', depth: 0 }))) : completion([], 'Finished.');
});
f.store.saveConfig({ ...openDefaults, contextLimit: 1000, maxOutputTokens: 256, mode: 'compatible', model: 'owned', baseUrl: server.baseUrl });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'X'.repeat(30000) + 'FULL-GOAL-END'); await until(() => !runtime.isActive(f.session.id), 180000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.failure)); assert.equal(server.requests.length, 32);
for (const plan of f.store.core.context_plan.values()) {
assert.equal(plan.archivedGroups, 0); assert.equal(plan.budget.inputBudget, null); verifyPlanEvidence(f.store, f.session, plan);
}
});
test('open runtime executes commands and file proposals automatically, retaining actual approvals and applied bytes', async t => {
const f = fixture(t, { modernDefaults: true }), provider = new Provider(); let count = 0;
const text = 'complete-file\n'.repeat(45000);
const server = await compatibleFixture(t, () => ++count === 1 ? completion([
call('write', 'propose_file', { path: 'generated.txt', content: text, reason: 'User requested this complete file.' }),
call('shell', 'run_command', { command: process.platform === 'win32' ? 'Write-Output "OPEN-COMMAND-OK"' : 'printf OPEN-COMMAND-OK', reason: 'Verify direct command execution.' }),
]) : completion([], 'Done.'));
f.store.saveConfig({ mode: 'compatible', model: 'owned', baseUrl: server.baseUrl, approvalMode: 'full' });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Create generated.txt and verify a command.'); await until(() => !runtime.isActive(f.session.id), 30000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.failure)); assert.equal(server.requests.length, 2);
assert.equal(fs.readFileSync(path.join(f.projectRoot, 'generated.txt'), 'utf8'), text);
assert.equal(f.session.changes[0].status, 'applied'); assert.equal(f.session.approvals[0].status, 'approved');
assert.equal(f.session.approvals[0].decisionSource, 'full-access-policy');
assert.ok(f.session.providerMessages.some(row => row.role === 'tool' && row.content.includes('OPEN-COMMAND-OK')));
});
test('a real open runtime file read returns the complete requested text beyond the old page size', async t => {
const f = fixture(t, { modernDefaults: true }), content = 'SOURCE-LINE\n'.repeat(3000) + 'READ-ORIGINAL-END';
fs.writeFileSync(path.join(f.projectRoot, 'source.txt'), content);
const server = await compatibleFixture(t, (_url, _body, requests) => requests.length === 1
? completion([call('full-read', 'read_file', { path: 'source.txt' })]) : completion([], 'Done.'));
f.store.saveConfig({ mode: 'compatible', model: 'owned', baseUrl: server.baseUrl });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Read source.txt completely.'); await until(() => !runtime.isActive(f.session.id), 30000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.failure));
const result = JSON.parse(f.session.providerMessages.find(row => row.tool_call_id === 'full-read').content);
assert.equal(result.content, content); assert.equal(result.truncated, false); assert.equal(result.budget_limited, false);
assert.ok(JSON.stringify(server.requests[1].input).includes('READ-ORIGINAL-END'));
});
for (const status of [400, 413, 429, 503]) test('upstream HTTP ' + status + ' returns its original body/code/id without retry or failover', async t => {
const f = fixture(t, { modernDefaults: true }), provider = new Provider();
const error = { error: { code: 'owned_' + status, type: 'owned_error', message: 'UPSTREAM-ORIGINAL-' + status, detail: 'd'.repeat(70000) } };
const server = await compatibleFixture(t, () => ({ status, body: error, headers: { 'x-request-id': 'request-' + status } }));
f.store.saveConfig({ mode: 'compatible', baseUrl: server.baseUrl, model: 'owned' });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Reply only.'); await until(() => !runtime.isActive(f.session.id), 15000);
assert.equal(server.requests.length, 1); const failure = f.session.failure;
assert.equal(failure.origin, 'upstream'); assert.equal(failure.httpStatus, status); assert.equal(failure.upstream.code, error.error.code);
assert.equal(failure.upstream.requestId, 'request-' + status); assert.equal(failure.retryDecision, 'passthrough');
assert.equal(failure.upstream.capture.complete, true); assert.equal(failure.upstream.capture.truncated, false);
assert.deepEqual(JSON.parse(f.store.artifactStore.read(f.store.core.artifact.get(failure.upstream.artifactId))), error);
});
test('an upstream length stop is retained once with usage and never executes a partial tool batch', async t => {
const f = fixture(t, { modernDefaults: true }), provider = new Provider();
const raw = completion([call('partial', 'propose_file', { path: 'never.txt', content: 'never', reason: 'Partial batch' })]); raw.choices[0].finish_reason = 'length';
const server = await compatibleFixture(t, () => raw);
f.store.saveConfig({ mode: 'compatible', model: 'owned', baseUrl: server.baseUrl });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Create the requested file.'); await until(() => !runtime.isActive(f.session.id), 15000);
assert.equal(server.requests.length, 1); assert.equal(f.session.failure.reason, 'output_truncated');
assert.equal(f.session.changes.length, 0); assert.equal(fs.existsSync(path.join(f.projectRoot, 'never.txt')), false);
assert.equal([...f.store.core.run.values()].at(-1).usage.requests, 1);
});
test('open local Git delivery executes the sealed plan without a check or review gate and preserves unrelated edits', async t => {
const f = fixture(t, { modernDefaults: true }); createDeliveryProject(f);
const staged = deliveryGit(f, ['show', ':owner-staged.txt']); let count = 0;
const server = await compatibleFixture(t, () => ++count === 1 ? completion([call('write', 'propose_file', {
path: 'src/add.mjs', content: 'export const add = (a, b) => a + b;\n', reason: 'Repair the requested source.'
})]) : count === 2 ? completion([call('commit', 'plan_delivery', { message: 'Repair addition in the owned fixture' })]) : completion([], 'Done.'));
f.store.saveConfig({ mode: 'compatible', model: 'owned', baseUrl: server.baseUrl });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Repair src/add.mjs and commit it locally without running checks.');
await until(() => !runtime.isActive(f.session.id), 45000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.failure));
const jobs = [...f.store.core.delivery_job.values()]; assert.equal(jobs.length, 1, JSON.stringify(f.session.events));
assert.equal(jobs[0].status, 'completed', jobs[0].summary); assert.equal(jobs[0].checkJobId, null);
assert.equal(deliveryGit(f, ['rev-parse', 'HEAD']).trim(), jobs[0].commit);
assert.equal(deliveryGit(f, ['show', ':owner-staged.txt']), staged);
assert.equal(deliveryGit(f, ['rev-list', '--count', 'HEAD']).trim(), '2');
assert.equal([...f.store.core.check_job.values()].length, 0);
assert.equal(f.store.core.task.get(f.session.taskId).verification.status, 'not_configured');
assert.deepEqual([...f.store.core.delivery_plan.values()][0].checkSummary, []);
});
test('an upstream JSON response larger than the old four MiB ceiling is delivered completely', async t => {
const f = fixture(t, { modernDefaults: true }), content = '完整上游正文'.repeat(350000) + 'ORIGINAL-END';
const server = await compatibleFixture(t, () => completion([], content));
f.store.saveConfig({ mode: 'compatible', model: 'owned', baseUrl: server.baseUrl });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Return the complete original response.'); await until(() => !runtime.isActive(f.session.id), 30000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.failure));
assert.equal(f.session.providerMessages.at(-1).content, content); assert.equal(server.requests.length, 1);
});
test('the real settings API saves and reloads all zero open defaults without changing the selected model', async t => {
const f = fixture(t), model = f.store.config.model; f.store.close();
const host = await launchWorkbench(t, f.data);
await host.api('/model-center/runtime-budgets', { executionMode: 'unrestricted' });
const state = await host.api('/bootstrap');
for (const [key, value] of Object.entries(openDefaults)) assert.equal(state.config[key], value, key);
assert.equal(state.config.model, model);
});
test('open native vendor parameters and large plans are kept without local capability or size rejection', () => {
const nativeParameters = { vendor_feature: { payload: '上游原生选项'.repeat(3000) }, temperature: 7, reasoning_effort: 'vendor-intensity' };
const body = completionBody({ ...openDefaults, model: 'owned', nativeParameters }, [{ role: 'user', content: 'Original goal' }], []);
for (const [key, value] of Object.entries(nativeParameters)) assert.deepEqual(body[key], value);
const steps = Array.from({ length: 25 }, (_, i) => ({ key: 'step_' + i, kind: 'changes', title: 'Goal ' + i,
criterion: 'Original criterion. '.repeat(100), paths: ['src/file_' + i + '.mjs'], depends_on: [] }));
assert.equal(taskPlanInput({ expected_revision: 0, steps }, openDefaults).steps.length, 25);
assert.throws(() => taskPlanInput({ expected_revision: 0, steps }), /12/);
});
test('an open SSE response larger than four MiB preserves the complete upstream text and usage', async t => {
const f = fixture(t, { modernDefaults: true }), content = 's'.repeat(5 * 1024 * 1024) + 'SSE-END';
const server = http.createServer(async (req, res) => {
for await (const _chunk of req) { /* Receive the exact request before replying. */ }
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
for (const value of [
{ choices: [{ index: 0, delta: { role: 'assistant', content }, finish_reason: null }] },
{ choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 8, completion_tokens: 9, total_tokens: 17 } }
]) res.write('data: ' + JSON.stringify(value) + '\n\n');
res.end('data: [DONE]\n\n');
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
const provider = new Provider(), hub = provider.modelHub(f.store);
const connection = hub.upsert({ name: 'Owned large stream', model: 'owned', requiresKey: false, streaming: true,
capabilityDeclarations: { streaming: 'supported' }, baseUrl: 'http://127.0.0.1:' + server.address().port + '/v1' });
hub.select({ type: 'connection', id: connection.id });
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, 'Return the complete stream.'); await until(() => !runtime.isActive(f.session.id), 30000);
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.failure)); assert.equal(f.session.providerMessages.at(-1).content, content);
assert.equal([...f.store.core.run.values()][0].usage.totalTokens, 17);
});