Files
Agent-Workbench/tests/turn-progress.test.mjs
T

217 lines
20 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { createHash, randomUUID } from 'node:crypto';
import { observeTurn, toolProgressFacts, TURN_PROGRESS_LIMITS } from '../src/core/context/turn-progress.mjs';
import { recordTurnProgress, inspectTurnProgress } from '../src/core/context/turn-progress-recorder.mjs';
import { fixture, compatibleFixture, call, completion, assertToolPairs, until } from './helpers.mjs';
import { beginPlanTask, completeRead } from './fixtures/task-plan-fixture.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Store } from '../src/server/store.mjs';
import { DeliveryReporter } from '../src/core/delivery-report.mjs';
import { RequirementReview } from '../src/core/tasks/requirement-review.mjs';
import { CheckRunner } from '../src/core/check-runner.mjs';
import { proposeFile, applyChange } from '../src/server/files.mjs';
const sha = value => createHash('sha256').update(value).digest('hex');
const facts = (fields = {}) => ({ projectId: 'p', sessionId: 's', taskId: 't', runId: 'r', stepIndex: 0, phase: 'settled', ...fields });
const read = (start, end, fields = {}) => ({ id: randomUUID(), name: 'read_file', argsSha256: 'args', read: { key: 'original-file-range', sha256: sha('source'), unit: 'utf16_in_original_range', start, end, ...fields } });
const fail = (category, signature = 'original error', fields = {}) => ({ id: randomUUID(), name: 'read_file', argsSha256: 'same-arguments', failure: { category, signature }, ...fields });
test('fixed normal exploration, pagination, SHA changes and repeated ranges are external observations, not a score', () => {
let state, result; const expected = [
[read(0, 4), 'evidence_growth', 4], [read(4, 8), 'evidence_growth', 4], [read(2, 6), 'repeated_read_candidate', 0],
[read(6, 10), 'evidence_growth', 2], [read(0, 4, { sha256: sha('changed source') }), 'evidence_growth', 4],
];
for (const [tool, classification, growth] of expected) { result = observeTurn(state, facts({ tools: [tool] })); state = result.state;
assert.equal(result.observation.classification, classification); assert.equal(result.observation.delta.read_characters, growth); }
assert.equal(state.mode, 'shadow'); assert.equal('score' in result.observation, false); assert.equal('shouldAbort' in result.observation, false);
});
test('read-only research is valid work; model summaries and unchanged timestamped listings do not become engineering evidence', () => {
const first = observeTurn(null, facts({ tools: [read(0, 30)] })); assert.equal(first.observation.classification, 'evidence_growth');
const conclusion = observeTurn(first.state, facts({ modelText: 'I have finished all requirements', privateReasoning: 'DO_NOT_SAVE' }));
assert.equal(conclusion.observation.classification, 'no_observed_change'); assert.ok(!JSON.stringify(conclusion).includes('DO_NOT_SAVE'));
const record = { id: 'first', tool: 'list_files', argsSha256: 'same' };
const a = toolProgressFacts(record, { files: ['a'], inspectedAt: 'first', tool_invocation_id: '1' });
const b = toolProgressFacts({ ...record, id: 'second' }, { files: ['a'], inspectedAt: 'later', tool_invocation_id: '2' });
const result = observeTurn(observeTurn(null, facts({ tools: [a] })).state, facts({ tools: [b] }));
assert.equal(result.observation.classification, 'no_observed_change');
});
test('new permissions, network and broken tools are failure evidence; repeated invalid inputs remain candidates', () => {
let state;
for (const category of ['permission', 'network', 'invalid_input', 'tool_or_environment']) {
let result = observeTurn(state, facts({ tools: [fail(category)] })); state = result.state; assert.equal(result.observation.classification, 'new_failure_evidence');
result = observeTurn(state, facts({ tools: [fail(category)] })); state = result.state; assert.equal(result.observation.classification, 'repeated_failure_candidate');
assert.equal(result.observation.delta.repeated_failures, 1);
}
const changed = observeTurn(state, facts({ tools: [fail('network', 'different actual error')] })); assert.equal(changed.observation.classification, 'new_failure_evidence');
});
test('waiting for original long checks, review and approvals stays waiting across repeated observations', () => {
let state;
for (const kind of ['check_job', 'approval', 'review']) for (let i = 0; i < 4; i++) {
const result = observeTurn(state, facts({ tools: [fail('permission')], waiting: [{ kind, id: 'owned-operation' }] })); state = result.state;
assert.equal(result.observation.classification, 'waiting'); assert.equal(result.observation.waiting[0].kind, kind);
}
});
test('baseline facts, real repairs, current-source check results and requirement evidence have distinct growth', () => {
const prior = { changes: [['a', 'sha-old']], checks: [['old-source', 'failed', 'log']], requirements: [['req', 'rev1', 'proof']] };
let result = observeTurn(null, facts({ ...prior, phase: 'baseline' })); assert.equal(result.observation.classification, 'baseline');
assert.equal(result.observation.delta.check_results, 0);
result = observeTurn(result.state, facts(prior)); assert.equal(result.observation.classification, 'no_observed_change');
result = observeTurn(result.state, facts({ changes: [['a', 'sha-new']], checks: [['new-source', 'passed', 'new-log']], requirements: [['req', 'rev2', 'new-proof']] }));
assert.equal(result.observation.delta.changed_files, 1); assert.equal(result.observation.delta.check_results, 1); assert.equal(result.observation.delta.requirement_evidence, 1);
result = observeTurn(result.state, facts({ userBoundaries: [['message-1', 'same-text'], ['message-2', 'same-text']] }));
assert.equal(result.observation.classification, 'user_boundary'); assert.equal(result.observation.delta.user_boundaries, 2);
});
test('bounded memory keeps unknown instead of claiming an unretained range is repeated; Task/Run instances cannot mix', () => {
let state;
for (let i = 0; i < TURN_PROGRESS_LIMITS.sources; i++) state = observeTurn(state, facts({ tools: [read(0, 1, { key: String(i) })] })).state;
const full = observeTurn(state, facts({ tools: [read(0, 1, { key: 'unretained' })] }));
assert.equal(full.observation.classification, 'unknown_bounded'); assert.equal(full.observation.delta.repeated_reads, 0); assert.equal(full.state.reads.length, TURN_PROGRESS_LIMITS.sources);
assert.throws(() => observeTurn(state, facts({ runId: 'another' })), /another Task\/Run/);
for (let i = 0; i < 70; i++) state = observeTurn(state, facts()).state;
assert.equal(state.turns.length, TURN_PROGRESS_LIMITS.turns); assert.equal(state.truncated, true);
});
test('actual runtime records growing pages then a repeated page without changing model rounds, stop rules or original tool pairs', async t => {
const f = fixture(t); let requests = 0;
const model = await compatibleFixture(t, (url, body) => {
requests++; assert.ok(!JSON.stringify(body).includes('same_sha_range_already_returned')); assert.ok(!JSON.stringify(body).includes('turn-progress-shadow'));
return { body: requests < 4 ? completion([call('page-' + requests, 'read_file', { path: 'README.md', offset: requests === 1 ? 0 : 10, limit: 10 })]) : completion([], '研究结论:已读取原说明前两页。') };
});
f.store.saveConfig({ mode: 'compatible', model: 'owned-shadow', baseUrl: model.baseUrl, maxSteps: 6 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '只研究原README说明,不修改文件。');
await runtime.active.get(f.session.id).promise;
const run = f.store.core.run.get(f.session.runId), turns = run.progressShadow.turns.filter(row => row.phase === 'settled');
assert.equal(requests, 4); assert.equal(turns.length, 4); assert.equal(turns[0].classification, 'evidence_growth'); assert.equal(turns[0].delta.read_characters, 10);
assert.equal(turns[1].delta.read_characters, 10); assert.equal(turns[2].classification, 'repeated_read_candidate');
assert.equal(turns[3].classification, 'no_observed_change'); assert.equal(f.session.status, 'completed'); assert.equal(run.stopReason, null);
assertToolPairs(f.session.providerMessages); assert.equal(f.session.changes.length, 0);
const original = structuredClone(run.progressShadow), report = await new DeliveryReporter(f.store).create(f.session);
assert.equal(report.report.turnObservations[0].evidence_verified, true);
f.store.close(); const restored = new Store(f.data); f.onCleanup(() => restored.close());
assert.deepEqual(restored.core.run.get(run.id).progressShadow, original);
const ref = turns[0].tools[0], record = restored.owned('tool', ref.id, f.session.id), argsSha = record.argsSha256;
record.argsSha256 = '0'.repeat(64); assert.equal(inspectTurnProgress(restored, restored.session(f.session.id), restored.core.run.get(run.id)).evidence_verified, false); record.argsSha256 = argsSha;
const artifact = restored.owned('artifact', ref.result_artifact_id, f.session.id); artifact.sha256 = '0'.repeat(64);
assert.equal(inspectTurnProgress(restored, restored.session(f.session.id), restored.core.run.get(run.id)).evidence_verified, false);
});
test('actual invalid tool calls repeat under the original step budget; observation never adds an early stop', async t => {
const f = fixture(t); let requests = 0;
const model = await compatibleFixture(t, () => ({ body: completion([call('bad-' + (++requests), 'read_file', { path: 17 })]) }));
f.store.saveConfig({ mode: 'compatible', model: 'owned-invalid', baseUrl: model.baseUrl, maxSteps: 4 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '研究已有文件'); await runtime.active.get(f.session.id).promise;
const run = f.store.core.run.get(f.session.runId), turns = run.progressShadow.turns.filter(row => row.phase === 'settled');
assert.equal(requests, 4); assert.equal(run.stopReason, 'step_limit'); assert.equal(turns[0].classification, 'new_failure_evidence');
assert.equal(turns[1].classification, 'repeated_failure_candidate'); assert.ok(turns[0].reasons.includes('new_failure:invalid_input')); assertToolPairs(f.session.providerMessages);
});
test('actual command approval is observed as waiting and denial stays a permission failure', async t => {
const f = fixture(t); let requests = 0;
const model = await compatibleFixture(t, () => ({ body: ++requests === 1 ? completion([call('read-before-approval', 'read_file', { path: 'README.md' }), call('approval', 'run_command', { command: 'node --version', reason: '所属环境检查' })]) : completion([], '审批拒绝,原目标保留。') }));
f.store.saveConfig({ mode: 'compatible', model: 'owned-waiting', baseUrl: model.baseUrl, maxSteps: 3 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '检查所属环境');
await until(() => f.session.status === 'awaiting_approval'); const run = f.store.core.run.get(f.session.runId);
assert.equal(run.progressShadow.turns.at(-1).classification, 'waiting'); assert.equal(run.progressShadow.turns.at(-1).waiting[0].kind, 'approval');
runtime.decide(f.session.id, f.session.approvals.at(-1).id, false); await runtime.active.get(f.session.id).promise;
assert.ok(run.progressShadow.turns.some(row => row.reasons.includes('new_failure:permission'))); assert.equal(requests, 2); assert.equal(f.session.approvals[0].status, 'denied');
const settled = run.progressShadow.turns.find(row => row.phase === 'settled');
assert.ok(settled.delta.read_characters > 0); assert.equal(settled.delta.repeated_reads, 0); assertToolPairs(f.session.providerMessages);
});
test('network errors remain model failures with shadow failure evidence, never a fabricated no-progress cause', async t => {
const f = fixture(t);
const model = await compatibleFixture(t, () => ({ status: 502, body: { error: { message: 'Owned upstream gateway failure' } } }));
f.store.saveConfig({ mode: 'compatible', model: 'owned-network', baseUrl: model.baseUrl, maxSteps: 2 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '读取当前工程'); await runtime.active.get(f.session.id).promise;
const run = f.store.core.run.get(f.session.runId); assert.equal(run.stopReason, null); assert.equal(f.session.status, 'failed');
assert.equal(f.session.lastFailure.runId, run.id); assert.equal(f.session.lastFailure.httpStatus, 502);
assert.ok(run.progressShadow.turns.some(row => row.classification === 'new_failure_evidence')); assert.equal(run.progressShadow.turns.at(-1).tools[0].tool, 'model_request');
assert.equal(inspectTurnProgress(f.store, f.session, run).evidence_verified, true);
f.session.events.find(row => row.id === run.progressShadow.turns.at(-1).tools[0].source_event_id).diagnostic.httpStatus = 503;
assert.equal(inspectTurnProgress(f.store, f.session, run).evidence_verified, false);
});
test('observer receipt failure cannot change tool results, model budget or completion', async t => {
const f = fixture(t), original = f.store.artifactStore.write.bind(f.store.artifactStore); let requests = 0;
f.store.artifactStore.write = (session, run, value, options) => { if (options?.kind === 'turn-progress-shadow') throw new Error('owned observer failure'); return original(session, run, value, options); };
const model = await compatibleFixture(t, () => ({ body: ++requests === 1 ? completion([call('read', 'read_file', { path: 'README.md' })]) : completion([], '原工具完成') }));
f.store.saveConfig({ mode: 'compatible', model: 'owned-observer-fault', baseUrl: model.baseUrl, maxSteps: 3 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '研究当前说明'); await runtime.active.get(f.session.id).promise;
assert.equal(requests, 2); assert.equal(f.session.status, 'completed'); assert.equal(f.store.core.run.get(f.session.runId).progressShadowError.kind, 'observation_unavailable');
assert.equal(f.store.savedToolResult(f.session.id, 'read').content.length > 0, true); assertToolPairs(f.session.providerMessages);
});
test('a later receipt recovers after an observation write failure without misaligning retained source receipts', async t => {
const f = fixture(t), original = f.store.artifactStore.write.bind(f.store.artifactStore); let requests = 0, observations = 0;
f.store.artifactStore.write = (session, run, value, options) => {
if (options?.kind === 'turn-progress-shadow' && ++observations === 2) throw new Error('owned single observation fault');
return original(session, run, value, options);
};
const model = await compatibleFixture(t, () => ({ body: ++requests < 3 ? completion([call('recovery-' + requests, 'read_file', { path: 'README.md', offset: (requests - 1) * 10, limit: 10 })]) : completion([], '原研究工具完成') }));
f.store.saveConfig({ mode: 'compatible', model: 'owned-recovery', baseUrl: model.baseUrl, maxSteps: 4 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll()); runtime.start(f.session.id, '研究当前说明'); await runtime.active.get(f.session.id).promise;
const run = f.store.core.run.get(f.session.runId), view = inspectTurnProgress(f.store, f.session, run);
assert.equal(requests, 3); assert.equal(f.session.status, 'completed'); assert.equal(view.evidence_verified, true); assert.match(view.error, /部分观察未能记录/);
assert.equal(view.turns.length, view.artifact_ids.length); assert.equal(view.turns.length, 3); assertToolPairs(f.session.providerMessages);
});
test('real applied files, counted checks and current requirement proofs grow after an existing-facts baseline', async t => {
const f = fixture(t);
fs.writeFileSync(path.join(f.projectRoot, 'package.json'), JSON.stringify({ private: true, type: 'module', scripts: { test: 'node --test check.test.mjs' } }));
fs.writeFileSync(path.join(f.projectRoot, 'check.test.mjs'), "import test from 'node:test'; import assert from 'node:assert/strict'; test('owned evidence',()=>assert.equal(2+2,4));\n");
const run = beginPlanTask(f, '解释README.md的实际内容,保留工程证据。'), manager = new RequirementReview(f.store), runner = new CheckRunner(f.store);
f.onCleanup(() => runner.stopAll());
const read = completeRead(f);
await manager.save(f.session, { expected_revision: 0, requirements: [{ key: 'read', title: '原说明', criterion: '原读取可核对', kind: 'analysis', tools: ['read_file'], method: 'evidence', source: { kind: 'goal', id: run.taskId, start: 0, end: 5 } }] });
await manager.verify(f.session, { expected_revision: 1, key: 'read', evidence_id: read.id });
let plan = await runner.plan(f.session, ['test']), job = await (await runner.start(f.session, plan.id)).promise;
assert.equal(job.status, 'passed'); assert.ok(job.results[0].tests.passed > 0);
const baseline = recordTurnProgress(f.store, f.session, run, null, 'baseline');
assert.equal(baseline.delta.check_results, 0); assert.equal(baseline.delta.requirement_evidence, 0);
assert.equal(recordTurnProgress(f.store, f.session, run, null).classification, 'no_observed_change');
const change = proposeFile(f.store, f.session, { path: 'README.md', content: '# Newly observed actual content\n', reason: 'owned applied evidence' });
applyChange(f.store, f.session, change.change_id);
const current = completeRead(f); await manager.verify(f.session, { expected_revision: 1, key: 'read', evidence_id: current.id });
plan = await runner.plan(f.session, ['test']); job = await (await runner.start(f.session, plan.id)).promise; assert.equal(job.status, 'passed');
const observed = recordTurnProgress(f.store, f.session, run, null);
assert.equal(observed.delta.changed_files, 1); assert.equal(observed.delta.check_results, 1); assert.equal(observed.delta.requirement_evidence, 1);
assert.equal(inspectTurnProgress(f.store, f.session, run).evidence_verified, true);
});
test('receipt retention stays aligned at the cap and a foreign step cannot enter the Run', t => {
const f = fixture(t), run = beginPlanTask(f);
for (let i = 0; i < 43; i++) assert.ok(recordTurnProgress(f.store, f.session, run, null, i === 0 ? 'baseline' : 'settled'));
assert.equal(run.progressShadow.turns.length, 40); assert.equal(run.progressShadowArtifactIds.length, 40);
assert.equal([...f.store.core.artifact.values()].filter(row => row.kind === 'turn-progress-shadow').length, 43);
assert.equal(inspectTurnProgress(f.store, f.session, run).evidence_verified, true);
assert.equal(recordTurnProgress(f.store, f.session, run, { runId: 'foreign' }), null); assert.equal(run.progressShadow.observations, 43);
});
test('an actual approved long check records its owned running job as waiting before the original counted result', async t => {
const f = fixture(t); let requests = 0;
fs.writeFileSync(path.join(f.projectRoot, 'package.json'), JSON.stringify({ private: true, type: 'module', scripts: { test: 'node --test check.test.mjs' } }));
fs.writeFileSync(path.join(f.projectRoot, 'check.test.mjs'), "import test from 'node:test'; test('owned long check',async()=>{await new Promise(resolve=>setTimeout(resolve,750));});\n");
const model = await compatibleFixture(t, () => ({ body: ++requests === 1 ? completion([call('check-plan', 'plan_checks', { names: ['test'] })])
: requests === 2 ? completion([call('check-run', 'run_checks', { plan_id: [...f.store.core.check_plan.values()].at(-1).id })]) : completion([], '所属实际检查结果已保留。') }));
f.store.saveConfig({ mode: 'compatible', model: 'owned-long-check', baseUrl: model.baseUrl, maxSteps: 4 });
const runtime = new Runtime(f.store, new Provider()); f.onCleanup(() => runtime.stopAll()); f.onCleanup(() => runtime.checkRunner().stopAll());
runtime.start(f.session.id, '检查当前工程并解释实际检查结果。');
await until(() => f.session.status === 'awaiting_approval'); runtime.decide(f.session.id, f.session.approvals.at(-1).id, true);
const run = f.store.core.run.get(f.session.runId);
await until(() => run.progressShadow.turns.at(-1)?.waiting.some(row => row.kind === 'check_job'));
assert.equal(requests, 2); assert.equal(run.progressShadow.turns.at(-1).classification, 'waiting');
await runtime.active.get(f.session.id).promise;
assert.equal(requests, 3); assert.equal(f.session.status, 'completed'); assert.equal(run.stopReason, null);
assert.equal([...f.store.core.check_job.values()].at(-1).status, 'passed');
assert.ok(run.progressShadow.turns.some(row => row.delta.check_results === 1)); assertToolPairs(f.session.providerMessages);
});