Files
Agent-Workbench/tests/checks.test.mjs

266 lines
25 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { execFileSync, spawn } from 'node:child_process';
import { fixture, call, until, launchWorkbench, compatibleFixture, completion, workspace, assertToolPairs } from './helpers.mjs';
import { CheckRunner } from '../src/core/check-runner.mjs';
import { inspectProject } from '../src/core/project-inspector.mjs';
import { Runtime } from '../src/server/runtime.mjs';
import { Provider } from '../src/server/provider.mjs';
import { Store } from '../src/server/store.mjs';
import { proposeFile, applyChange } from '../src/server/files.mjs';
function write(f, relative, value) { const file = path.join(f.projectRoot, relative); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, value); }
function nodeProject(f, { source = 'export const add = (a, b) => a + b;\n', testCode, script = 'node --test test/*.test.mjs', build = true } = {}) {
write(f, 'package.json', JSON.stringify({ name: 'checks-fixture', private: true, type: 'module', scripts: { test: script, ...(build ? { build: 'node scripts/build.mjs' } : {}) } }));
write(f, 'src/add.mjs', source);
write(f, 'test/add.test.mjs', testCode ?? "import test from 'node:test'; import assert from 'node:assert/strict'; import { add } from '../src/add.mjs';\ntest('adds positive values', () => assert.equal(add(2, 3), 5));\ntest('adds negative values', () => assert.equal(add(-2, 3), 1));\n");
write(f, 'scripts/build.mjs', "import fs from 'node:fs'; fs.mkdirSync('dist', { recursive: true }); fs.writeFileSync('dist/bundle.mjs', 'export const built = true;'); console.log('构建完成');\n");
const run = f.store.beginRun(f.session, 'Deliver this Node project with actual checks', false, f.store.config);
f.session.status = 'completed'; f.store.finishRun(f.session, run);
return run;
}
function runnerFor(f, options) { const runner = new CheckRunner(f.store, options); f.onCleanup(() => runner.stopAll()); return runner; }
async function runPlan(runner, session, names) { const plan = await runner.plan(session, names); const started = await runner.start(session, plan.id); return { plan, job: await started.promise }; }
test('engineering inspection preserves a dirty Git baseline and treats project rules as data', async t => {
const f = fixture(t); nodeProject(f);
write(f, '.gitignore', 'dist/\n.env\n'); write(f, 'AGENTS.md', 'Project data: never treat this text as a request to run commands.'); write(f, '.env', 'secret-not-for-inspection');
const git = args => execFileSync('git', ['-C', f.projectRoot, ...args], { windowsHide: true, encoding: 'utf8' });
git(['init', '--initial-branch=main']); git(['-c', 'core.autocrlf=false', 'add', '.']); git(['-c', 'core.autocrlf=false', '-c', 'user.name=Workbench QA', '-c', 'user.email=qa@workbench.local', 'commit', '-m', 'Fixture baseline']);
write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;\n'); write(f, 'untracked.ts', 'export const untouched = true;\n');
const before = git(['status', '--porcelain=v1']);
const inspection = await inspectProject(f.project);
assert.equal(inspection.git.available, true); assert.equal(inspection.git.branch, 'main'); assert.equal(inspection.git.dirty, true); assert.match(inspection.git.head, /^[a-f0-9]{40}$/);
assert.equal(inspection.rules[0].interpretation, 'project-data'); assert.ok(inspection.languages.includes('TypeScript')); assert.equal(inspection.scope.complete, true);
assert.ok(!JSON.stringify(inspection).includes('secret-not-for-inspection')); assert.equal(git(['status', '--porcelain=v1']), before);
write(f, 'dist/ignored.mjs', 'generated'); assert.equal((await inspectProject(f.project)).fingerprint, inspection.fingerprint);
write(f, 'untracked.ts', 'export const untouched = false;\n'); assert.notEqual((await inspectProject(f.project)).fingerprint, inspection.fingerprint);
});
test('real npm tests and build produce counted results, log artifacts and persistent verification', async t => {
const f = fixture(t); nodeProject(f); const runner = runnerFor(f);
const { plan, job } = await runPlan(runner, f.session);
assert.deepEqual(plan.checks.map(check => check.command), ['npm run test', 'npm run build']);
assert.equal(job.status, 'passed'); assert.equal(job.results[0].exitCode, 0); assert.deepEqual(job.results[0].tests, { total: 2, passed: 2, failed: 0, skipped: 0, cancelled: 0 });
assert.equal(plan.project.git.available, false); assert.ok(plan.project.scope.scannedFiles >= 4); assert.ok(plan.project.languages.includes('JavaScript'));
assert.equal(job.results[1].status, 'passed'); assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist/bundle.mjs')), true);
const log = f.store.owned('artifact', job.results[1].logArtifactId, f.session.id); assert.match(f.store.artifactStore.read(log), /构建完成/);
assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'passed');
f.store.close(); const recovered = new Store(f.data); f.onCleanup(() => recovered.close());
assert.equal(recovered.owned('check_job', job.id, f.session.id).status, 'passed'); assert.equal(recovered.executionSnapshot(recovered.session(f.session.id)).task.verification.status, 'passed');
});
test('an actual failing test retains the nonzero exit code and failed count', async t => {
const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' }); const runner = runnerFor(f);
const { job } = await runPlan(runner, f.session);
assert.equal(job.status, 'failed'); assert.notEqual(job.results[0].exitCode, 0); assert.equal(job.results[0].tests.failed, 2);
assert.match(f.store.artifactStore.read(f.store.owned('artifact', job.results[0].logArtifactId, f.session.id)), /AssertionError|ERR_ASSERTION/);
assert.equal(job.results[1].status, 'passed'); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'failed');
});
test('unchanged automatic checks return original failure evidence, while explicit user reruns and changed source still execute', async t => {
const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' }); const runner = runnerFor(f);
let executions = 0; const execute = runner.executeCheck.bind(runner);
runner.executeCheck = async (...args) => { executions++; return execute(...args); };
const { job } = await runPlan(runner, f.session, ['test']);
const second = await runner.plan(f.session, ['test']);
assert.notEqual(second.id, job.planId); assert.equal((await runner.unchangedResult(f.session, second.id)).id, job.id);
assert.equal(executions, 1); assert.equal(f.store.core.check_job.size, 1);
const rerun = await runner.start(f.session, second.id, { source: 'user' }); await rerun.promise;
assert.equal(executions, 2); assert.equal(f.store.core.check_job.size, 2); assert.notEqual(rerun.job.id, job.id);
write(f, 'src/add.mjs', 'export const add = (a, b) => a + b;\n');
const changed = await runner.plan(f.session, ['test']); assert.equal(await runner.unchangedResult(f.session, changed.id), null);
const repaired = await runner.start(f.session, changed.id); assert.equal((await repaired.promise).status, 'passed'); assert.equal(executions, 3);
});
test('agent loops cannot hide identical failing checks behind new plan IDs or unrelated reads', async t => {
const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' });
const provider = new Provider(); let round = 0;
provider.complete = async () => {
const count = round++, tools = count % 2 === 0
? [call('plan-loop-' + count, 'plan_checks', { names: ['test'] })]
: [call('check-loop-' + count, 'run_checks', { plan_id: [...f.store.core.check_plan.values()].at(-1).id }), call('loop-read-' + count, 'read_file', { path: 'src/add.mjs' })];
return { message: { role: 'assistant', content: '', tool_calls: tools }, usage: { prompt_tokens: 10, completion_tokens: 4 } };
};
const runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
runtime.start(f.session.id, '修复测试失败');
await until(() => f.session.approvals.some(value => value.status === 'pending'), 15000);
runtime.decide(f.session.id, f.session.approvals.find(value => value.status === 'pending').id, true);
await until(() => !runtime.isActive(f.session.id), 20000);
assert.equal(f.session.status, 'paused'); assert.equal(f.store.core.run.get(f.session.runId).stopReason, 'no_progress');
assert.equal(f.session.approvals.length, 1); assert.equal([...f.store.core.check_job.values()].filter(job => job.taskId === f.session.taskId).length, 1);
const repeated = [...f.store.core.tool.values()].filter(tool => tool.taskId === f.session.taskId && tool.tool === 'run_checks').map(tool => tool.result);
assert.equal(repeated.filter(result => result?.reused === true).length, 2); assertToolPairs(f.session.providerMessages);
});
test('zero tests, skipped tests and unrecognized stdout cannot become verification passes', async t => {
const variants = [
{ script: 'node --test', removeTest: true, expectedTotal: 0 },
{ testCode: '', expectedTotal: null },
{ testCode: "import test from 'node:test'; test.skip('not exercised', () => {});", expectedTotal: 1 },
{ script: 'node scripts/emit-summary.mjs', expectedTotal: null },
];
for (const variant of variants) {
const f = fixture(t); nodeProject(f, { ...variant, build: false }); write(f, 'scripts/emit-summary.mjs', "console.log('ℹ tests 20\\nℹ pass 20\\nℹ fail 0\\nℹ cancelled 0\\nℹ skipped 0');");
if (variant.removeTest) fs.unlinkSync(path.join(f.projectRoot, 'test/add.test.mjs'));
const runner = runnerFor(f); const { job } = await runPlan(runner, f.session);
assert.equal(job.status, 'unverified'); assert.equal(job.results[0].exitCode, 0); assert.equal(job.results[0].tests?.total ?? null, variant.expectedTotal);
assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'unverified');
}
});
test('a successful build with no selected tests does not verify the task', async t => {
const f = fixture(t); nodeProject(f); const { job } = await runPlan(runnerFor(f), f.session, ['build']);
assert.equal(job.results[0].status, 'passed'); assert.equal(job.status, 'unverified');
});
test('stale check plans and forged script names execute no project commands', async t => {
const f = fixture(t); nodeProject(f); const runner = runnerFor(f), plan = await runner.plan(f.session);
write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;\n');
const { promise } = await runner.start(f.session, plan.id), job = await promise;
assert.equal(job.status, 'stale'); assert.equal(job.results.length, 0); assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist')), false);
await assert.rejects(runner.plan(f.session, ['test; echo invalid']), /已识别/);
const other = f.store.createSession(f.project.id); await assert.rejects(runner.start(other, plan.id), { status: 404 });
});
test('applying a later proposal invalidates the prior passed engineering snapshot', async t => {
const f = fixture(t); nodeProject(f); const runner = runnerFor(f), { job } = await runPlan(runner, f.session);
const proposal = proposeFile(f.store, f.session, { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b + 1;\n', reason: 'Controlled change after verification' });
await assert.rejects(runner.start(f.session, (await runner.plan(f.session)).id), /待审查/);
applyChange(f.store, f.session, proposal.change_id);
assert.equal(job.stale, true); assert.equal(f.store.executionSnapshot(f.session).task.verification.status, 'stale');
});
test('stopping an actual check terminates its child process and blocks concurrent jobs', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('long running check', async () => { fs.writeFileSync('.awb-check-entered', 'entered'); await new Promise(resolve => setTimeout(resolve, 30000)); });\n", build: false });
const runner = runnerFor(f), plan = await runner.plan(f.session), started = await runner.start(f.session, plan.id);
await until(() => fs.existsSync(path.join(f.projectRoot, '.awb-check-entered')), 10000);
await assert.rejects(runner.start(f.session, plan.id), { status: 409 });
await runner.stop(f.session.id, started.job.id);
const job = await started.promise; assert.equal(job.status, 'cancelled'); assert.equal(job.results[0].status, 'cancelled'); assert.equal(runner.active.size, 0);
});
test('check timeout records a failure with its actual termination result', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; test('timeout', async () => { await new Promise(resolve => setTimeout(resolve, 30000)); });", build: false });
const { job } = await runPlan(runnerFor(f, { timeoutMs: 400 }), f.session);
assert.equal(job.status, 'failed'); assert.equal(job.results[0].timedOut, true); assert.match(job.results[0].reason, /时间上限/);
});
test('source edits during a successful check make the result stale', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('check snapshot', async () => { fs.writeFileSync('.awb-check-entered', 'entered'); await new Promise(resolve => setTimeout(resolve, 1200)); });", build: false });
const runner = runnerFor(f), plan = await runner.plan(f.session), started = await runner.start(f.session, plan.id);
await until(() => fs.existsSync(path.join(f.projectRoot, '.awb-check-entered')), 10000); write(f, 'src/add.mjs', 'export const add = (a, b) => a + b + 0;');
const job = await started.promise; assert.equal(job.results[0].status, 'passed'); assert.equal(job.status, 'stale'); assert.equal(job.stale, true);
});
test('bounded large logs retain the test footer and readable artifact instead of inventing counts', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; test('large output', () => { console.log('中'.repeat(600000)); });", build: false });
const { job } = await runPlan(runnerFor(f), f.session);
assert.equal(job.status, 'passed'); assert.equal(job.results[0].truncated, true); assert.equal(job.results[0].tests.total, 1);
const artifact = f.store.owned('artifact', job.results[0].logArtifactId, f.session.id), text = f.store.artifactStore.read(artifact);
assert.match(text, /中间日志超出保存上限/); assert.ok(artifact.bytes < 2 * 1024 * 1024); assert.match(text, /tests 1/);
});
test('model-selected checks require a run-bound decision and denial creates no check job', async t => {
const f = fixture(t); nodeProject(f); const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let round = 0;
provider.complete = async (_config, messages) => {
const planResult = messages.filter(message => message.role === 'tool').map(message => { try { return JSON.parse(message.content); } catch { return {}; } }).find(value => value.checks && value.fingerprint);
const calls = round++ === 0 ? [call('plan-for-model', 'plan_checks', {})] : round === 2 ? [call('run-for-model', 'run_checks', { plan_id: planResult.id })] : [];
return { message: { role: 'assistant', content: calls.length ? '' : 'Checks were declined; task has no engineering pass.', tool_calls: calls }, usage: { prompt_tokens: 10, completion_tokens: 5 } };
};
runtime.start(f.session.id, 'Validate this engineering project'); const promise = runtime.active.get(f.session.id).promise;
await until(() => f.session.approvals.some(value => value.status === 'pending'), 8000);
assert.equal(f.store.core.check_job.size, 0); const approval = f.session.approvals.at(-1);
assert.equal(approval.runId, f.session.runId); runtime.decide(f.session.id, approval.id, false); await promise;
assert.equal(f.store.core.check_job.size, 0); assert.equal(f.session.events.find(event => event.tool === 'run_checks').phase, 'denied'); assertToolPairs(f.session.providerMessages);
});
test('check APIs expose owned plans and jobs and preserve the authoritative results across restart', async t => {
const f = fixture(t); nodeProject(f); f.store.close(); const app = await launchWorkbench(t, f.data);
const planned = await app.api('/sessions/' + f.session.id + '/check-plans', {}), plan = planned.execution.plans.at(-1);
const accepted = await app.api('/sessions/' + f.session.id + '/check-jobs', { planId: plan.id }); const jobId = accepted.execution.checks.at(-1).id;
await until(async () => !['queued', 'running'].includes((await app.api('/sessions/' + f.session.id + '/check-jobs/' + jobId)).status), 12000);
const finished = await app.api('/sessions/' + f.session.id + '/check-jobs/' + jobId); assert.equal(finished.status, 'passed');
const other = await app.api('/sessions', { projectId: f.project.id }); await assert.rejects(app.api('/sessions/' + other.id + '/check-jobs/' + jobId), { status: 404 });
const log = await app.api('/artifacts/' + finished.results[0].logArtifactId + '?session=' + f.session.id); assert.match(log.content, /tests 2/);
const exported = await app.api('/sessions/' + f.session.id + '/export'); assert.equal(exported.execution.checks.at(-1).id, jobId);
await app.close(); const resumed = await launchWorkbench(t, f.data); assert.equal((await resumed.api('/sessions/' + f.session.id)).execution.task.verification.status, 'passed');
});
test('reviewed changes continue the original task without reopening a generic completed run', async t => {
const f = fixture(t); nodeProject(f); const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let round = 0;
provider.complete = async () => ({ message: { role: 'assistant', content: round++ === 0 ? '' : 'Reviewed changes need actual verification.',
...(round === 1 ? { tool_calls: [call('reviewed-proposal', 'propose_file', { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b + 0;\n', reason: 'Verify the reviewed-task continuation' })] } : {}) }, usage: { prompt_tokens: 5, completion_tokens: 5 } });
runtime.start(f.session.id, 'Keep this original engineering goal'); await runtime.active.get(f.session.id).promise;
assert.equal(f.session.status, 'needs_review'); const taskId = f.session.taskId, firstRun = f.session.runId;
assert.throws(() => runtime.continueReviewed(f.session.id), { status: 409 });
applyChange(f.store, f.session, f.session.changes.at(-1).id); assert.equal(f.session.status, 'completed');
assert.throws(() => runtime.resume(f.session.id), { status: 409 });
const before = JSON.stringify(f.store.publicSession(f.session)), config = f.store.config;
f.store.config = { ...config, mode: 'compatible', model: '' };
assert.throws(() => runtime.continueReviewed(f.session.id), /模型名称/);
assert.equal(JSON.stringify(f.store.publicSession(f.session)), before, 'rejected continuation must not append an instruction or run');
f.store.config = config;
runtime.continueReviewed(f.session.id); assert.throws(() => runtime.continueReviewed(f.session.id), { status: 409 });
await runtime.active.get(f.session.id).promise;
assert.equal(f.session.taskId, taskId); assert.equal(f.store.core.task.get(taskId).goal, 'Keep this original engineering goal');
assert.equal(f.store.core.run.get(f.session.runId).resumeOf, firstRun); assert.equal(f.store.core.task.get(taskId).instructions.length, 0);
assert.equal(f.store.core.run.get(f.session.runId).source, 'review-action');
assert.throws(() => runtime.continueReviewed(f.session.id), { status: 409 }); assertToolPairs(f.session.providerMessages);
});
test('concurrent review-continuation API requests accept one run and retain the goal after restart', async t => {
const f = fixture(t); nodeProject(f);
let requests = 0;
const model = await compatibleFixture(t, async () => {
if (requests++ === 0) return completion([call('api-review-proposal', 'propose_file', { path: 'src/add.mjs', content: 'export const add = (a, b) => a + b + 0;\n', reason: 'A reviewable API fixture change' })]);
await new Promise(resolve => setTimeout(resolve, 120));
return completion([], 'Continue the same delivery goal with the saved results.');
});
const app = await launchWorkbench(t, f.data);
await app.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'review-fixture' });
await app.api('/sessions/' + f.session.id + '/messages', { content: 'Retain this delivery goal' });
let reviewed; await until(async () => { reviewed = await app.api('/sessions/' + f.session.id); return reviewed.status === 'needs_review'; });
await app.api('/sessions/' + f.session.id + '/changes/' + reviewed.changes.at(-1).id + '/apply', {});
const responses = await Promise.allSettled([app.api('/sessions/' + f.session.id + '/continue-reviewed', {}), app.api('/sessions/' + f.session.id + '/continue-reviewed', {})]);
assert.equal(responses.filter(value => value.status === 'fulfilled').length, 1);
assert.equal(responses.find(value => value.status === 'rejected').reason.status, 409);
let delivered; await until(async () => { delivered = await app.api('/sessions/' + f.session.id); return delivered.status === 'completed'; });
assert.equal(delivered.execution.task.id, reviewed.execution.task.id); assert.equal(delivered.execution.task.goal, 'Retain this delivery goal');
assert.equal(delivered.execution.run.resumeOf, reviewed.execution.run.id);
assert.equal(delivered.execution.task.instructions.filter(value => value.source === 'review-action').length, 0);
assert.equal(delivered.execution.run.source, 'review-action');
await app.close(); const restarted = await launchWorkbench(t, f.data);
const saved = await restarted.api('/sessions/' + f.session.id);
assert.equal(saved.execution.task.id, delivered.execution.task.id); assert.equal(saved.execution.run.id, delivered.execution.run.id);
assert.equal(saved.execution.task.instructions.filter(value => value.source === 'review-action').length, 0);
assert.equal(saved.execution.run.source, 'review-action');
assertToolPairs((await restarted.api('/sessions/' + f.session.id + '/export')).session.providerMessages);
});
test('the model reads owned failure logs without running the check again', async t => {
const f = fixture(t); nodeProject(f, { source: 'export const add = (a, b) => a - b;\n' });
const { job } = await runPlan(runnerFor(f), f.session); const artifactId = job.results[0].logArtifactId;
assert.match(job.results[0].logExcerpt, /AssertionError|ERR_ASSERTION/);
const provider = new Provider(), runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
let round = 0; provider.complete = async () => ({ message: round++ === 0 ? { role: 'assistant', content: '', tool_calls: [call('read-check-log', 'read_artifact', { artifact_id: artifactId, offset: 0, limit: 16000 })] } : { role: 'assistant', content: 'Failure log read without replay.' }, usage: { prompt_tokens: 5, completion_tokens: 5 } });
runtime.start(f.session.id, 'Explain the saved check failure'); await runtime.active.get(f.session.id).promise;
assert.equal(f.store.core.check_job.size, 1); const result = f.store.savedToolResult(f.session.id, 'read-check-log'); assert.match(result.content, /AssertionError|ERR_ASSERTION/); assert.equal(result.artifact_id, artifactId);
const other = f.store.createSession(f.project.id); assert.throws(() => f.store.owned('artifact', artifactId, other.id), { status: 404 });
});
for (const phase of ['queued', 'result_committed']) test('process exit at check ' + phase + ' preserves results and never replays commands', async t => {
const f = fixture(t); nodeProject(f, { testCode: "import test from 'node:test'; import fs from 'node:fs'; test('one side effect', () => fs.writeFileSync('.awb-effect', '1'));" }); f.store.close();
const worker = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/check-crash-worker.mjs'), f.data, f.session.id, phase], { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let error = ''; worker.stderr.on('data', bytes => { error += bytes.toString(); }); const code = await new Promise(resolve => worker.once('exit', resolve)); assert.equal(code, 69, error);
const recovered = new Store(f.data); f.onCleanup(() => recovered.close()); const session = recovered.session(f.session.id), job = recovered.executionSnapshot(session).checks.at(-1);
assert.equal(job.status, phase === 'queued' ? 'interrupted' : 'unknown'); assert.equal(session.execution, undefined);
if (phase === 'result_committed') { assert.equal(job.results.length, 1); assert.equal(job.results[0].tests.total, 1); assert.equal(fs.readFileSync(path.join(f.projectRoot, '.awb-effect'), 'utf8'), '1'); }
else assert.equal(fs.existsSync(path.join(f.projectRoot, '.awb-effect')), false);
assert.equal(fs.existsSync(path.join(f.projectRoot, 'dist')), false); assert.equal(recovered.executionSnapshot(session).task.verification.status, job.status);
});