214 lines
18 KiB
JavaScript
214 lines
18 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { CheckRunner } from '../src/core/check-runner.mjs';
|
|
import { inspectProject, createCheckList } from '../src/core/project-inspector.mjs';
|
|
import { resolvePythonRuntime, verifyPythonCheckEvidence } from '../src/core/python-runtime.mjs';
|
|
import { proposeFile, applyChange } from '../src/server/files.mjs';
|
|
import { DeliveryReporter } from '../src/core/delivery-report.mjs';
|
|
import { fixture, until } from './helpers.mjs';
|
|
|
|
const write = (f, name, text) => { const target = path.join(f.projectRoot, name); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, text); };
|
|
function pythonProject(f, testCode, framework = 'unittest') {
|
|
write(f, 'pyproject.toml', '[project]\nname = "owned-python-checks"\nversion = "0.1.0"\n\n[tool.agent-workbench.python]\nframework = "' + framework + '"\ntest-paths = ["tests"]\n');
|
|
write(f, 'tests/__init__.py', ''); write(f, 'tests/test_actual.py', testCode);
|
|
const run = f.store.beginRun(f.session, 'Repair this Python project, preserve tests and report actual checks.', false, f.store.config);
|
|
f.session.status = 'completed'; f.store.finishRun(f.session, run);
|
|
}
|
|
async function runnerFor(f, options = {}) {
|
|
const runtime = await resolvePythonRuntime(f.project, { executable: process.env.AGENT_WORKBENCH_TEST_PYTHON });
|
|
assert.equal(runtime.status, 'available', 'Regression checks require an installed Python 3.10+ interpreter.');
|
|
const runner = new CheckRunner(f.store, { pythonExecutable: runtime.logicalExecutable, ...options });
|
|
f.onCleanup(() => runner.stopAll()); return runner;
|
|
}
|
|
async function run(runner, f, names) {
|
|
const plan = await runner.plan(f.session, names), { promise } = await runner.start(f.session, plan.id);
|
|
return { plan, job: await promise };
|
|
}
|
|
const onePass = 'import unittest\nclass Actual(unittest.TestCase):\n def test_actual(self): self.assertEqual(2 + 3, 5)\n';
|
|
|
|
test('Python static inspection does not import project code or probe a project interpreter before approval', async t => {
|
|
const f = fixture(t); pythonProject(f, onePass);
|
|
write(f, 'sitecustomize.py', 'from pathlib import Path\nPath("unexpected-import").write_text("ran")\n');
|
|
const host = await resolvePythonRuntime(f.project, { executable: process.env.AGENT_WORKBENCH_TEST_PYTHON }); assert.equal(host.status, 'available');
|
|
const copied = path.join(f.projectRoot, '.venv/Scripts/python.exe'); fs.mkdirSync(path.dirname(copied), { recursive: true }); fs.copyFileSync(host.logicalExecutable, copied);
|
|
const runner = new CheckRunner(f.store); f.onCleanup(() => runner.stopAll());
|
|
const inspection = await inspectProject(f.project), plan = await runner.plan(f.session);
|
|
assert.equal(inspection.python.framework, 'unittest'); assert.deepEqual(plan.checks.map(check => check.name), ['python:test', 'python:compile']);
|
|
assert.ok(plan.checks.every(check => check.python.runtime.status === 'deferred' && check.python.runtime.version === null));
|
|
assert.equal(fs.existsSync(path.join(f.projectRoot, 'unexpected-import')), false);
|
|
assert.ok(inspection.scope.complete); assert.ok(inspection.scope.excludedFiles >= 0);
|
|
});
|
|
|
|
test('a real three-module Python failure, reviewed repairs, tests and compilation retain exact structured evidence', async t => {
|
|
const f = fixture(t);
|
|
pythonProject(f, 'import unittest\nfrom workflow.normalize import normalize\nfrom workflow.summary import summarize\nfrom workflow.render import render\n'
|
|
+ 'class Workflow(unittest.TestCase):\n def test_number(self): self.assertEqual(normalize("3"), 3)\n'
|
|
+ ' def test_negative(self): self.assertEqual(normalize(-2), 0)\n def test_empty(self): self.assertEqual(summarize([]), {"count": 0, "total": 0})\n'
|
|
+ ' def test_summary(self): self.assertEqual(summarize([2, 3]), {"count": 2, "total": 5})\n'
|
|
+ ' def test_render(self): self.assertEqual(render({"count": 2, "total": 5}), "共 2 项 · 合计 5")\n'
|
|
+ ' def test_zero(self): self.assertEqual(render({"count": 0, "total": 0}), "共 0 项 · 合计 0")\n');
|
|
write(f, 'src/workflow/__init__.py', ''); write(f, 'src/workflow/normalize.py', 'def normalize(value):\n return -float(value)\n');
|
|
write(f, 'src/workflow/summary.py', 'def summarize(values):\n return {"count": len(values), "total": 0}\n');
|
|
write(f, 'src/workflow/render.py', 'def render(value):\n return ""\n');
|
|
const testBytes = fs.readFileSync(path.join(f.projectRoot, 'tests/test_actual.py')), declarationBytes = fs.readFileSync(path.join(f.projectRoot, 'pyproject.toml'));
|
|
const runner = await runnerFor(f), first = await run(runner, f);
|
|
assert.equal(first.job.status, 'failed'); assert.equal(first.job.results[0].tests.failed, 5); assert.equal(first.job.results[0].tests.total, 6);
|
|
const fixes = { 'src/workflow/normalize.py': 'def normalize(value):\n return max(0, float(value))\n',
|
|
'src/workflow/summary.py': 'def summarize(values):\n return {"count": len(values), "total": sum(values)}\n',
|
|
'src/workflow/render.py': 'def render(value):\n return f"共 {value[\'count\']} 项 · 合计 {value[\'total\']:g}"\n' };
|
|
for (const [file, content] of Object.entries(fixes)) { const change = proposeFile(f.store, f.session, { path: file, content, reason: 'Repair an actual sealed failure' }); applyChange(f.store, f.session, change.change_id); }
|
|
assert.equal(first.job.status, 'failed'); assert.notEqual(first.job.sourceFingerprint, (await inspectProject(f.project)).sourceFingerprint);
|
|
const { plan, job } = await run(runner, f);
|
|
assert.equal(job.status, 'passed'); assert.deepEqual(job.results[0].tests, { total: 6, passed: 6, failed: 0, skipped: 0, cancelled: 0 });
|
|
assert.equal(job.results[1].compilation.compiled, 6); assert.equal(job.results[1].compilation.failures.length, 0);
|
|
for (const result of job.results) {
|
|
assert.equal(result.python.runtime.version.split('.')[0], '3'); assert.equal(result.evidenceValidated, true);
|
|
assert.match(result.metricsSha256, /^[a-f0-9]{64}$/); verifyPythonCheckEvidence(f.store, f.session, job, result);
|
|
}
|
|
assert.equal(plan.checks[0].python.runtime.status, 'available');
|
|
assert.ok(fs.readFileSync(path.join(f.projectRoot, 'tests/test_actual.py')).equals(testBytes));
|
|
assert.ok(fs.readFileSync(path.join(f.projectRoot, 'pyproject.toml')).equals(declarationBytes));
|
|
assert.equal(fs.existsSync(path.join(f.projectRoot, 'src/workflow/__pycache__')), false);
|
|
const report = await new DeliveryReporter(f.store).create(f.session);
|
|
assert.equal(report.report.status, 'verified_uncommitted'); assert.equal(report.report.checks.at(-1).results[0].tests.passed, 6);
|
|
job.results[0].tests.passed = 999;
|
|
const tampered = await new DeliveryReporter(f.store).create(f.session);
|
|
assert.equal(tampered.report.status, 'needs_verification');
|
|
});
|
|
|
|
test('unittest counts repeated execution ordinals and subtest skips rather than deduplicating display names', async t => {
|
|
const f = fixture(t);
|
|
pythonProject(f, 'import unittest\nclass Same(unittest.TestCase):\n def test_wrong(self): self.assertEqual(1, 2)\n'
|
|
+ 'def load_tests(loader, suite, pattern):\n repeated = Same("test_wrong")\n return unittest.TestSuite([repeated, repeated])\n');
|
|
const runner = await runnerFor(f), failed = await run(runner, f, ['python:test']);
|
|
assert.deepEqual(failed.job.results[0].tests, { total: 2, passed: 0, failed: 2, skipped: 0, cancelled: 0 });
|
|
write(f, 'tests/test_actual.py', 'import unittest\nclass Sub(unittest.TestCase):\n def test_sub(self):\n with self.subTest(part=1): self.skipTest("not exercised")\n'
|
|
+ ' def test_pass(self): self.assertEqual(1, 1)\n');
|
|
const skipped = await run(runner, f, ['python:test']);
|
|
assert.equal(skipped.job.status, 'unverified'); assert.deepEqual(skipped.job.results[0].tests, { total: 2, passed: 1, failed: 0, skipped: 1, cancelled: 0 });
|
|
});
|
|
|
|
test('unittest empty, skipped, expected failure and class initialization errors never verify a task', async t => {
|
|
const variants = [
|
|
{ code: '', status: 'unverified', total: 0 },
|
|
{ code: 'import unittest\nclass Skip(unittest.TestCase):\n @unittest.skip("not run")\n def test_skip(self): pass\n', status: 'unverified', total: 1 },
|
|
{ code: 'import unittest\nclass Expected(unittest.TestCase):\n @unittest.expectedFailure\n def test_expected(self): self.fail("expected")\n', status: 'unverified', total: 1 },
|
|
{ code: 'import unittest\nclass Broken(unittest.TestCase):\n @classmethod\n def setUpClass(cls): raise RuntimeError("actual class initialization error")\n def test_never(self): pass\n', status: 'failed', total: 0 },
|
|
];
|
|
for (const variant of variants) {
|
|
const f = fixture(t); pythonProject(f, variant.code); const { job } = await run(await runnerFor(f), f, ['python:test']);
|
|
assert.equal(job.status, variant.status); assert.equal(job.results[0].tests.total, variant.total);
|
|
if (variant.status === 'failed') assert.equal(job.results[0].collectionErrors, 1);
|
|
}
|
|
});
|
|
|
|
test('stdout claims cannot replace actual unittest results and an early exit cannot forge a successful collector', async t => {
|
|
const f = fixture(t); pythonProject(f, 'import unittest\nclass Actual(unittest.TestCase):\n def test_real(self):\n print("Tests: 999 passed, 999 total")\n self.assertEqual(1, 2)\n');
|
|
const runner = await runnerFor(f), failed = await run(runner, f, ['python:test']);
|
|
assert.equal(failed.job.status, 'failed'); assert.equal(failed.job.results[0].tests.failed, 1);
|
|
write(f, 'tests/test_actual.py', 'import unittest, os\nclass Exit(unittest.TestCase):\n def test_exit(self):\n print("Tests: 999 passed, 999 total", flush=True)\n os._exit(0)\n');
|
|
const exited = await run(runner, f, ['python:test']);
|
|
assert.equal(exited.job.results[0].exitCode, 0); assert.equal(exited.job.status, 'failed');
|
|
assert.equal(exited.job.results[0].tests, null); assert.match(exited.job.results[0].reason, /结构化结果/);
|
|
});
|
|
|
|
test('compilation reads actual Python source bytes without executing top-level project code', async t => {
|
|
const f = fixture(t); pythonProject(f, onePass);
|
|
write(f, 'src/top_level.py', 'from pathlib import Path\nPath("unexpected-execution").write_text("should never execute")\nraise RuntimeError("compile only")\n');
|
|
const runner = await runnerFor(f), { job } = await run(runner, f, ['python:compile']);
|
|
assert.equal(job.results[0].status, 'passed'); assert.equal(job.status, 'unverified'); assert.equal(fs.existsSync(path.join(f.projectRoot, 'unexpected-execution')), false);
|
|
write(f, 'src/top_level.py', 'def broken(:\n pass\n');
|
|
const broken = await run(runner, f, ['python:compile']);
|
|
assert.equal(broken.job.status, 'failed'); assert.equal(broken.job.results[0].compilation.failures[0].path, 'src/top_level.py');
|
|
});
|
|
|
|
test('Node and Python mixed projects retain the existing npm default and require an explicit Python selection', async t => {
|
|
const f = fixture(t); pythonProject(f, onePass);
|
|
write(f, 'package.json', JSON.stringify({ scripts: { test: 'node --test tests/node.test.mjs', build: 'node build.mjs' } }));
|
|
write(f, 'tests/node.test.mjs', "import test from 'node:test'; test('actual Node case', () => {});\n"); write(f, 'build.mjs', '');
|
|
const inspection = await inspectProject(f.project);
|
|
assert.deepEqual(createCheckList(inspection).map(check => check.name), ['test', 'build']);
|
|
assert.deepEqual(createCheckList(inspection, ['python:test', 'test']).map(check => check.name), ['python:test', 'test']);
|
|
const { job } = await run(await runnerFor(f), f, ['python:test', 'test']);
|
|
assert.equal(job.status, 'passed'); assert.equal(job.results[0].engine, 'python'); assert.equal(job.results[1].tests.passed, 1);
|
|
});
|
|
|
|
test('changed project interpreters and forged Python plan bindings execute no check', async t => {
|
|
const f = fixture(t); pythonProject(f, onePass);
|
|
const host = await resolvePythonRuntime(f.project, { executable: process.env.AGENT_WORKBENCH_TEST_PYTHON }); assert.equal(host.status, 'available');
|
|
const copied = path.join(f.projectRoot, '.venv/Scripts/python.exe'); fs.mkdirSync(path.dirname(copied), { recursive: true }); fs.copyFileSync(host.logicalExecutable, copied);
|
|
const runner = new CheckRunner(f.store); f.onCleanup(() => runner.stopAll());
|
|
const plan = await runner.plan(f.session, ['python:test']);
|
|
fs.appendFileSync(copied, 'changed after approval plan');
|
|
const { promise } = await runner.start(f.session, plan.id), job = await promise;
|
|
assert.equal(job.status, 'stale', job.results[0]?.reason); assert.match(job.results[0].reason, /未执行探测或检查/);
|
|
const forged = await runner.plan(f.session, ['python:test']); forged.checks[0].python.runtime.sha256 = '0'.repeat(64);
|
|
await assert.rejects(runner.start(f.session, forged.id), /封存/);
|
|
});
|
|
|
|
test('malformed TOML and invalid Python test paths are reported without expanding npm check scope', async t => {
|
|
const f = fixture(t); pythonProject(f, onePass); const runner = await runnerFor(f);
|
|
write(f, 'pyproject.toml', '[tool.agent-workbench.python\n');
|
|
await assert.rejects(runner.plan(f.session), /声明无效/);
|
|
write(f, 'package.json', JSON.stringify({ scripts: { test: 'node --test' } }));
|
|
const plan = await runner.plan(f.session); assert.deepEqual(plan.checks.map(check => check.name), ['test']);
|
|
await assert.rejects(runner.plan(f.session, ['python:test']), /声明无效/);
|
|
write(f, 'pyproject.toml', '[tool.agent-workbench.python]\nframework="pytest"\ntest-paths=["../outside"]\n');
|
|
await assert.rejects(runner.plan(f.session, ['python:test']), /声明无效/);
|
|
});
|
|
|
|
test('stopping a real Python check terminates its child and keeps the completed result as cancelled', { timeout: 20000 }, async t => {
|
|
const f = fixture(t); pythonProject(f, 'import unittest, time\nfrom pathlib import Path\nclass Slow(unittest.TestCase):\n def test_slow(self):\n Path(".awb-started").write_text("started")\n time.sleep(30)\n');
|
|
const runner = await runnerFor(f), plan = await runner.plan(f.session, ['python:test']), started = await runner.start(f.session, plan.id);
|
|
await until(() => fs.existsSync(path.join(f.projectRoot, '.awb-started')), 5000);
|
|
await runner.stop(f.session.id, started.job.id); const job = await started.promise;
|
|
assert.equal(job.status, 'cancelled'); assert.equal(job.results[0].status, 'cancelled'); assert.equal(runner.isActive(f.session.id), false);
|
|
});
|
|
|
|
test('real pytest pass, failure, skip, collection errors and teardown errors use collected case outcomes', { timeout: 60000 }, async t => {
|
|
const variants = [
|
|
{ code: 'def test_actual(): assert 2 + 3 == 5\n', status: 'passed', passed: 1, failed: 0 },
|
|
{ code: '', status: 'failed', passed: 0, failed: 0 },
|
|
{ code: 'def test_actual():\n print("Tests 999 passed (999)")\n assert 1 == 2\n', status: 'failed', passed: 0, failed: 1 },
|
|
{ code: 'import pytest\n@pytest.mark.skip(reason="not exercised")\ndef test_skip(): pass\n', status: 'unverified', passed: 0, failed: 0 },
|
|
{ code: 'import pytest\n@pytest.mark.xfail(reason="expected failure")\ndef test_expected(): assert 1 == 2\n', status: 'unverified', passed: 0, failed: 0 },
|
|
{ code: 'raise RuntimeError("actual collection error")\n', status: 'failed', collection: 1 },
|
|
{ code: 'import pytest\n@pytest.fixture\ndef resource():\n yield 1\n raise RuntimeError("actual teardown error")\ndef test_actual(resource): assert resource == 1\n', status: 'failed', passed: 0, failed: 1 },
|
|
];
|
|
for (const variant of variants) {
|
|
const f = fixture(t); pythonProject(f, variant.code, 'pytest'); const { job } = await run(await runnerFor(f), f, ['python:test']);
|
|
assert.equal(job.status, variant.status, job.results[0].reason);
|
|
if (variant.collection) assert.equal(job.results[0].collectionErrors, variant.collection);
|
|
else { assert.equal(job.results[0].tests.passed, variant.passed); assert.equal(job.results[0].tests.failed, variant.failed); }
|
|
assert.equal(job.results[0].evidenceValidated, true);
|
|
}
|
|
});
|
|
|
|
test('a real project virtual environment is detected statically and its actual version is recorded only during the approved check', { timeout: 20000 }, async t => {
|
|
const f = fixture(t); pythonProject(f, onePass);
|
|
const host = await resolvePythonRuntime(f.project, { executable: process.env.AGENT_WORKBENCH_TEST_PYTHON });
|
|
// This scenario executes stdlib unittest. It needs a real project venv,
|
|
// not ensurepip's unrelated setuptools fixtures (overlong Windows paths).
|
|
execFileSync(host.baseExecutable, ['-I', '-X', 'utf8', '-m', 'venv', '--without-pip', path.join(f.projectRoot, '.venv')], { windowsHide: true, cwd: f.root, stdio: 'pipe' });
|
|
const runner = new CheckRunner(f.store); f.onCleanup(() => runner.stopAll());
|
|
const plan = await runner.plan(f.session);
|
|
assert.equal(plan.checks[0].python.runtime.status, 'deferred'); assert.equal(plan.checks[0].python.runtime.version, null);
|
|
const { promise } = await runner.start(f.session, plan.id), job = await promise;
|
|
assert.equal(job.status, 'passed', job.results[0]?.reason); assert.equal(job.results[0].python.runtime.version, host.version);
|
|
assert.ok(job.results[0].python.runtime.executable.startsWith(f.projectRoot));
|
|
});
|
|
|
|
test('a missing interpreter and repeated pytest item objects cannot create a false verification pass', { timeout: 20000 }, async t => {
|
|
const f = fixture(t); pythonProject(f, 'def test_actual(): assert 2 + 3 == 5\n', 'pytest');
|
|
const missing = await runnerFor(f, { pythonExecutable: path.join(f.root, 'missing-python.exe') });
|
|
const failed = await run(missing, f, ['python:test']);
|
|
assert.equal(failed.job.status, 'failed'); assert.equal(failed.job.results[0].exitCode, null);
|
|
write(f, 'tests/conftest.py', 'def pytest_collection_modifyitems(items):\n items.append(items[0])\n');
|
|
const actual = await run(await runnerFor(f), f, ['python:test']);
|
|
assert.equal(actual.job.status, 'passed', actual.job.results[0]?.reason);
|
|
assert.deepEqual(actual.job.results[0].tests, { total: 2, passed: 2, failed: 0, skipped: 0, cancelled: 0 });
|
|
});
|