Files
Agent-Workbench/scripts/python-check-runner.py

224 lines
9.3 KiB
Python

"""Collect actual Workbench Python checks after their execution is approved."""
import hashlib
import json
import os
from pathlib import Path
import sys
import traceback
sys.dont_write_bytecode = True
def sha(data):
return hashlib.sha256(data).hexdigest()
def checked_path(root, relative):
if not isinstance(relative, str) or not relative or "\0" in relative:
raise ValueError("Invalid planned Python path")
candidate = (root / relative).resolve(strict=True)
if not candidate.is_relative_to(root):
raise ValueError("Python input is outside the sealed project")
return candidate
def verify_sources(root, files):
for item in files:
filename = checked_path(root, item["path"])
if not filename.is_file() or sha(filename.read_bytes()) != item["sha256"]:
raise ValueError("Planned Python source changed: " + item["path"])
def compile_sources(root, files):
failures = []
for item in files:
filename = checked_path(root, item["path"])
data = filename.read_bytes()
try:
compile(data, str(filename), "exec", dont_inherit=True)
except (SyntaxError, ValueError) as error:
failures.append({"path": item["path"], "message": str(error), "line": getattr(error, "lineno", None)})
traceback.print_exception(type(error), error, error.__traceback__)
print("Python source compilation: %d / %d" % (len(files) - len(failures), len(files)))
return {"compilation": {"compiled": len(files) - len(failures), "total": len(files), "failures": failures}}, 1 if failures else 0
def count_records(records):
states = [item["status"] for item in records]
return {"total": len(states), "passed": states.count("passed"), "failed": states.count("failed"),
"skipped": states.count("skipped"), "cancelled": states.count("pending")}
def unittest_sources(root, settings):
import unittest
class CollectedResult(unittest.TextTestResult):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.records = []
self.active = {}
self.collection_errors = 0
self.collection_skipped = 0
def startTest(self, test):
super().startTest(test)
# Test.id() is a display name. Repeated cases and subtests belong
# to their actual startTest execution ordinal, not that name.
self.active[id(test)] = len(self.records)
self.records.append({"status": "pending"})
def stopTest(self, test):
self.active.pop(id(test), None)
super().stopTest(test)
def mark(self, test, status):
index = self.active.get(id(test))
if index is None:
index = self.active.get(id(getattr(test, "test_case", None)))
if index is None:
if status == "failed":
self.collection_errors += 1
elif status == "skipped":
self.collection_skipped += 1
return
record = self.records[index]
if record["status"] != "failed" and (record["status"] != "skipped" or status == "failed"):
record["status"] = status
def addSuccess(self, test):
self.mark(test, "passed")
super().addSuccess(test)
def addFailure(self, test, error):
self.mark(test, "failed")
super().addFailure(test, error)
def addError(self, test, error):
self.mark(test, "failed")
super().addError(test, error)
def addSkip(self, test, reason):
self.mark(test, "skipped")
super().addSkip(test, reason)
def addExpectedFailure(self, test, error):
self.mark(test, "skipped")
super().addExpectedFailure(test, error)
def addUnexpectedSuccess(self, test):
self.mark(test, "failed")
super().addUnexpectedSuccess(test)
def addSubTest(self, test, subtest, error):
if error is not None:
self.mark(test, "failed")
super().addSubTest(test, subtest, error)
start = checked_path(root, settings["startDirectory"])
if not start.is_dir():
raise ValueError("Unittest discovery needs a project directory")
sys.path[:0] = [str(root), str(root / "src")]
loader = unittest.TestLoader()
suite = loader.discover(str(start), pattern=settings["pattern"])
result = unittest.TextTestRunner(verbosity=2, resultclass=CollectedResult).run(suite)
if result.testsRun != len(result.records):
raise ValueError("Unittest execution count was not collected consistently")
return {"tests": count_records(result.records), "framework": "unittest", "frameworkVersion": sys.version.split()[0],
"collectionErrors": result.collection_errors, "collectionSkipped": result.collection_skipped}, 0 if result.wasSuccessful() else 1
def pytest_sources(root, settings):
# Import the host framework before adding project imports. Autoloaded
# environment plugins and addopts are excluded by the host check contract.
import pytest
sys.path[:0] = [str(root), str(root / "src")]
class Collector:
def __init__(self):
self.records = []
self.ordinals = {}
self.active = None
self.collection_errors = 0
self.collection_skipped = 0
def pytest_collection_finish(self, session):
self.records = [{"status": "pending", "nodeid": item.nodeid} for item in session.items]
self.ordinals = {}
for index, item in enumerate(session.items):
self.ordinals.setdefault(id(item), []).append(index)
def pytest_collectreport(self, report):
if report.failed:
self.collection_errors += 1
elif report.skipped:
self.collection_skipped += 1
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
def pytest_runtest_protocol(self, item, nextitem):
pending = self.ordinals.get(id(item), [])
self.active = pending.pop(0) if pending else None
try:
yield
finally:
self.active = None
def pytest_runtest_logreport(self, report):
if self.active is None or self.records[self.active]["nodeid"] != report.nodeid:
self.collection_errors += 1
return
record = self.records[self.active]
if report.failed:
record["status"] = "failed"
elif record["status"] != "failed" and (report.skipped or hasattr(report, "wasxfail")):
record["status"] = "skipped"
elif record["status"] == "pending" and report.when == "call" and report.passed:
record["status"] = "passed"
collector = Collector()
args = [str(checked_path(root, relative)) for relative in settings["testPaths"]]
args += ["-p", "no:cacheprovider", "--import-mode=importlib", "-o", "addopts="]
if settings.get("configPath"):
args += ["-c", str(checked_path(root, settings["configPath"]))]
code = int(pytest.main(args, plugins=[collector]))
return {"tests": count_records(collector.records), "framework": "pytest", "frameworkVersion": pytest.__version__,
"collectionErrors": collector.collection_errors, "collectionSkipped": collector.collection_skipped}, code
def main():
payload_bytes = sys.stdin.buffer.read()
payload = json.loads(payload_bytes)
result_path = Path(sys.argv[1]).resolve()
root = Path(payload["root"]).resolve(strict=True)
if not root.is_dir() or result_path.is_relative_to(root):
raise ValueError("Result collection must use the owned host directory")
executable = Path(os.path.realpath(sys.executable))
base_executable = Path(os.path.realpath(getattr(sys, "_base_executable", sys.executable)))
identity = {"schemaVersion": 1, "kind": "python-check-result", "invocationId": payload["invocationId"],
"inputSha256": sha(payload_bytes), "mode": payload["mode"], "runnerSha256": sha(Path(__file__).read_bytes()),
"runtime": {"executable": str(executable), "version": sys.version.split()[0], "sha256": sha(executable.read_bytes()),
"baseExecutable": str(base_executable), "baseSha256": sha(base_executable.read_bytes())}}
try:
verify_sources(root, payload["files"])
if payload["mode"] == "compile":
evidence, code = compile_sources(root, payload["files"])
elif payload["mode"] == "unittest":
evidence, code = unittest_sources(root, payload["settings"])
elif payload["mode"] == "pytest":
evidence, code = pytest_sources(root, payload["settings"])
else:
raise ValueError("Unknown built-in Python check")
output = {**identity, **evidence, "exitCode": code}
except BaseException as error:
traceback.print_exception(type(error), error, error.__traceback__)
output = {**identity, "exitCode": 1, "collectorError": type(error).__name__ + ": " + str(error)}
code = 1
temporary = result_path.with_suffix(".tmp")
with temporary.open("x", encoding="utf-8") as stream:
json.dump(output, stream, ensure_ascii=False)
temporary.replace(result_path)
return code
if __name__ == "__main__":
raise SystemExit(main())