"""Collect actual Workbench Python checks after their execution is approved.""" import hashlib import json import os from pathlib import Path import sys import traceback sys.dont_write_bytecode = True def sha(data): return hashlib.sha256(data).hexdigest() def checked_path(root, relative): if not isinstance(relative, str) or not relative or "\0" in relative: raise ValueError("Invalid planned Python path") candidate = (root / relative).resolve(strict=True) if not candidate.is_relative_to(root): raise ValueError("Python input is outside the sealed project") return candidate def verify_sources(root, files): for item in files: filename = checked_path(root, item["path"]) if not filename.is_file() or sha(filename.read_bytes()) != item["sha256"]: raise ValueError("Planned Python source changed: " + item["path"]) def compile_sources(root, files): failures = [] for item in files: filename = checked_path(root, item["path"]) data = filename.read_bytes() try: compile(data, str(filename), "exec", dont_inherit=True) except (SyntaxError, ValueError) as error: failures.append({"path": item["path"], "message": str(error), "line": getattr(error, "lineno", None)}) traceback.print_exception(type(error), error, error.__traceback__) print("Python source compilation: %d / %d" % (len(files) - len(failures), len(files))) return {"compilation": {"compiled": len(files) - len(failures), "total": len(files), "failures": failures}}, 1 if failures else 0 def count_records(records): states = [item["status"] for item in records] return {"total": len(states), "passed": states.count("passed"), "failed": states.count("failed"), "skipped": states.count("skipped"), "cancelled": states.count("pending")} def unittest_sources(root, settings): import unittest class CollectedResult(unittest.TextTestResult): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.records = [] self.active = {} self.collection_errors = 0 self.collection_skipped = 0 def startTest(self, test): super().startTest(test) # Test.id() is a display name. Repeated cases and subtests belong # to their actual startTest execution ordinal, not that name. self.active[id(test)] = len(self.records) self.records.append({"status": "pending"}) def stopTest(self, test): self.active.pop(id(test), None) super().stopTest(test) def mark(self, test, status): index = self.active.get(id(test)) if index is None: index = self.active.get(id(getattr(test, "test_case", None))) if index is None: if status == "failed": self.collection_errors += 1 elif status == "skipped": self.collection_skipped += 1 return record = self.records[index] if record["status"] != "failed" and (record["status"] != "skipped" or status == "failed"): record["status"] = status def addSuccess(self, test): self.mark(test, "passed") super().addSuccess(test) def addFailure(self, test, error): self.mark(test, "failed") super().addFailure(test, error) def addError(self, test, error): self.mark(test, "failed") super().addError(test, error) def addSkip(self, test, reason): self.mark(test, "skipped") super().addSkip(test, reason) def addExpectedFailure(self, test, error): self.mark(test, "skipped") super().addExpectedFailure(test, error) def addUnexpectedSuccess(self, test): self.mark(test, "failed") super().addUnexpectedSuccess(test) def addSubTest(self, test, subtest, error): if error is not None: self.mark(test, "failed") super().addSubTest(test, subtest, error) start = checked_path(root, settings["startDirectory"]) if not start.is_dir(): raise ValueError("Unittest discovery needs a project directory") sys.path[:0] = [str(root), str(root / "src")] loader = unittest.TestLoader() suite = loader.discover(str(start), pattern=settings["pattern"]) result = unittest.TextTestRunner(verbosity=2, resultclass=CollectedResult).run(suite) if result.testsRun != len(result.records): raise ValueError("Unittest execution count was not collected consistently") return {"tests": count_records(result.records), "framework": "unittest", "frameworkVersion": sys.version.split()[0], "collectionErrors": result.collection_errors, "collectionSkipped": result.collection_skipped}, 0 if result.wasSuccessful() else 1 def pytest_sources(root, settings): # Import the host framework before adding project imports. Autoloaded # environment plugins and addopts are excluded by the host check contract. import pytest sys.path[:0] = [str(root), str(root / "src")] class Collector: def __init__(self): self.records = [] self.ordinals = {} self.active = None self.collection_errors = 0 self.collection_skipped = 0 def pytest_collection_finish(self, session): self.records = [{"status": "pending", "nodeid": item.nodeid} for item in session.items] self.ordinals = {} for index, item in enumerate(session.items): self.ordinals.setdefault(id(item), []).append(index) def pytest_collectreport(self, report): if report.failed: self.collection_errors += 1 elif report.skipped: self.collection_skipped += 1 @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_runtest_protocol(self, item, nextitem): pending = self.ordinals.get(id(item), []) self.active = pending.pop(0) if pending else None try: yield finally: self.active = None def pytest_runtest_logreport(self, report): if self.active is None or self.records[self.active]["nodeid"] != report.nodeid: self.collection_errors += 1 return record = self.records[self.active] if report.failed: record["status"] = "failed" elif record["status"] != "failed" and (report.skipped or hasattr(report, "wasxfail")): record["status"] = "skipped" elif record["status"] == "pending" and report.when == "call" and report.passed: record["status"] = "passed" collector = Collector() args = [str(checked_path(root, relative)) for relative in settings["testPaths"]] args += ["-p", "no:cacheprovider", "--import-mode=importlib", "-o", "addopts="] if settings.get("configPath"): args += ["-c", str(checked_path(root, settings["configPath"]))] code = int(pytest.main(args, plugins=[collector])) return {"tests": count_records(collector.records), "framework": "pytest", "frameworkVersion": pytest.__version__, "collectionErrors": collector.collection_errors, "collectionSkipped": collector.collection_skipped}, code def main(): payload_bytes = sys.stdin.buffer.read() payload = json.loads(payload_bytes) result_path = Path(sys.argv[1]).resolve() root = Path(payload["root"]).resolve(strict=True) if not root.is_dir() or result_path.is_relative_to(root): raise ValueError("Result collection must use the owned host directory") executable = Path(os.path.realpath(sys.executable)) base_executable = Path(os.path.realpath(getattr(sys, "_base_executable", sys.executable))) identity = {"schemaVersion": 1, "kind": "python-check-result", "invocationId": payload["invocationId"], "inputSha256": sha(payload_bytes), "mode": payload["mode"], "runnerSha256": sha(Path(__file__).read_bytes()), "runtime": {"executable": str(executable), "version": sys.version.split()[0], "sha256": sha(executable.read_bytes()), "baseExecutable": str(base_executable), "baseSha256": sha(base_executable.read_bytes())}} try: verify_sources(root, payload["files"]) if payload["mode"] == "compile": evidence, code = compile_sources(root, payload["files"]) elif payload["mode"] == "unittest": evidence, code = unittest_sources(root, payload["settings"]) elif payload["mode"] == "pytest": evidence, code = pytest_sources(root, payload["settings"]) else: raise ValueError("Unknown built-in Python check") output = {**identity, **evidence, "exitCode": code} except BaseException as error: traceback.print_exception(type(error), error, error.__traceback__) output = {**identity, "exitCode": 1, "collectorError": type(error).__name__ + ": " + str(error)} code = 1 temporary = result_path.with_suffix(".tmp") with temporary.open("x", encoding="utf-8") as stream: json.dump(output, stream, ensure_ascii=False) temporary.replace(result_path) return code if __name__ == "__main__": raise SystemExit(main())