Files

224 lines
9.3 KiB
Python

"""Collect actual Workbench Python checks after their execution is approved."""
import hashlib
import json
import os
from pathlib import Path
import sys
import traceback
sys.dont_write_bytecode = True
def sha(data):
return hashlib.sha256(data).hexdigest()
def checked_path(root, relative):
if not isinstance(relative, str) or not relative or "\0" in relative:
raise ValueError("Invalid planned Python path")
candidate = (root / relative).resolve(strict=True)
if not candidate.is_relative_to(root):
raise ValueError("Python input is outside the sealed project")
return candidate
def verify_sources(root, files):
for item in files:
filename = checked_path(root, item["path"])
if not filename.is_file() or sha(filename.read_bytes()) != item["sha256"]:
raise ValueError("Planned Python source changed: " + item["path"])
def compile_sources(root, files):
failures = []
for item in files:
filename = checked_path(root, item["path"])
data = filename.read_bytes()
try:
compile(data, str(filename), "exec", dont_inherit=True)
except (SyntaxError, ValueError) as error:
failures.append({"path": item["path"], "message": str(error), "line": getattr(error, "lineno", None)})
traceback.print_exception(type(error), error, error.__traceback__)
print("Python source compilation: %d / %d" % (len(files) - len(failures), len(files)))
return {"compilation": {"compiled": len(files) - len(failures), "total": len(files), "failures": failures}}, 1 if failures else 0
def count_records(records):
states = [item["status"] for item in records]
return {"total": len(states), "passed": states.count("passed"), "failed": states.count("failed"),
"skipped": states.count("skipped"), "cancelled": states.count("pending")}
def unittest_sources(root, settings):
import unittest
class CollectedResult(unittest.TextTestResult):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.records = []
self.active = {}
self.collection_errors = 0
self.collection_skipped = 0
def startTest(self, test):
super().startTest(test)
# Test.id() is a display name. Repeated cases and subtests belong
# to their actual startTest execution ordinal, not that name.
self.active[id(test)] = len(self.records)
self.records.append({"status": "pending"})
def stopTest(self, test):
self.active.pop(id(test), None)
super().stopTest(test)
def mark(self, test, status):
index = self.active.get(id(test))
if index is None:
index = self.active.get(id(getattr(test, "test_case", None)))
if index is None:
if status == "failed":
self.collection_errors += 1
elif status == "skipped":
self.collection_skipped += 1
return
record = self.records[index]
if record["status"] != "failed" and (record["status"] != "skipped" or status == "failed"):
record["status"] = status
def addSuccess(self, test):
self.mark(test, "passed")
super().addSuccess(test)
def addFailure(self, test, error):
self.mark(test, "failed")
super().addFailure(test, error)
def addError(self, test, error):
self.mark(test, "failed")
super().addError(test, error)
def addSkip(self, test, reason):
self.mark(test, "skipped")
super().addSkip(test, reason)
def addExpectedFailure(self, test, error):
self.mark(test, "skipped")
super().addExpectedFailure(test, error)
def addUnexpectedSuccess(self, test):
self.mark(test, "failed")
super().addUnexpectedSuccess(test)
def addSubTest(self, test, subtest, error):
if error is not None:
self.mark(test, "failed")
super().addSubTest(test, subtest, error)
start = checked_path(root, settings["startDirectory"])
if not start.is_dir():
raise ValueError("Unittest discovery needs a project directory")
sys.path[:0] = [str(root), str(root / "src")]
loader = unittest.TestLoader()
suite = loader.discover(str(start), pattern=settings["pattern"])
result = unittest.TextTestRunner(verbosity=2, resultclass=CollectedResult).run(suite)
if result.testsRun != len(result.records):
raise ValueError("Unittest execution count was not collected consistently")
return {"tests": count_records(result.records), "framework": "unittest", "frameworkVersion": sys.version.split()[0],
"collectionErrors": result.collection_errors, "collectionSkipped": result.collection_skipped}, 0 if result.wasSuccessful() else 1
def pytest_sources(root, settings):
# Import the host framework before adding project imports. Autoloaded
# environment plugins and addopts are excluded by the host check contract.
import pytest
sys.path[:0] = [str(root), str(root / "src")]
class Collector:
def __init__(self):
self.records = []
self.ordinals = {}
self.active = None
self.collection_errors = 0
self.collection_skipped = 0
def pytest_collection_finish(self, session):
self.records = [{"status": "pending", "nodeid": item.nodeid} for item in session.items]
self.ordinals = {}
for index, item in enumerate(session.items):
self.ordinals.setdefault(id(item), []).append(index)
def pytest_collectreport(self, report):
if report.failed:
self.collection_errors += 1
elif report.skipped:
self.collection_skipped += 1
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
def pytest_runtest_protocol(self, item, nextitem):
pending = self.ordinals.get(id(item), [])
self.active = pending.pop(0) if pending else None
try:
yield
finally:
self.active = None
def pytest_runtest_logreport(self, report):
if self.active is None or self.records[self.active]["nodeid"] != report.nodeid:
self.collection_errors += 1
return
record = self.records[self.active]
if report.failed:
record["status"] = "failed"
elif record["status"] != "failed" and (report.skipped or hasattr(report, "wasxfail")):
record["status"] = "skipped"
elif record["status"] == "pending" and report.when == "call" and report.passed:
record["status"] = "passed"
collector = Collector()
args = [str(checked_path(root, relative)) for relative in settings["testPaths"]]
args += ["-p", "no:cacheprovider", "--import-mode=importlib", "-o", "addopts="]
if settings.get("configPath"):
args += ["-c", str(checked_path(root, settings["configPath"]))]
code = int(pytest.main(args, plugins=[collector]))
return {"tests": count_records(collector.records), "framework": "pytest", "frameworkVersion": pytest.__version__,
"collectionErrors": collector.collection_errors, "collectionSkipped": collector.collection_skipped}, code
def main():
payload_bytes = sys.stdin.buffer.read()
payload = json.loads(payload_bytes)
result_path = Path(sys.argv[1]).resolve()
root = Path(payload["root"]).resolve(strict=True)
if not root.is_dir() or result_path.is_relative_to(root):
raise ValueError("Result collection must use the owned host directory")
executable = Path(os.path.realpath(sys.executable))
base_executable = Path(os.path.realpath(getattr(sys, "_base_executable", sys.executable)))
identity = {"schemaVersion": 1, "kind": "python-check-result", "invocationId": payload["invocationId"],
"inputSha256": sha(payload_bytes), "mode": payload["mode"], "runnerSha256": sha(Path(__file__).read_bytes()),
"runtime": {"executable": str(executable), "version": sys.version.split()[0], "sha256": sha(executable.read_bytes()),
"baseExecutable": str(base_executable), "baseSha256": sha(base_executable.read_bytes())}}
try:
verify_sources(root, payload["files"])
if payload["mode"] == "compile":
evidence, code = compile_sources(root, payload["files"])
elif payload["mode"] == "unittest":
evidence, code = unittest_sources(root, payload["settings"])
elif payload["mode"] == "pytest":
evidence, code = pytest_sources(root, payload["settings"])
else:
raise ValueError("Unknown built-in Python check")
output = {**identity, **evidence, "exitCode": code}
except BaseException as error:
traceback.print_exception(type(error), error, error.__traceback__)
output = {**identity, "exitCode": 1, "collectorError": type(error).__name__ + ": " + str(error)}
code = 1
temporary = result_path.with_suffix(".tmp")
with temporary.open("x", encoding="utf-8") as stream:
json.dump(output, stream, ensure_ascii=False)
temporary.replace(result_path)
return code
if __name__ == "__main__":
raise SystemExit(main())