224 lines
9.3 KiB
Python
224 lines
9.3 KiB
Python
"""Collect actual Workbench Python checks after their execution is approved."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import traceback
|
|
|
|
sys.dont_write_bytecode = True
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def checked_path(root, relative):
|
|
if not isinstance(relative, str) or not relative or "\0" in relative:
|
|
raise ValueError("Invalid planned Python path")
|
|
candidate = (root / relative).resolve(strict=True)
|
|
if not candidate.is_relative_to(root):
|
|
raise ValueError("Python input is outside the sealed project")
|
|
return candidate
|
|
|
|
|
|
def verify_sources(root, files):
|
|
for item in files:
|
|
filename = checked_path(root, item["path"])
|
|
if not filename.is_file() or sha(filename.read_bytes()) != item["sha256"]:
|
|
raise ValueError("Planned Python source changed: " + item["path"])
|
|
|
|
|
|
def compile_sources(root, files):
|
|
failures = []
|
|
for item in files:
|
|
filename = checked_path(root, item["path"])
|
|
data = filename.read_bytes()
|
|
try:
|
|
compile(data, str(filename), "exec", dont_inherit=True)
|
|
except (SyntaxError, ValueError) as error:
|
|
failures.append({"path": item["path"], "message": str(error), "line": getattr(error, "lineno", None)})
|
|
traceback.print_exception(type(error), error, error.__traceback__)
|
|
print("Python source compilation: %d / %d" % (len(files) - len(failures), len(files)))
|
|
return {"compilation": {"compiled": len(files) - len(failures), "total": len(files), "failures": failures}}, 1 if failures else 0
|
|
|
|
|
|
def count_records(records):
|
|
states = [item["status"] for item in records]
|
|
return {"total": len(states), "passed": states.count("passed"), "failed": states.count("failed"),
|
|
"skipped": states.count("skipped"), "cancelled": states.count("pending")}
|
|
|
|
|
|
def unittest_sources(root, settings):
|
|
import unittest
|
|
|
|
class CollectedResult(unittest.TextTestResult):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.records = []
|
|
self.active = {}
|
|
self.collection_errors = 0
|
|
self.collection_skipped = 0
|
|
|
|
def startTest(self, test):
|
|
super().startTest(test)
|
|
# Test.id() is a display name. Repeated cases and subtests belong
|
|
# to their actual startTest execution ordinal, not that name.
|
|
self.active[id(test)] = len(self.records)
|
|
self.records.append({"status": "pending"})
|
|
|
|
def stopTest(self, test):
|
|
self.active.pop(id(test), None)
|
|
super().stopTest(test)
|
|
|
|
def mark(self, test, status):
|
|
index = self.active.get(id(test))
|
|
if index is None:
|
|
index = self.active.get(id(getattr(test, "test_case", None)))
|
|
if index is None:
|
|
if status == "failed":
|
|
self.collection_errors += 1
|
|
elif status == "skipped":
|
|
self.collection_skipped += 1
|
|
return
|
|
record = self.records[index]
|
|
if record["status"] != "failed" and (record["status"] != "skipped" or status == "failed"):
|
|
record["status"] = status
|
|
|
|
def addSuccess(self, test):
|
|
self.mark(test, "passed")
|
|
super().addSuccess(test)
|
|
|
|
def addFailure(self, test, error):
|
|
self.mark(test, "failed")
|
|
super().addFailure(test, error)
|
|
|
|
def addError(self, test, error):
|
|
self.mark(test, "failed")
|
|
super().addError(test, error)
|
|
|
|
def addSkip(self, test, reason):
|
|
self.mark(test, "skipped")
|
|
super().addSkip(test, reason)
|
|
|
|
def addExpectedFailure(self, test, error):
|
|
self.mark(test, "skipped")
|
|
super().addExpectedFailure(test, error)
|
|
|
|
def addUnexpectedSuccess(self, test):
|
|
self.mark(test, "failed")
|
|
super().addUnexpectedSuccess(test)
|
|
|
|
def addSubTest(self, test, subtest, error):
|
|
if error is not None:
|
|
self.mark(test, "failed")
|
|
super().addSubTest(test, subtest, error)
|
|
|
|
start = checked_path(root, settings["startDirectory"])
|
|
if not start.is_dir():
|
|
raise ValueError("Unittest discovery needs a project directory")
|
|
sys.path[:0] = [str(root), str(root / "src")]
|
|
loader = unittest.TestLoader()
|
|
suite = loader.discover(str(start), pattern=settings["pattern"])
|
|
result = unittest.TextTestRunner(verbosity=2, resultclass=CollectedResult).run(suite)
|
|
if result.testsRun != len(result.records):
|
|
raise ValueError("Unittest execution count was not collected consistently")
|
|
return {"tests": count_records(result.records), "framework": "unittest", "frameworkVersion": sys.version.split()[0],
|
|
"collectionErrors": result.collection_errors, "collectionSkipped": result.collection_skipped}, 0 if result.wasSuccessful() else 1
|
|
|
|
|
|
def pytest_sources(root, settings):
|
|
# Import the host framework before adding project imports. Autoloaded
|
|
# environment plugins and addopts are excluded by the host check contract.
|
|
import pytest
|
|
sys.path[:0] = [str(root), str(root / "src")]
|
|
|
|
class Collector:
|
|
def __init__(self):
|
|
self.records = []
|
|
self.ordinals = {}
|
|
self.active = None
|
|
self.collection_errors = 0
|
|
self.collection_skipped = 0
|
|
|
|
def pytest_collection_finish(self, session):
|
|
self.records = [{"status": "pending", "nodeid": item.nodeid} for item in session.items]
|
|
self.ordinals = {}
|
|
for index, item in enumerate(session.items):
|
|
self.ordinals.setdefault(id(item), []).append(index)
|
|
|
|
def pytest_collectreport(self, report):
|
|
if report.failed:
|
|
self.collection_errors += 1
|
|
elif report.skipped:
|
|
self.collection_skipped += 1
|
|
|
|
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
|
|
def pytest_runtest_protocol(self, item, nextitem):
|
|
pending = self.ordinals.get(id(item), [])
|
|
self.active = pending.pop(0) if pending else None
|
|
try:
|
|
yield
|
|
finally:
|
|
self.active = None
|
|
|
|
def pytest_runtest_logreport(self, report):
|
|
if self.active is None or self.records[self.active]["nodeid"] != report.nodeid:
|
|
self.collection_errors += 1
|
|
return
|
|
record = self.records[self.active]
|
|
if report.failed:
|
|
record["status"] = "failed"
|
|
elif record["status"] != "failed" and (report.skipped or hasattr(report, "wasxfail")):
|
|
record["status"] = "skipped"
|
|
elif record["status"] == "pending" and report.when == "call" and report.passed:
|
|
record["status"] = "passed"
|
|
|
|
collector = Collector()
|
|
args = [str(checked_path(root, relative)) for relative in settings["testPaths"]]
|
|
args += ["-p", "no:cacheprovider", "--import-mode=importlib", "-o", "addopts="]
|
|
if settings.get("configPath"):
|
|
args += ["-c", str(checked_path(root, settings["configPath"]))]
|
|
code = int(pytest.main(args, plugins=[collector]))
|
|
return {"tests": count_records(collector.records), "framework": "pytest", "frameworkVersion": pytest.__version__,
|
|
"collectionErrors": collector.collection_errors, "collectionSkipped": collector.collection_skipped}, code
|
|
|
|
|
|
def main():
|
|
payload_bytes = sys.stdin.buffer.read()
|
|
payload = json.loads(payload_bytes)
|
|
result_path = Path(sys.argv[1]).resolve()
|
|
root = Path(payload["root"]).resolve(strict=True)
|
|
if not root.is_dir() or result_path.is_relative_to(root):
|
|
raise ValueError("Result collection must use the owned host directory")
|
|
executable = Path(os.path.realpath(sys.executable))
|
|
base_executable = Path(os.path.realpath(getattr(sys, "_base_executable", sys.executable)))
|
|
identity = {"schemaVersion": 1, "kind": "python-check-result", "invocationId": payload["invocationId"],
|
|
"inputSha256": sha(payload_bytes), "mode": payload["mode"], "runnerSha256": sha(Path(__file__).read_bytes()),
|
|
"runtime": {"executable": str(executable), "version": sys.version.split()[0], "sha256": sha(executable.read_bytes()),
|
|
"baseExecutable": str(base_executable), "baseSha256": sha(base_executable.read_bytes())}}
|
|
try:
|
|
verify_sources(root, payload["files"])
|
|
if payload["mode"] == "compile":
|
|
evidence, code = compile_sources(root, payload["files"])
|
|
elif payload["mode"] == "unittest":
|
|
evidence, code = unittest_sources(root, payload["settings"])
|
|
elif payload["mode"] == "pytest":
|
|
evidence, code = pytest_sources(root, payload["settings"])
|
|
else:
|
|
raise ValueError("Unknown built-in Python check")
|
|
output = {**identity, **evidence, "exitCode": code}
|
|
except BaseException as error:
|
|
traceback.print_exception(type(error), error, error.__traceback__)
|
|
output = {**identity, "exitCode": 1, "collectorError": type(error).__name__ + ": " + str(error)}
|
|
code = 1
|
|
temporary = result_path.with_suffix(".tmp")
|
|
with temporary.open("x", encoding="utf-8") as stream:
|
|
json.dump(output, stream, ensure_ascii=False)
|
|
temporary.replace(result_path)
|
|
return code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|