diff options
Diffstat (limited to 'debug/risc-v-gsp-rm-vm/tests/test_contracts.py')
| -rw-r--r-- | debug/risc-v-gsp-rm-vm/tests/test_contracts.py | 171 |
1 files changed, 171 insertions, 0 deletions
diff --git a/debug/risc-v-gsp-rm-vm/tests/test_contracts.py b/debug/risc-v-gsp-rm-vm/tests/test_contracts.py new file mode 100644 index 0000000..3199b09 --- /dev/null +++ b/debug/risc-v-gsp-rm-vm/tests/test_contracts.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Tests for the wall-contract registry and runner (``harness/contracts.py``). + +The fast contracts execute the proven offline entry points (valprobe matrix, +GMMU golden vector, launch audit, rehost demo, RM-capture demo) and are +asserted to pass; prerequisites that are genuinely absent are reported as +skips, never as passes. No hardware, no module, no guest, no network. + +Run:: + + python3 -m unittest tests.test_contracts -v +""" +from __future__ import annotations + +import contextlib +import importlib +import io +import json +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL_ROOT = os.path.abspath(os.path.join(HERE, "..")) +if TOOL_ROOT not in sys.path: + sys.path.insert(0, TOOL_ROOT) + +from harness import run_contracts as runner # noqa: E402 +from harness.contracts import ( # noqa: E402 + FAST, + SLOW, + WALL_CLASSES, + Contract, + ContractSkip, + all_contracts, + fast_contracts, + run_contracts, + summarize, +) + +MISSING_PATH = os.path.join(TOOL_ROOT, "contract-test-missing-prerequisite") + + +def _entry_target(entry: str) -> tuple[str, str]: + module, _, rest = entry.partition(":") + return module, rest.split("(", 1)[0].strip() + + +def _raise(exc: Exception): + def _run(): + raise exc + + return _run + + +def _fake(name: str, **overrides) -> Contract: + fields = { + "name": name, + "wall_class": "a", + "doc": "docs/50", + "entry": "tests:fake", + "speed": FAST, + "expect": "test expectation", + "run": lambda: {"ok": True}, + "check": lambda report: (report.get("ok") is True, "ok"), + "requires": (), + } + fields.update(overrides) + return Contract(**fields) + + +def _run_main(argv: list[str]) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + rc = runner.main(argv) + except SystemExit as exc: + rc = int(exc.code or 0) + return rc, out.getvalue(), err.getvalue() + + +class RegistryShapeTests(unittest.TestCase): + def test_registry_shape(self): + contracts = all_contracts() + names = [contract.name for contract in contracts] + self.assertEqual(len(names), len(set(names)), "contract names must be unique") + self.assertGreaterEqual(len(contracts), 3, "keep at least the proven contracts") + for contract in contracts: + with self.subTest(contract=contract.name): + self.assertIn(contract.wall_class, WALL_CLASSES) + self.assertIn(contract.speed, (FAST, SLOW)) + self.assertTrue(contract.doc.startswith("docs/")) + self.assertTrue(contract.entry) + self.assertTrue(contract.expect) + self.assertTrue(fast_contracts(), "registry must have fast contracts") + + def test_entries_resolve_to_real_callables(self): + for contract in all_contracts(): + with self.subTest(contract=contract.name): + module_name, func_name = _entry_target(contract.entry) + module = importlib.import_module(module_name) + self.assertTrue(callable(getattr(module, func_name, None)), contract.entry) + + +class RunnerSemanticsTests(unittest.TestCase): + def test_classifies_pass_fail_skip(self): + results = run_contracts( + [ + _fake("good"), + _fake("raising", run=_raise(RuntimeError("boom"))), + _fake("missing-prereq", requires=(MISSING_PATH,)), + _fake("declared-skip", run=_raise(ContractSkip("not applicable here"))), + _fake("failed-check", check=lambda report: (False, "checker rejected")), + ] + ) + by_name = {result.name: result for result in results} + self.assertEqual(by_name["good"].status, "pass") + self.assertEqual(by_name["missing-prereq"].status, "skip") + self.assertEqual(by_name["declared-skip"].status, "skip") + self.assertEqual(by_name["raising"].status, "fail") + self.assertIn("boom", by_name["raising"].detail) + self.assertEqual(by_name["failed-check"].status, "fail") + + summary = summarize(results) + self.assertFalse(summary["ok"]) + self.assertEqual(summary["counts"], {"pass": 1, "fail": 2, "skip": 2}) + domains = {domain["wall_class"]: domain for domain in summary["domains"]} + self.assertEqual(set(domains), set(WALL_CLASSES)) + self.assertEqual(domains["a"]["total"], 5) + self.assertEqual(domains["d"]["total"], 0) + + def test_fast_contracts_pass(self): + contracts = fast_contracts() + self.assertGreaterEqual(len(contracts), 3) + results = run_contracts(contracts) + for result in results: + with self.subTest(contract=result.name): + self.assertNotEqual(result.status, "fail", result.detail) + if result.status == "skip": + self.skipTest(result.detail) + self.assertTrue(any(result.status == "pass" for result in results)) + + +class RunnerCliTests(unittest.TestCase): + def test_single_contract_json_summary(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "contracts.json") + rc, out, err = _run_main( + ["--fast", "--name", "gmmu-golden-vector", "--json", path] + ) + self.assertEqual(rc, 0, err) + self.assertIn("PASS", out) + with open(path, "r", encoding="utf-8") as fh: + summary = json.load(fh) + self.assertTrue(summary["ok"]) + self.assertEqual(summary["counts"]["pass"], 1) + self.assertEqual( + [contract["name"] for contract in summary["contracts"]], + ["gmmu-golden-vector"], + ) + self.assertEqual(len(summary["domains"]), len(WALL_CLASSES)) + + def test_unknown_contract_name_is_usage_error(self): + rc, _out, err = _run_main(["--fast", "--name", "no-such-contract"]) + self.assertEqual(rc, 2) + self.assertIn("unknown contract", err) + + +if __name__ == "__main__": + unittest.main() |