diff options
Diffstat (limited to 'debug/risc-v-gsp-rm-vm/tests/test_sim_qemu_line.py')
| -rw-r--r-- | debug/risc-v-gsp-rm-vm/tests/test_sim_qemu_line.py | 323 |
1 files changed, 323 insertions, 0 deletions
diff --git a/debug/risc-v-gsp-rm-vm/tests/test_sim_qemu_line.py b/debug/risc-v-gsp-rm-vm/tests/test_sim_qemu_line.py new file mode 100644 index 0000000..5a798f2 --- /dev/null +++ b/debug/risc-v-gsp-rm-vm/tests/test_sim_qemu_line.py @@ -0,0 +1,323 @@ +"""Tests for the S4 QEMU-line evaluation harness (harness/sim_qemu_line.py). + +The pure-function tests need no firmware. The integration tests use the real +TU104 570.144 fwimage read-only and **skip cleanly** when it (or +qemu-system-riscv64, or the bundled demo firmware) is absent. All runs are +bounded (<= 20k native instructions in tests, <= 1 s QEMU) and touch no +hardware, GPU, module, guest or network. + +Run:: + + python3 -m unittest tests.test_sim_qemu_line -v +""" +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL_ROOT = os.path.abspath(os.path.join(HERE, "..")) +REAL_IMAGE = os.path.normpath( + os.path.join(TOOL_ROOT, "..", "..", "firmware", "57X-GSP", "570.144", "gsp-570.144.fwimage.bin") +) +DEMO_ELF = os.path.join(TOOL_ROOT, "firmware", "uproc_demo", "build", "uproc_demo.elf") +if TOOL_ROOT not in sys.path: + sys.path.insert(0, TOOL_ROOT) + +from greeneye.device_map import QEMU_REFERENCE, firmware_sha256 # noqa: E402 +from harness.common import qemu_available # noqa: E402 +from harness.sim_qemu_line import FROZEN_THRESHOLDS # noqa: E402 + + +class PureUnitTests(unittest.TestCase): + """No firmware required: rung maths, thresholds and report rendering.""" + + def test_rung_ordinal_and_above(self): + from harness.sim_qemu_line import above_qemu_line, rung_ordinal + + self.assertEqual(rung_ordinal("pre-B0"), 0) + self.assertEqual(rung_ordinal("B0"), 1) + self.assertIsNone(rung_ordinal("B9")) + self.assertTrue(above_qemu_line("pre-B0", "B0")) + self.assertFalse(above_qemu_line("pre-B0", "pre-B0")) + self.assertFalse(above_qemu_line("B1", "B0")) + self.assertFalse(above_qemu_line(None, "B0")) + + def test_threshold_checks_require_device_model(self): + from harness.sim_qemu_line import threshold_checks + + qemu = {"ran": True, "first_touch_match": True, "rung": "pre-B0"} + native = { + "ran": True, + "rung": "B0", + "repeat": 2, + "identical": True, + "pair_wall_s": 1.0, + "device_model_used": False, + } + checks = threshold_checks({"qemu": qemu, "native": native}) + self.assertTrue(checks["qemu_first_fault_reproduced"]) + self.assertTrue(checks["native_above_qemu"]) + self.assertTrue(checks["identical_runs"]) + self.assertFalse(checks["device_model_documented"]) + self.assertFalse(checks["all"]) + native["device_model_used"] = True + self.assertTrue(threshold_checks({"qemu": qemu, "native": native})["all"]) + + def test_build_report_verdicts(self): + from harness.sim_qemu_line import build_report + + qemu = {"ran": True, "first_touch_match": True, "rung": "pre-B0"} + native_ok = { + "ran": True, + "rung": "B1", + "repeat": 2, + "identical": True, + "pair_wall_s": 1.0, + "device_model_used": True, + "runs": [], + } + report = build_report( + image=os.path.join(TOOL_ROOT, "does-not-exist.bin"), + qemu=qemu, + native=native_ok, + features=None, + s1={"source": None, "declared_rung": None, "notes": []}, + thresholds=dict(FROZEN_THRESHOLDS), + ) + self.assertEqual(report["verdict"], "above") + self.assertTrue(report["above_qemu_line"]) + self.assertTrue(report["threshold_checks"]["all"]) + + lenient = dict(native_ok, device_model_used=False) + report = build_report( + image="missing.bin", + qemu=qemu, + native=lenient, + features=None, + s1={"source": None, "declared_rung": None, "notes": []}, + thresholds=dict(FROZEN_THRESHOLDS), + ) + self.assertEqual(report["verdict"], "above_lenient_only") + self.assertFalse(report["above_qemu_line"]) + + report = build_report( + image="missing.bin", + qemu={"ran": False, "rung": None}, + native=native_ok, + features=None, + s1={"source": None, "declared_rung": None, "notes": []}, + thresholds=dict(FROZEN_THRESHOLDS), + ) + self.assertEqual(report["verdict"], "unverified_qemu_absent") + + report = build_report( + image="missing.bin", + qemu={"ran": True, "first_touch_match": False, "rung": "pre-B0"}, + native=native_ok, + features=None, + s1={"source": None, "declared_rung": None, "notes": []}, + thresholds=dict(FROZEN_THRESHOLDS), + ) + self.assertEqual(report["verdict"], "unverified_qemu_mismatch") + + def test_render_markdown_shape(self): + from harness.sim_qemu_line import build_report, render_markdown + + report = build_report( + image="missing.bin", + qemu={ + "ran": True, + "first_touch_match": True, + "rung": "pre-B0", + "wall_s": 1.0, + "first_fault": {"desc": "fault_load", "epc_fw": 0x4009014, + "access_addr": QEMU_REFERENCE["access_addr"], + "size": 4, "rw": "read"}, + "session_hashes": {"session.json": "ab" * 32}, + "session": "harness/qemu_line/sessions/qemu-probe", + }, + native={ + "ran": True, + "rung": "B0", + "rung_basis": "test", + "repeat": 2, + "identical": True, + "pair_wall_s": 1.0, + "device_model_used": False, + "runs": [ + { + "rung": "B0", + "instructions": 10, + "cycles": 12, + "wall_s": 0.1, + "stop": {"reason": "budget", "pc_fw": 0x4009070}, + "session": "harness/qemu_line/sessions/native-1", + "session_hashes": {"session.json": "cd" * 32}, + } + ], + "compare": {"ran": True, "headers": []}, + }, + features={"features": {"mcp": {"status": "not_run", "reason": "S3"}}}, + s1={"source": None, "declared_rung": None, "notes": []}, + thresholds=dict(FROZEN_THRESHOLDS), + ) + text = render_markdown(report) + self.assertIn("Rung table", text) + self.assertIn("Session hashes", text) + self.assertIn("Frozen thresholds", text) + self.assertIn("above_lenient_only", text) + self.assertIn("native run 1", text) + + +@unittest.skipUnless(os.path.isfile(REAL_IMAGE), "TU104 570.144 fwimage absent") +class NativeIntegrationTests(unittest.TestCase): + """Native-only run on the real image: shape + two-run determinism.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="sim-qemu-line-test-") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_profile_discovery_when_s1_present(self): + from harness.sim_qemu_line import discover_s1_device_model + + profile = os.path.join(TOOL_ROOT, "sim", "profiles", "tu104-570.144.json") + if not os.path.isfile(profile): + self.skipTest("S1 profile absent") + s1 = discover_s1_device_model(REAL_IMAGE, firmware_sha256(REAL_IMAGE)) + self.assertEqual(os.path.abspath(s1.get("profile") or ""), profile) + self.assertTrue(any("greeneye-sim-profile" in note for note in s1["notes"])) + + def test_native_report_shape_and_determinism(self): + from harness.sim_qemu_line import run_harness + + report = run_harness( + image=REAL_IMAGE, + out_dir=self.tmp, + instructions=20_000, + repeat=2, + features=False, + run_qemu=False, + ) + self.assertEqual(report["schema"], "greeneye-sim-qemu-line/1") + self.assertTrue(report["firmware"]["sha256"]) + native = report["native"] + self.assertTrue(native["ran"]) + self.assertEqual(native["repeat"], 2) + self.assertTrue(native["identical"]) + runs = native["runs"] + self.assertEqual(len(runs), 2) + keys = ( + "instructions", "cycles", "stop", "first_unknown_mmio", "mmio_unknown", + "rung", "device_state_sha256", "milestones", + ) + self.assertEqual( + [{k: r.get(k) for k in keys} for r in runs], + [{k: runs[0].get(k) for k in keys}] * 2, + ) + for run in runs: + self.assertGreater(run["instructions"], 0) + self.assertLessEqual(run["instructions"], 20_000) + self.assertGreater(run["cycles"], 0) + self.assertTrue(run["session_hashes"]["session.json"]) + if native.get("device_model_used"): + # S1 semantic path: device model + recorded shim, B2 observed. + self.assertEqual(native["rung"], "B2") + self.assertIsNone(runs[0]["first_unknown_mmio"]) + self.assertEqual(runs[0]["mmio_unknown"], 0) + self.assertEqual(runs[0]["milestones"], ["bootargs_consumer", "relocation_copy"]) + self.assertTrue(runs[0]["device_state_sha256"]) + else: + # Lenient path (no S1 in the tree): B0 observation baseline. + self.assertEqual(native["rung"], "B0") + self.assertEqual(runs[0]["first_unknown_mmio"]["addr"], QEMU_REFERENCE["access_addr"]) + self.assertEqual(report["verdict"], "unverified_qemu_absent") + for name in ("sim-qemu-line.json", "sim-qemu-line.md"): + path = os.path.join(self.tmp, name) + self.assertTrue(os.path.isfile(path), path) + with open(os.path.join(self.tmp, "sim-qemu-line.json"), encoding="utf-8") as fh: + payload = json.load(fh) + self.assertEqual(payload["schema"], report["schema"]) + with open(os.path.join(self.tmp, "sim-qemu-line.md"), encoding="utf-8") as fh: + md = fh.read() + self.assertIn("| native run 1 |", md) + self.assertIn("Session hashes", md) + + def test_native_lenient_path_still_available(self): + from harness.sim_qemu_line import run_native_baseline + + native = run_native_baseline( + image=REAL_IMAGE, + out_dir=self.tmp, + instructions=20_000, + repeat=1, + profile=None, + device_map=None, + ) + self.assertTrue(native["ran"]) + self.assertFalse(native["device_model_used"]) + self.assertEqual(native["rung"], "B0") + self.assertEqual( + native["runs"][0]["first_unknown_mmio"]["addr"], QEMU_REFERENCE["access_addr"] + ) + + +@unittest.skipUnless( + os.path.isfile(REAL_IMAGE) and qemu_available(), + "TU104 570.144 fwimage or qemu-system-riscv64 absent", +) +class QemuIntegrationTests(unittest.TestCase): + """QEMU probe on the real image must reproduce its documented first fault.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="sim-qemu-line-test-") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_qemu_first_fault_reproduced(self): + from harness.sim_qemu_line import run_qemu_baseline + + result = run_qemu_baseline(image=REAL_IMAGE, out_dir=self.tmp, duration=0.5) + self.assertTrue(result["ran"], result.get("reason")) + self.assertEqual(result["rung"], "pre-B0") + self.assertTrue(result["first_touch_match"], "documented first fault not reproduced") + first = result["first_fault"] + self.assertEqual(first["epc_fw"], QEMU_REFERENCE["firmware_vaddr"]) + self.assertEqual(first["access_addr"], QEMU_REFERENCE["access_addr"]) + self.assertTrue(result["session_hashes"]["session.json"]) + self.assertTrue(os.path.isfile(os.path.join(self.tmp, "sessions", "qemu-probe", "session.json"))) + + +@unittest.skipUnless(os.path.isfile(DEMO_ELF), "demo firmware absent (run make demo)") +class FeatureMatrixTests(unittest.TestCase): + """The native feature matrix runs on the bundled demo firmware.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="sim-qemu-line-test-") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_feature_matrix_shape(self): + from harness.sim_qemu_line import run_feature_matrix + + result = run_feature_matrix(self.tmp) + self.assertTrue(result["ran"], result.get("reason")) + features = result["features"] + for name in ("hooks", "breakpoints", "watch", "memory", "struct", "sessions", "trace"): + self.assertIn(name, features) + self.assertEqual(features[name]["status"], "pass", features[name]) + self.assertEqual(features["mcp"]["status"], "not_run") + self.assertIn(features["injection"]["status"], ("pass", "not_run")) + self.assertIn(features["analyze_compare"]["status"], ("pass", "not_run")) + + +if __name__ == "__main__": + unittest.main() |