"""Tests for the S4 QEMU-line evaluation harness (harness/sim_qemu_line.py). The pure-function tests need no firmware. The integration tests use the real TU104 570.144 fwimage read-only and **skip cleanly** when it (or qemu-system-riscv64, or the bundled demo firmware) is absent. All runs are bounded (<= 20k native instructions in tests, <= 1 s QEMU) and touch no hardware, GPU, module, guest or network. Run:: python3 -m unittest tests.test_sim_qemu_line -v """ from __future__ import annotations import json import os import shutil import sys import tempfile import unittest HERE = os.path.dirname(os.path.abspath(__file__)) TOOL_ROOT = os.path.abspath(os.path.join(HERE, "..")) REAL_IMAGE = os.path.normpath( os.path.join(TOOL_ROOT, "..", "..", "firmware", "57X-GSP", "570.144", "gsp-570.144.fwimage.bin") ) DEMO_ELF = os.path.join(TOOL_ROOT, "firmware", "uproc_demo", "build", "uproc_demo.elf") if TOOL_ROOT not in sys.path: sys.path.insert(0, TOOL_ROOT) from greeneye.device_map import QEMU_REFERENCE, firmware_sha256 # noqa: E402 from harness.common import qemu_available # noqa: E402 from harness.sim_qemu_line import FROZEN_THRESHOLDS # noqa: E402 class PureUnitTests(unittest.TestCase): """No firmware required: rung maths, thresholds and report rendering.""" def test_rung_ordinal_and_above(self): from harness.sim_qemu_line import above_qemu_line, rung_ordinal self.assertEqual(rung_ordinal("pre-B0"), 0) self.assertEqual(rung_ordinal("B0"), 1) self.assertIsNone(rung_ordinal("B9")) self.assertTrue(above_qemu_line("pre-B0", "B0")) self.assertFalse(above_qemu_line("pre-B0", "pre-B0")) self.assertFalse(above_qemu_line("B1", "B0")) self.assertFalse(above_qemu_line(None, "B0")) def test_threshold_checks_require_device_model(self): from harness.sim_qemu_line import threshold_checks qemu = {"ran": True, "first_touch_match": True, "rung": "pre-B0"} native = { "ran": True, "rung": "B0", "repeat": 2, "identical": True, "pair_wall_s": 1.0, "device_model_used": False, } checks = threshold_checks({"qemu": qemu, "native": native}) self.assertTrue(checks["qemu_first_fault_reproduced"]) self.assertTrue(checks["native_above_qemu"]) self.assertTrue(checks["identical_runs"]) self.assertFalse(checks["device_model_documented"]) self.assertFalse(checks["all"]) native["device_model_used"] = True self.assertTrue(threshold_checks({"qemu": qemu, "native": native})["all"]) def test_build_report_verdicts(self): from harness.sim_qemu_line import build_report qemu = {"ran": True, "first_touch_match": True, "rung": "pre-B0"} native_ok = { "ran": True, "rung": "B1", "repeat": 2, "identical": True, "pair_wall_s": 1.0, "device_model_used": True, "runs": [], } report = build_report( image=os.path.join(TOOL_ROOT, "does-not-exist.bin"), qemu=qemu, native=native_ok, features=None, s1={"source": None, "declared_rung": None, "notes": []}, thresholds=dict(FROZEN_THRESHOLDS), ) self.assertEqual(report["verdict"], "above") self.assertTrue(report["above_qemu_line"]) self.assertTrue(report["threshold_checks"]["all"]) lenient = dict(native_ok, device_model_used=False) report = build_report( image="missing.bin", qemu=qemu, native=lenient, features=None, s1={"source": None, "declared_rung": None, "notes": []}, thresholds=dict(FROZEN_THRESHOLDS), ) self.assertEqual(report["verdict"], "above_lenient_only") self.assertFalse(report["above_qemu_line"]) report = build_report( image="missing.bin", qemu={"ran": False, "rung": None}, native=native_ok, features=None, s1={"source": None, "declared_rung": None, "notes": []}, thresholds=dict(FROZEN_THRESHOLDS), ) self.assertEqual(report["verdict"], "unverified_qemu_absent") report = build_report( image="missing.bin", qemu={"ran": True, "first_touch_match": False, "rung": "pre-B0"}, native=native_ok, features=None, s1={"source": None, "declared_rung": None, "notes": []}, thresholds=dict(FROZEN_THRESHOLDS), ) self.assertEqual(report["verdict"], "unverified_qemu_mismatch") def test_render_markdown_shape(self): from harness.sim_qemu_line import build_report, render_markdown report = build_report( image="missing.bin", qemu={ "ran": True, "first_touch_match": True, "rung": "pre-B0", "wall_s": 1.0, "first_fault": {"desc": "fault_load", "epc_fw": 0x4009014, "access_addr": QEMU_REFERENCE["access_addr"], "size": 4, "rw": "read"}, "session_hashes": {"session.json": "ab" * 32}, "session": "harness/qemu_line/sessions/qemu-probe", }, native={ "ran": True, "rung": "B0", "rung_basis": "test", "repeat": 2, "identical": True, "pair_wall_s": 1.0, "device_model_used": False, "runs": [ { "rung": "B0", "instructions": 10, "cycles": 12, "wall_s": 0.1, "stop": {"reason": "budget", "pc_fw": 0x4009070}, "session": "harness/qemu_line/sessions/native-1", "session_hashes": {"session.json": "cd" * 32}, } ], "compare": {"ran": True, "headers": []}, }, features={"features": {"mcp": {"status": "not_run", "reason": "S3"}}}, s1={"source": None, "declared_rung": None, "notes": []}, thresholds=dict(FROZEN_THRESHOLDS), ) text = render_markdown(report) self.assertIn("Rung table", text) self.assertIn("Session hashes", text) self.assertIn("Frozen thresholds", text) self.assertIn("above_lenient_only", text) self.assertIn("native run 1", text) @unittest.skipUnless(os.path.isfile(REAL_IMAGE), "TU104 570.144 fwimage absent") class NativeIntegrationTests(unittest.TestCase): """Native-only run on the real image: shape + two-run determinism.""" def setUp(self): self.tmp = tempfile.mkdtemp(prefix="sim-qemu-line-test-") def tearDown(self): shutil.rmtree(self.tmp, ignore_errors=True) def test_profile_discovery_when_s1_present(self): from harness.sim_qemu_line import discover_s1_device_model profile = os.path.join(TOOL_ROOT, "sim", "profiles", "tu104-570.144.json") if not os.path.isfile(profile): self.skipTest("S1 profile absent") s1 = discover_s1_device_model(REAL_IMAGE, firmware_sha256(REAL_IMAGE)) self.assertEqual(os.path.abspath(s1.get("profile") or ""), profile) self.assertTrue(any("greeneye-sim-profile" in note for note in s1["notes"])) def test_native_report_shape_and_determinism(self): from harness.sim_qemu_line import run_harness report = run_harness( image=REAL_IMAGE, out_dir=self.tmp, instructions=20_000, repeat=2, features=False, run_qemu=False, ) self.assertEqual(report["schema"], "greeneye-sim-qemu-line/1") self.assertTrue(report["firmware"]["sha256"]) native = report["native"] self.assertTrue(native["ran"]) self.assertEqual(native["repeat"], 2) self.assertTrue(native["identical"]) runs = native["runs"] self.assertEqual(len(runs), 2) keys = ( "instructions", "cycles", "stop", "first_unknown_mmio", "mmio_unknown", "rung", "device_state_sha256", "milestones", ) self.assertEqual( [{k: r.get(k) for k in keys} for r in runs], [{k: runs[0].get(k) for k in keys}] * 2, ) for run in runs: self.assertGreater(run["instructions"], 0) self.assertLessEqual(run["instructions"], 20_000) self.assertGreater(run["cycles"], 0) self.assertTrue(run["session_hashes"]["session.json"]) if native.get("device_model_used"): # S1 semantic path: device model + recorded shim, B2 observed. self.assertEqual(native["rung"], "B2") self.assertIsNone(runs[0]["first_unknown_mmio"]) self.assertEqual(runs[0]["mmio_unknown"], 0) self.assertEqual(runs[0]["milestones"], ["bootargs_consumer", "relocation_copy"]) self.assertTrue(runs[0]["device_state_sha256"]) else: # Lenient path (no S1 in the tree): B0 observation baseline. self.assertEqual(native["rung"], "B0") self.assertEqual(runs[0]["first_unknown_mmio"]["addr"], QEMU_REFERENCE["access_addr"]) self.assertEqual(report["verdict"], "unverified_qemu_absent") for name in ("sim-qemu-line.json", "sim-qemu-line.md"): path = os.path.join(self.tmp, name) self.assertTrue(os.path.isfile(path), path) with open(os.path.join(self.tmp, "sim-qemu-line.json"), encoding="utf-8") as fh: payload = json.load(fh) self.assertEqual(payload["schema"], report["schema"]) with open(os.path.join(self.tmp, "sim-qemu-line.md"), encoding="utf-8") as fh: md = fh.read() self.assertIn("| native run 1 |", md) self.assertIn("Session hashes", md) def test_native_lenient_path_still_available(self): from harness.sim_qemu_line import run_native_baseline native = run_native_baseline( image=REAL_IMAGE, out_dir=self.tmp, instructions=20_000, repeat=1, profile=None, device_map=None, ) self.assertTrue(native["ran"]) self.assertFalse(native["device_model_used"]) self.assertEqual(native["rung"], "B0") self.assertEqual( native["runs"][0]["first_unknown_mmio"]["addr"], QEMU_REFERENCE["access_addr"] ) @unittest.skipUnless( os.path.isfile(REAL_IMAGE) and qemu_available(), "TU104 570.144 fwimage or qemu-system-riscv64 absent", ) class QemuIntegrationTests(unittest.TestCase): """QEMU probe on the real image must reproduce its documented first fault.""" def setUp(self): self.tmp = tempfile.mkdtemp(prefix="sim-qemu-line-test-") def tearDown(self): shutil.rmtree(self.tmp, ignore_errors=True) def test_qemu_first_fault_reproduced(self): from harness.sim_qemu_line import run_qemu_baseline result = run_qemu_baseline(image=REAL_IMAGE, out_dir=self.tmp, duration=0.5) self.assertTrue(result["ran"], result.get("reason")) self.assertEqual(result["rung"], "pre-B0") self.assertTrue(result["first_touch_match"], "documented first fault not reproduced") first = result["first_fault"] self.assertEqual(first["epc_fw"], QEMU_REFERENCE["firmware_vaddr"]) self.assertEqual(first["access_addr"], QEMU_REFERENCE["access_addr"]) self.assertTrue(result["session_hashes"]["session.json"]) self.assertTrue(os.path.isfile(os.path.join(self.tmp, "sessions", "qemu-probe", "session.json"))) @unittest.skipUnless(os.path.isfile(DEMO_ELF), "demo firmware absent (run make demo)") class FeatureMatrixTests(unittest.TestCase): """The native feature matrix runs on the bundled demo firmware.""" def setUp(self): self.tmp = tempfile.mkdtemp(prefix="sim-qemu-line-test-") def tearDown(self): shutil.rmtree(self.tmp, ignore_errors=True) def test_feature_matrix_shape(self): from harness.sim_qemu_line import run_feature_matrix result = run_feature_matrix(self.tmp) self.assertTrue(result["ran"], result.get("reason")) features = result["features"] for name in ("hooks", "breakpoints", "watch", "memory", "struct", "sessions", "trace"): self.assertIn(name, features) self.assertEqual(features[name]["status"], "pass", features[name]) self.assertEqual(features["mcp"]["status"], "not_run") self.assertIn(features["injection"]["status"], ("pass", "not_run")) self.assertIn(features["analyze_compare"]["status"], ("pass", "not_run")) if __name__ == "__main__": unittest.main()