diff options
Diffstat (limited to 'debug/risc-v-gsp-rm-vm/tests/test_harness.py')
| -rw-r--r-- | debug/risc-v-gsp-rm-vm/tests/test_harness.py | 144 |
1 files changed, 144 insertions, 0 deletions
diff --git a/debug/risc-v-gsp-rm-vm/tests/test_harness.py b/debug/risc-v-gsp-rm-vm/tests/test_harness.py new file mode 100644 index 0000000..55938ae --- /dev/null +++ b/debug/risc-v-gsp-rm-vm/tests/test_harness.py @@ -0,0 +1,144 @@ +"""Tests for the green-eye automation harness (roadmap R5/R6, H4.1/H4.3). + +The integration tests drive the two harness CLIs as subprocesses against the +bundled demo firmware and skip cleanly when qemu-system-riscv64 or the demo +ELF is absent. The pure-function tests need neither. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL_ROOT = os.path.abspath(os.path.join(HERE, "..")) +DEMO_ELF = os.path.join(TOOL_ROOT, "firmware", "uproc_demo", "build", "uproc_demo.elf") +MATRIX = os.path.join(TOOL_ROOT, "harness", "portability_matrix.py") +MUTATION = os.path.join(TOOL_ROOT, "harness", "mutation_harness.py") +if TOOL_ROOT not in sys.path: + sys.path.insert(0, TOOL_ROOT) + +from harness.common import qemu_available # noqa: E402 +from harness.mutation_harness import Mutation, expected_for, expected_ok, generate_mutations # noqa: E402 +from harness.portability_matrix import counter_delta # noqa: E402 + + +def _run(args: list[str], timeout: float = 600.0) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable] + args, + capture_output=True, + text=True, + cwd=TOOL_ROOT, + timeout=timeout, + ) + + +class HarnessUnitTests(unittest.TestCase): + def test_generate_mutations_deterministic(self): + first = generate_mutations(9, seed=5) + second = generate_mutations(9, seed=5) + self.assertEqual([m.to_json() for m in first], [m.to_json() for m in second]) + self.assertEqual( + {m.strategy for m in first}, + {"cmd-invalid", "arg-boundary", "bit-flip", "buffer-length"}, + ) + self.assertNotEqual([m.to_json() for m in first], [m.to_json() for m in generate_mutations(9, seed=6)]) + + def test_memory_region_strategy(self): + mutations = generate_mutations(5, seed=1, region=(0x80001000, 64)) + region = [m for m in mutations if m.strategy == "mem-region"] + self.assertEqual(len(region), 1) + self.assertEqual(region[0].region["addr"], 0x80001000) + self.assertTrue(0 <= region[0].region["offset"] < 64) + + def test_demo_oracle(self): + before = {"result": 0x30, "last_rpc": 5} + ping = Mutation(0, "bit-flip", cmd=1, arg0=0x1234) + self.assertEqual(expected_for(ping, before, 0x80000800)["value"], 0x1234 ^ 0xBEEF) + fill = Mutation(1, "arg-boundary", cmd=3, arg1=1 << 40) + self.assertEqual(expected_for(fill, before, 0)["value"], 256) + invalid = Mutation(2, "cmd-invalid", cmd=13) + after = {"result": (~13) & ((1 << 64) - 1), "last_rpc": 13} + self.assertTrue(expected_ok(expected_for(invalid, before, 0), invalid, before, after)) + idle = Mutation(3, "cmd-invalid", cmd=0) + unchanged = {"result": 0x30, "last_rpc": 5} + self.assertTrue(expected_ok(expected_for(idle, before, 0), idle, before, unchanged)) + + def test_counter_delta_zero_for_identical_profiles(self): + rows = counter_delta({"rpc_ping": 2}, {"rpc_ping": 2}) + self.assertTrue(all(row["delta"] == 0 for row in rows)) + + +@unittest.skipUnless(qemu_available(), "qemu-system-riscv64 not installed") +@unittest.skipUnless(os.path.exists(DEMO_ELF), "demo firmware not built (make demo)") +class HarnessIntegrationTests(unittest.TestCase): + def test_portability_matrix_self_comparison(self): + with tempfile.TemporaryDirectory(prefix="harness-matrix-") as out: + result = _run( + [ + MATRIX, + "--target", + DEMO_ELF, + "--target", + DEMO_ELF, + "--out", + out, + "--probe-instructions", + "200", + "--scenario-duration", + "0.2", + ] + ) + self.assertEqual(result.returncode, 0, result.stderr[-2000:]) + with open(os.path.join(out, "portability-report.json"), encoding="utf-8") as fh: + report = json.load(fh) + self.assertEqual([t["status"] for t in report["targets"]], ["ok", "ok"]) + self.assertEqual(len(report["comparisons"]), 1) + comparison = report["comparisons"][0] + self.assertTrue(comparison["all_zero"], comparison["deltas"]) + for rows in comparison["deltas"].values(): + for row in rows: + self.assertEqual(row["delta"], 0) + self.assertTrue(os.path.exists(os.path.join(out, "portability-report.md"))) + self.assertTrue(all(t["scenario"]["event_counts"] for t in report["targets"])) + + def test_mutation_run_and_seed_determinism(self): + with tempfile.TemporaryDirectory(prefix="harness-mut-a-") as out_a, tempfile.TemporaryDirectory( + prefix="harness-mut-b-" + ) as out_b: + base = [ + MUTATION, + "--kernel", + DEMO_ELF, + "--iterations", + "12", + "--seed", + "7", + "--timeout", + "0.3", + "--settle", + "0.1", + ] + first = _run(base + ["--out", out_a]) + self.assertEqual(first.returncode, 0, first.stderr[-2000:]) + second = _run(base + ["--out", out_b]) + self.assertEqual(second.returncode, 0, second.stderr[-2000:]) + findings_path = os.path.join(out_a, "findings.jsonl") + self.assertTrue(os.path.exists(findings_path)) + with open(findings_path, encoding="utf-8") as fh: + findings = [json.loads(line) for line in fh if line.strip()] + self.assertEqual(len(findings), 12) + self.assertTrue({finding["outcome"] for finding in findings}) + self.assertTrue(all("mutation" in finding and "detail" in finding for finding in findings)) + self.assertTrue(os.path.exists(os.path.join(out_a, "summary.md"))) + with open(findings_path, encoding="utf-8") as fa, open( + os.path.join(out_b, "findings.jsonl"), encoding="utf-8" + ) as fb: + self.assertEqual(fa.read(), fb.read(), "the same seed must replay byte-for-byte") + + +if __name__ == "__main__": + unittest.main() |