From 5d4d26f8e773f2871c1b5c9266b82495d390970e Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 21 Sep 2026 12:37:46 -0400
Subject: kernel: Fix guest-hang root cause (VFIO mmap garbage PFN) + rebuild
 mdev remove path

VFIO mmap (main.c): virt_to_phys() was used on vmalloc-family addresses
(vzalloc'd BAR0 shadow, ioremap'd BAR2/3), producing garbage PFNs. Guest
BAR0 access then hit a KVM make_spte reserved-bits WARN and an
EPT-violation livelock, hard-wedging the guest on every nvidia driver
probe (confirmed via QEMU GDB stub). BAR0 now maps per-page via
vmalloc_to_pfn(); BAR2/3/4/5 map bars[N].gpa directly (like BAR1);
config space uses new nv_vgpu_mmap_virt_to_pfn() helper.

mdev remove path (main.c): old path never list_del'd the device
(available_instances leaked), never ran cleanup (leaked gsp_emu kthread),
never unregistered VFIO (leaked /dev/vfio/N + iommu group), never put
pdev, never freed the struct. New kvmgt-style path: cancel init_work ->
wait fds -> vfio_unregister_group_dev -> cleanup work -> list_del ->
pci_dev_put -> vfio_put_device.

gsp_rpc_client.c: fix dma_map_page dma_addr_t->int truncation in
gsp_rpc_alloc_emul (corrupts >=4GB IOVAs); export gsp_client_init_emulation.

gsp_emulation.c: export gsp_emu_process_rpc and gsp_emu_get_stats.

main.c: remove wrong MODULE_SOFTDEP("pre: nvidia") - module has zero
nvidia imports, loads standalone, and must load BEFORE nvidia.ko
(it exports nvidia_vgpu_vfio_* for nvidia.ko to import).

nvidia-vgpu-vfio.h: NULL-safe nv_vgpu_{dbg,info,warn,err} macros.

tests/gsp_emulation_test.c: rewritten against current API - drives
gsp_emu_process_rpc() with crafted shared-memory messages (protocol
validation, SCHEDULE/YIELD/NOTIFY/DEBUG dispatch, stress). Builds with
KBUILD_EXTRA_SYMBOLS; depends: open-iov-vfio.

stack_manager.sh: vfio_mdev doesn't exist on >=6.4 (probe per-module);
load-nvidia detects foreign nvidia via /sys/module/nvidia/version and
errors with guidance instead of silently skipping.
---
 scripts/stack_manager.sh | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

(limited to 'scripts')

diff --git a/scripts/stack_manager.sh b/scripts/stack_manager.sh
index c4685da..0b62170 100755
--- a/scripts/stack_manager.sh
+++ b/scripts/stack_manager.sh
@@ -81,7 +81,14 @@ cmd_build_all() {
 
 cmd_load_prereqs() {
     log_info "Loading kernel prerequisites (vfio, mdev)..."
-    sudo modprobe vfio vfio_iommu_type1 vfio_pci mdev vfio_mdev 2>/dev/null || true
+    # vfio_mdev does not exist on kernels >= 6.4 (mdev core is standalone mdev.ko);
+    # modprobe each individually so a missing one never blocks the rest.
+    local m
+    for m in vfio vfio_iommu_type1 vfio_pci mdev vfio_mdev; do
+        if modinfo "$m" &>/dev/null; then
+            sudo modprobe "$m" 2>/dev/null || true
+        fi
+    done
     lsmod | grep -E "vfio|mdev"
 }
 
@@ -104,13 +111,26 @@ cmd_load_nvidia() {
         log_err "Missing $NVIDIA_KO - run 'build-nvidia' first"
         return 1
     fi
-    if ! lsmod | grep -q "^nvidia "; then
-        log_info "Loading nvidia.ko (OpenRM)..."
-        sudo insmod "$NVIDIA_KO" || { log_err "Failed to load nvidia.ko"; return 1; }
-        log_ok "nvidia.ko loaded"
-    else
-        log_ok "nvidia.ko already loaded"
+    if lsmod | grep -q "^nvidia "; then
+        # A module named 'nvidia' is already loaded. Verify it is OUR build
+        # by comparing versions (ours: 570.144 OpenRM w/ vGPU VFIO patches).
+        local loaded_ver want_ver
+        loaded_ver=$(cat /sys/module/nvidia/version 2>/dev/null)
+        want_ver=$(modinfo -F version "$NVIDIA_KO" 2>/dev/null)
+        if [[ -n "$loaded_ver" && "$loaded_ver" == "$want_ver" ]]; then
+            log_ok "nvidia.ko (Open-IOV OpenRM $loaded_ver) already loaded"
+            return 0
+        fi
+        log_err "A foreign nvidia module (version '$loaded_ver') is already loaded;"
+        log_err "Open-IOV needs its own OpenRM build ('$want_ver'). Cannot load both."
+        log_err "Unload the stock driver first (requires nothing holding /dev/nvidia*,"
+        log_err "i.e. no GUI session), or reboot - stock nvidia is blacklisted via"
+        log_err "/etc/modprobe.d/blacklist-nvidia-open-iov.conf."
+        return 1
     fi
+    log_info "Loading nvidia.ko (OpenRM)..."
+    sudo insmod "$NVIDIA_KO" || { log_err "Failed to load nvidia.ko"; return 1; }
+    log_ok "nvidia.ko loaded"
 }
 
 cmd_install_firmware() {
-- 
cgit v1.2.3