From 60da11fe771968bbc0cc65b8f629cc5e6b3f44a1 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Tue, 22 Sep 2026 09:43:22 -0400
Subject: host stability: BAR3 shadow + skip submodule nvidia + teardown-safe
 garbage handler

Root causes of host GSP crash (WPR2 stuck, 'unexpected WPR2 already up'):
1. stack_manager load-nvidia tried to load our submodule nvidia.ko (610.57.04-base)
   alongside/over apt nvidia 570.211.01 -> GSP crash during driver conflict.
   Fix: skip submodule when apt nvidia is already loaded (not needed for QEMU).
2. BAR3 (32MB instance memory window) was mapped to the physical GPU's real BAR3,
   so guest BAR3 writes could corrupt the host GPU's instance memory.
   Fix: vzalloc'd shadow buffer (like BAR1 emulated vidmem), per-page mmap.
3. GSP-RM garbage element handler declared GSP death on teardown queue garbage,
   preventing driver cleanup frees from completing.
   Fix: stop polling quietly, let Booter-unload kick handle reset.

Verified: full lifecycle (load, mdev create, QEMU guest boot, guest probe,
mdev remove, unload, reload) with host nvidia-smi (apt 570.211.01) stable
throughout. No GSP crash, no WPR2 stuck, no module unload hang.
---
 scripts/stack_manager.sh | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

(limited to 'scripts')

diff --git a/scripts/stack_manager.sh b/scripts/stack_manager.sh
index ef8385a..c247d08 100755
--- a/scripts/stack_manager.sh
+++ b/scripts/stack_manager.sh
@@ -110,25 +110,20 @@ cmd_load_open_iov() {
 
 cmd_load_nvidia() {
     if [[ ! -f "$NVIDIA_KO" ]]; then
-        log_err "Missing $NVIDIA_KO - run 'build-nvidia' first"
-        return 1
+        log_warn "NVIDIA submodule not built - skipping (apt nvidia works for host GPU)"
+        return 0
     fi
     if lsmod | grep -q "^nvidia "; then
-        # A module named 'nvidia' is already loaded. Verify it is OUR build
-        # by comparing versions (ours: 570.144 OpenRM w/ vGPU VFIO patches).
-        local loaded_ver want_ver
+        # A module named 'nvidia' is already loaded. For QEMU/vGPU with
+        # open-iov-vfio.ko (emulated VFIO device), we do NOT need our
+        # submodule nvidia.ko on the host - the apt nvidia driver manages
+        # the physical GPU and open-iov-vfio handles the emulated vGPU.
+        # Loading our submodule would conflict with the apt driver and
+        # potentially crash the GSP (WPR2 stuck). Skip it.
+        local loaded_ver
         loaded_ver=$(cat /sys/module/nvidia/version 2>/dev/null)
-        want_ver=$(modinfo -F version "$NVIDIA_KO" 2>/dev/null)
-        if [[ -n "$loaded_ver" && "$loaded_ver" == "$want_ver" ]]; then
-            log_ok "nvidia.ko (Open-IOV OpenRM $loaded_ver) already loaded"
-            return 0
-        fi
-        log_err "A foreign nvidia module (version '$loaded_ver') is already loaded;"
-        log_err "Open-IOV needs its own OpenRM build ('$want_ver'). Cannot load both."
-        log_err "Unload the stock driver first (requires nothing holding /dev/nvidia*,"
-        log_err "i.e. no GUI session), or reboot - stock nvidia is blacklisted via"
-        log_err "/etc/modprobe.d/blacklist-nvidia-open-iov.conf."
-        return 1
+        log_ok "apt nvidia ($loaded_ver) already loaded - submodule not needed for QEMU"
+        return 0
     fi
     log_info "Loading nvidia.ko (OpenRM)..."
     sudo insmod "$NVIDIA_KO" || { log_err "Failed to load nvidia.ko"; return 1; }
-- 
cgit v1.2.3