host stability: BAR3 shadow + skip submodule nvidia + teardown-safe garbage handler

Root causes of host GSP crash (WPR2 stuck, 'unexpected WPR2 already up'):
1. stack_manager load-nvidia tried to load our submodule nvidia.ko (610.57.04-base)
   alongside/over apt nvidia 570.211.01 -> GSP crash during driver conflict.
   Fix: skip submodule when apt nvidia is already loaded (not needed for QEMU).
2. BAR3 (32MB instance memory window) was mapped to the physical GPU's real BAR3,
   so guest BAR3 writes could corrupt the host GPU's instance memory.
   Fix: vzalloc'd shadow buffer (like BAR1 emulated vidmem), per-page mmap.
3. GSP-RM garbage element handler declared GSP death on teardown queue garbage,
   preventing driver cleanup frees from completing.
   Fix: stop polling quietly, let Booter-unload kick handle reset.

Verified: full lifecycle (load, mdev create, QEMU guest boot, guest probe,
mdev remove, unload, reload) with host nvidia-smi (apt 570.211.01) stable
throughout. No GSP crash, no WPR2 stuck, no module unload hang.

4 files changed, 51 insertions, 32 deletions

diff --git a/kernel/bar_init.c b/kernel/bar_init.c
index a3ed5fa..9926e2b 100644
--- a/kernel/bar_init.c
+++ b/kernel/bar_init.c
@@ -88,12 +88,18 @@ int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu)
 	if (bar_addr && bar_size) {
 		vgpu->bars[3].gpa = bar_addr;
 		vgpu->bars[3].size = bar_size;
-		vgpu->bars[3].vaddr = ioremap_wc(bar_addr, bar_size);
+		/* Emulated BAR3: dedicated host-RAM shadow. Do NOT ioremap the
+		 * host GPU's real BAR3 - guest writes through this region must
+		 * hit our own clean buffer, not the (running) host GPU's
+		 * instance memory window. 32MB to match the GSCI-reported BAR3. */
+		vgpu->bars[3].vaddr = vzalloc(bar_size);
 		if (!vgpu->bars[3].vaddr) {
-			nv_vgpu_err(vgpu, "Failed to map BAR3");
+			nv_vgpu_err(vgpu, "Failed to allocate emulated BAR3 (%llu bytes)",
+				    (unsigned long long)bar_size);
 			return -ENOMEM;
 		}
-		nv_vgpu_info(vgpu, "BAR3: gpa=0x%llx, size=0x%llx, vaddr=%p", bar_addr, bar_size, vgpu->bars[3].vaddr);
+		nv_vgpu_info(vgpu, "BAR3: gpa=0x%llx, size=0x%llx, emulated shadow=%p",
+			     bar_addr, bar_size, vgpu->bars[3].vaddr);
 	}
 
 	ret = nv_vgpu_bar_emul_init(vgpu);
@@ -130,9 +136,15 @@ void nv_vgpu_bar_cleanup(struct nv_vgpu_device *vgpu)
 		vgpu->bars[1].vaddr = NULL;
 	}
 
+	/* Free the emulated BAR3 shadow (vzalloc'd, not ioremap'd) */
+	if (vgpu->bars[3].vaddr) {
+		vfree(vgpu->bars[3].vaddr);
+		vgpu->bars[3].vaddr = NULL;
+	}
+
 	for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) {
-		if (i == 1)
-			continue; /* vidmem buffer already freed above */
+		if (i == 1 || i == 3)
+			continue; /* shadow buffers already freed above */
 		if (vgpu->bars[i].vaddr) {
 			iounmap(vgpu->bars[i].vaddr);
 			vgpu->bars[i].vaddr = NULL;
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 2d38829..44e716b 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -1530,16 +1530,17 @@ static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
 
 		if (signature != GSPRM_RPC_SIGNATURE || elem_count == 0 ||
 		    elem_count > rm->cmd_msgcount) {
-			/* The driver always writes a valid element BEFORE
-			 * bumping writePtr. Garbage content means the queue
-			 * buffer is stale (freed and reused by the guest
-			 * after unload/crash) - declare the emulated GSP
-			 * dead so the next probe re-boots. */
-			nv_vgpu_warn(ctx->vgpu, "GSPRM: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, GSP boot declared dead",
+			/* Garbage element: could be teardown (driver stopped
+			 * writing valid elements after UNLOADING_GUEST_DRIVER/
+			 * FREE sequence) or genuine queue death. Don't declare
+			 * death here - stop polling quietly and let the
+			 * Booter-unload kick (SEC2 CPUCTL) trigger the reset
+			 * via gsp_emu_clear_boot_state. This allows the
+			 * driver's cleanup frees to complete. */
+			nv_vgpu_info(ctx->vgpu, "GSPRM: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, stopping poll (teardown?)",
 				     rm->cmd_read, signature, elem_count,
 				     hdr->function);
 			kfree(elem);
-			gsp_emu_clear_boot_state(ctx, "garbage GSP-RM queue");
 			return;
 		}
 
diff --git a/kernel/main.c b/kernel/main.c
index 5e99881..f37be6e 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -1180,17 +1180,28 @@ static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct
 		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
 	}
 
-	/* BAR3 region - ioremap'd host BAR; map backing physical address */
+	/* BAR3 region - emulated shadow buffer (not physical GPU BAR) */
 	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR3_REGION_INDEX);
 	region_size = vgpu->bars[3].size;
 	if (offset >= region_base && offset < region_base + region_size) {
+		unsigned long bar3_off = offset - region_base;
+		unsigned long pos;
+
 		if (!vgpu->bars[3].vaddr || region_size == 0)
 			return -ENODEV;
 		if (offset + size > region_base + region_size)
 			return -EINVAL;
-		pfn = (vgpu->bars[3].gpa + (offset - region_base)) >> PAGE_SHIFT;
-		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+		/* BAR3 shadow is vzalloc'd: per-page vmalloc_to_pfn like BAR0 shadow.
+		 * Guest BAR3 accesses hit our clean host-RAM backing, not the
+		 * host GPU's real instance memory aperture. */
+		vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
+		for (pos = 0; pos < size; pos += PAGE_SIZE) {
+			pfn = vmalloc_to_pfn(vgpu->bars[3].vaddr + bar3_off + pos);
+			if (remap_pfn_range(vma, vma->vm_start + pos, pfn, PAGE_SIZE,
+					    vma->vm_page_prot))
+				return -EAGAIN;
+		}
+		return 0;
 	}
 
 	/* BAR4 region - map backing physical address if present */
diff --git a/scripts/stack_manager.sh b/scripts/stack_manager.sh
index ef8385a..c247d08 100755
--- a/scripts/stack_manager.sh
+++ b/scripts/stack_manager.sh
@@ -110,25 +110,20 @@ cmd_load_open_iov() {
 
 cmd_load_nvidia() {
     if [[ ! -f "$NVIDIA_KO" ]]; then
-        log_err "Missing $NVIDIA_KO - run 'build-nvidia' first"
-        return 1
+        log_warn "NVIDIA submodule not built - skipping (apt nvidia works for host GPU)"
+        return 0
     fi
     if lsmod | grep -q "^nvidia "; then
-        # A module named 'nvidia' is already loaded. Verify it is OUR build
-        # by comparing versions (ours: 570.144 OpenRM w/ vGPU VFIO patches).
-        local loaded_ver want_ver
+        # A module named 'nvidia' is already loaded. For QEMU/vGPU with
+        # open-iov-vfio.ko (emulated VFIO device), we do NOT need our
+        # submodule nvidia.ko on the host - the apt nvidia driver manages
+        # the physical GPU and open-iov-vfio handles the emulated vGPU.
+        # Loading our submodule would conflict with the apt driver and
+        # potentially crash the GSP (WPR2 stuck). Skip it.
+        local loaded_ver
         loaded_ver=$(cat /sys/module/nvidia/version 2>/dev/null)
-        want_ver=$(modinfo -F version "$NVIDIA_KO" 2>/dev/null)
-        if [[ -n "$loaded_ver" && "$loaded_ver" == "$want_ver" ]]; then
-            log_ok "nvidia.ko (Open-IOV OpenRM $loaded_ver) already loaded"
-            return 0
-        fi
-        log_err "A foreign nvidia module (version '$loaded_ver') is already loaded;"
-        log_err "Open-IOV needs its own OpenRM build ('$want_ver'). Cannot load both."
-        log_err "Unload the stock driver first (requires nothing holding /dev/nvidia*,"
-        log_err "i.e. no GUI session), or reboot - stock nvidia is blacklisted via"
-        log_err "/etc/modprobe.d/blacklist-nvidia-open-iov.conf."
-        return 1
+        log_ok "apt nvidia ($loaded_ver) already loaded - submodule not needed for QEMU"
+        return 0
     fi
     log_info "Loading nvidia.ko (OpenRM)..."
     sudo insmod "$NVIDIA_KO" || { log_err "Failed to load nvidia.ko"; return 1; }