summaryrefslogtreecommitdiffstats
path: root/kernel/bar_emulation.c
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-09-21 14:10:43 -0400
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-09-21 14:11:45 -0400
commitc8249854284c3e8181587b96e3c5b4177d705e0c (patch)
treef0f1c16b055c88b059d159db715407d2f7406090 /kernel/bar_emulation.c
parent85d888e4763837a5b2f5843c771bf6b6d6e6bfe3 (diff)
downloadopen-iov-c8249854284c3e8181587b96e3c5b4177d705e0c.tar.gz
open-iov-c8249854284c3e8181587b96e3c5b4177d705e0c.zip
kernel: Present physical TU104 to guest + emulate GSP boot (FWSEC/Booter) + VBIOS serving
Guest driver probe fixes (unmodified 570.144 open driver on physical path): - kernel/main.c: add show_description to the mdev driver. Userspace mdev management tools parse 'framebuffer=<n>M' from every mdev type's description sysfs file before creating an mdev; the missing file made such tools fail to create the vGPU. Also fix create_mdev.sh for the real type dir name pci-nvidia-vgpu-* (mdev core names dirs dev_driver_string(parent)-<sysfs_name>; the parent registers before nvidia.ko binds so dev_driver_string == 'pci'). - kernel/bar_emulation.c: seed BAR0 shadow so guest sees a REAL, PHYSICAL, non-virtual TU104 (user directive: no PV/VF vGPU paths, unmodified driver): * PMC_BOOT_0/1/42 = physical TU104 values captured from host hardware (0x164000a1 / 0x0 / 0x164a1000). Without these, halmgrGetHalForGpu() fails and probe aborts with 'does not include the required GSP'. * Serve a valid VBIOS at the PROM window (BAR0+0x300000) loaded from /lib/firmware/tu104_vbios.bin (dump of host TU104 ROM). Guest's kgspExtractVbiosFromRom_TU102 needs it for the mandatory FWSEC ucode (kgspGetFrtsSize is hardwired to 1MB on Turing). * Pre-set GSP secure-boot (GFW) completion state: PGC6 PLM (0x118128)=1, GFW_BOOT progress (0x118234)=0xFF, GSP falcon CPUCTL HALTED (0x110100). The driver's post-FLR GFW wait only *waits* (never kicks), so the pre-set state satisfies it immediately. - kernel/gsp_emulation.c/h: physical GSP boot emulation. * Falcon boot-watch: poll GSP falcon/PGC6/WPR2/scratch registers and log changes (guest BAR0 is mmap-based, otherwise invisible to host). * gsp_emu_poll_boot: STARTCPU auto-complete state machine. Guest kicks FWSEC then Booter Load via GSP falcon CPUCTL; detect each kick and complete the stage (HALTED + FRTS err NONE + WPR2 + mailbox + RISCV-active) so the guest believes its ucodes ran. Diagnosis that led here (documented in Handoff-vGPU-IO.md): - Open nvidia.ko can never be an SR-IOV VF vGPU guest: os_is_grid_supported() is a stub returning FALSE in the open build (disassembled). PV path also rejected per user directive. Only the physical path with a fully emulated local GSP boot remains.
Diffstat (limited to 'kernel/bar_emulation.c')
-rw-r--r--kernel/bar_emulation.c89
1 files changed, 89 insertions, 0 deletions
diff --git a/kernel/bar_emulation.c b/kernel/bar_emulation.c
index 009f0e2..e120b3c 100644
--- a/kernel/bar_emulation.c
+++ b/kernel/bar_emulation.c
@@ -6,6 +6,7 @@
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/eventfd.h>
+#include <linux/firmware.h>
#include "nvidia-vgpu-vfio.h"
#include "bar_emulation.h"
@@ -14,6 +15,11 @@
#define NV_VGPU_BAR0_TRAP_REGIONS 64
+/* VBIOS (PROM window) emulation - see nv_vgpu_bar_emul_init */
+#define NV_VGPU_VBIOS_FW_NAME "tu104_vbios.bin"
+#define NV_VGPU_PROM_WINDOW_OFFSET 0x300000
+#define NV_VGPU_VBIOS_MAX_SIZE 0x100000 /* 1MB (s_getBaseBiosMaxSize_TU102) */
+
/* MSI-X Table entry size (16 bytes: addr_lo, addr_hi, data, vector_control) */
#define NV_VGPU_MSIX_ENTRY_SIZE 16
#define NV_VGPU_MSIX_TABLE_ENTRIES 32
@@ -202,6 +208,89 @@ int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu)
/* Initialize MSI-X emulation */
nv_vgpu_msix_init(ctx);
+ /*
+ * Seed PMC boot registers so the guest driver sees a REAL,
+ * PHYSICAL, non-virtual TU104 GPU. Values captured from the host's
+ * physical TU104 (RTX 2080 SUPER, PCI 0000:01:00.0):
+ * NV_PMC_BOOT_0 (BAR0+0x000) = 0x164000a1 (arch TU104, rev A1)
+ * NV_PMC_BOOT_1 (BAR0+0x004) = 0x00000000 (VGPU field = REAL,
+ * no PV/VF - the guest
+ * must take the physical
+ * probe path, never a
+ * para-virtual vGPU path)
+ * NV_PMC_BOOT_42 (BAR0+0xa00) = 0x164a1000 (chip id / revision)
+ *
+ * The guest reads these via the VFIO-mmap'd shadow (and via VFIO
+ * region reads, which hit the same shadow). Without them the open
+ * driver's halmgrGetHalForGpu() fails and probe aborts with the
+ * "does not include the required GPU System Processor (GSP)" error.
+ */
+ *(u32 *)((u8 *)ctx->bar0_shadow + 0x000) = 0x164000a1;
+ *(u32 *)((u8 *)ctx->bar0_shadow + 0x004) = 0x00000000;
+ *(u32 *)((u8 *)ctx->bar0_shadow + 0xa00) = 0x164a1000;
+
+ /*
+ * Pre-set the GSP secure-boot (GFW) completion state.
+ *
+ * During RmInitAdapter the guest driver waits for the GPU's
+ * post-FLR secure boot (kgspWaitForGfwBootOk -> gpuWaitForGfwBootComplete
+ * -> _gpuIsGfwBootCompleted_TU102 + kflcnWaitForHalt). On real hardware
+ * the boot ROM runs this automatically after reset; the driver only
+ * *waits* for it (it never kicks the falcon here). We present the
+ * "already booted" state so the wait succeeds immediately:
+ * 0x118128 NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK
+ * bit0 READ_PROTECTION_LEVEL0 = ENABLE (FWSEC lowered PLM)
+ * 0x118234 NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT
+ * bits7:0 PROGRESS = 0xFF (COMPLETED)
+ * 0x110100 GSP falcon CPUCTL bit4 = HALTED
+ *
+ * Driver-kicked falcon runs later (FWSEC/Booter) clear CPUCTL via
+ * STARTCPU; the GSP emulation thread re-completes those (see
+ * gsp_emu_poll_boot in gsp_emulation.c).
+ */
+ *(u32 *)((u8 *)ctx->bar0_shadow + 0x118128) = 0x00000001;
+ *(u32 *)((u8 *)ctx->bar0_shadow + 0x118234) = 0x000000ff;
+ *(u32 *)((u8 *)ctx->bar0_shadow + 0x110100) = 0x00000010;
+
+ /*
+ * Serve a valid VBIOS at the PROM window (BAR0+0x300000).
+ *
+ * The guest driver's physical GSP boot (kgspInitRm -> TU102 path)
+ * reads the VBIOS via NV_PROM_DATA (BAR0+0x300000) to extract the
+ * FWSEC ucode (mandatory on Turing: kgspGetFrtsSize is hardwired
+ * to 1MB, so FWSEC/FRTS cannot be skipped). Without a valid ROM
+ * signature the probe aborts with:
+ * "kgspExtractVbiosFromRom_TU102: did not find valid ROM signature"
+ *
+ * The image is a dump of the host's physical TU104 VBIOS (read via
+ * its own PROM window), loaded from /lib/firmware/tu104_vbios.bin.
+ * Non-fatal if absent: probe then fails later in FWSEC, not here.
+ */
+ {
+ const struct firmware *vbios_fw;
+ int fw_ret = request_firmware(&vbios_fw, NV_VGPU_VBIOS_FW_NAME,
+ &vgpu->pdev->dev);
+ if (fw_ret == 0) {
+ u32 copy_size = min_t(u32, vbios_fw->size,
+ NV_VGPU_VBIOS_MAX_SIZE);
+
+ if (NV_VGPU_PROM_WINDOW_OFFSET + copy_size <= ctx->bar0_shadow_size) {
+ memcpy((u8 *)ctx->bar0_shadow + NV_VGPU_PROM_WINDOW_OFFSET,
+ vbios_fw->data, copy_size);
+ nv_vgpu_info(vgpu, "VBIOS loaded at BAR0+0x%x (%u bytes)",
+ NV_VGPU_PROM_WINDOW_OFFSET, copy_size);
+ } else {
+ nv_vgpu_warn(vgpu, "VBIOS too large for PROM window (%u bytes)",
+ copy_size);
+ }
+ release_firmware(vbios_fw);
+ } else {
+ nv_vgpu_warn(vgpu, "VBIOS firmware %s not found (%d); "
+ "guest GSP boot will fail at ROM parse",
+ NV_VGPU_VBIOS_FW_NAME, fw_ret);
+ }
+ }
+
vgpu->bar_emul_ctx = ctx;
return 0;