Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 936 insertions, 0 deletions

diff --git a/kernel/gsp_emulation.c b/kernel/gsp_emulation.c
new file mode 100644
index 0000000..833ce42
--- /dev/null
+++ b/kernel/gsp_emulation.c
@@ -0,0 +1,936 @@
+/*
+ * GSP Emulation Engine Implementation
+ * Host-side GSP emulation thread that processes RPCs from guest via shared memory
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <linux/workqueue.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/ktime.h>
+#include <linux/eventfd.h>
+#include <asm/io.h>
+
+#include "nvidia-vgpu-vfio.h"
+#include "gsp_emulation.h"
+#include "gsp_rm_emulation.h"
+#include "gsp_rpc_client.h"
+#include "bar_emulation.h"
+#include "irq.h"
+#include "memory.h"
+
+/* ============================================================
+ * Constants
+ * ============================================================ */
+
+#define GSP_EMU_DOORBELL_OFFSET     0x110C00
+#define GSP_EMU_SHARED_MEM_OFFSET   0x1000
+#define GSP_EMU_SHARED_MEM_SIZE     4096
+#define GSP_EMU_MSIX_TABLE_OFFSET   0x2000
+#define GSP_EMU_MSIX_PBA_OFFSET     0x3000
+#define GSP_EMU_POLL_BUSY_US        100
+#define GSP_EMU_POLL_SLEEP_MIN_US   50
+#define GSP_EMU_POLL_SLEEP_MAX_US   100
+
+/* ============================================================
+ * Helper Functions
+ * ============================================================ */
+
+static inline u32 gsp_emu_read_doorbell(struct gsp_emu_context *ctx)
+{
+    return readl(ctx->doorbell_reg);
+}
+
+static inline void gsp_emu_write_doorbell(struct gsp_emu_context *ctx, u32 val)
+{
+    writel(val, ctx->doorbell_reg);
+}
+
+static inline void gsp_emu_copy_request_from_shared(struct gsp_emu_context *ctx,
+                                                     struct gsp_rpc_msg *req)
+{
+    memcpy(req, ctx->shared_mem_virt, sizeof(*req));
+}
+
+static inline void gsp_emu_copy_response_to_shared(struct gsp_emu_context *ctx,
+                                                    struct gsp_rpc_response *rsp)
+{
+    memcpy(ctx->shared_mem_virt + 0x80, rsp, sizeof(*rsp));
+}
+
+/* ============================================================
+ * GSP Falcon Boot-Watch (observability for physical GSP boot)
+ * ============================================================ */
+
+/*
+ * The guest's physical GSP boot (FWSEC -> Booter -> RISCV) pokes the GSP
+ * falcon block via the mmap'd shadow, which the host cannot trap. Poll these
+ * BAR0 offsets and log changes so the boot sequence is visible in host dmesg.
+ */
+static const struct {
+	u32 offset;
+	const char *name;
+} gsp_emu_falcon_watch[GSP_EMU_FALCON_WATCH_COUNT] = {
+	{ 0x110040, "GSP_MAILBOX0" },
+	{ 0x110044, "GSP_MAILBOX1" },
+	{ 0x110080, "GSP_FALCON_OS" },
+	{ 0x110100, "GSP_CPUCTL" },
+	{ 0x110180, "GSP_IMEMC" },
+	{ 0x110184, "GSP_IMEMD" },
+	{ 0x1101c0, "GSP_DMEMC" },
+	{ 0x1101c4, "GSP_DMEMD" },
+	{ 0x1103c0, "GSP_ENGINE" },
+	{ 0x110c04, "GSP_QUEUE_HEAD1" },
+	{ 0x110c08, "GSP_QUEUE_TAIL0" },
+	{ 0x110c0c, "GSP_QUEUE_TAIL1" },
+	{ 0x118128, "PGC6_PLM_GFW" },
+	{ 0x118234, "PGC6_GFW_BOOT" },
+	{ 0x1fa824, "WPR2_ADDR_LO" },
+	{ 0x1fa828, "WPR2_ADDR_HI" },
+	{ 0x001438, "VBIOS_SCRATCH_0E" },
+	{ 0x840040, "SEC2_MAILBOX0" },
+	{ 0x840100, "SEC2_CPUCTL" },
+	{ 0x8403c0, "SEC2_ENGINE" },
+};
+
+static void gsp_emu_poll_falcon_watch(struct gsp_emu_context *ctx)
+{
+	int i;
+
+	for (i = 0; i < GSP_EMU_FALCON_WATCH_COUNT; i++) {
+		u32 val = readl(ctx->bar0_vaddr + gsp_emu_falcon_watch[i].offset);
+
+		if (!ctx->falcon_watch_valid) {
+			ctx->falcon_watch_last[i] = val;
+			continue;
+		}
+
+		if (val != ctx->falcon_watch_last[i]) {
+			nv_vgpu_info(ctx->vgpu, "FALCON %s (0x%x): 0x%08x -> 0x%08x",
+				     gsp_emu_falcon_watch[i].name,
+				     gsp_emu_falcon_watch[i].offset,
+				     ctx->falcon_watch_last[i], val);
+			ctx->falcon_watch_last[i] = val;
+		}
+	}
+	ctx->falcon_watch_valid = true;
+}
+
+/* ============================================================
+ * Physical GSP Boot Emulation (FWSEC / Booter falcon auto-complete)
+ * ============================================================ */
+
+/* Register offsets + CPUCTL bits live in gsp_emulation.h (shared with
+ * gsp_rm_emulation.c), as are gsp_emu_shadow_read32/write32. */
+
+enum gsp_emu_boot_stage {
+	GSP_BOOT_STAGE_IDLE = 0,	/* waiting for first kick (FWSEC) */
+	GSP_BOOT_STAGE_FWSEC_DONE,	/* FWSEC FRTS completed */
+	GSP_BOOT_STAGE_BOOTER_DONE,	/* Booter Load completed */
+};
+
+/*
+ * gsp_emu_clear_boot_state - return the emulated GSP to a cold state.
+ *
+ * On real hardware, driver unload tears down WPR2 (Booter Unload) so the
+ * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce
+ * no observable BAR0 writes on unload, so callers must trigger this on
+ * indirect death signals (ENGINE reset, GSP-RM queue going stale).
+ * Clears WPR2, RISCV active status and the falcon stage tracker, and
+ * drops the parsed GSP-RM queue state.
+ */
+/*
+ * gsp_emu_clear_boot_state - return the emulated GSP to a cold state.
+ *
+ * On real hardware, driver unload tears down WPR2 (Booter Unload) so the
+ * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce
+ * no observable BAR0 writes on unload, so callers must trigger this on
+ * indirect death signals (ENGINE reset, GSP-RM queue going stale).
+ *
+ * IMPORTANT: on the Booter-unload kick we clear the FALCON boot state
+ * (WPR2/RISCV/stages) but PRESERVE the GSP-RM queue state. The driver
+ * sends cleanup FREEs AFTER the Booter-unload kick; if we reset the queue
+ * here, those frees time out (fn-10). The queue state is only cleared when
+ * the queue is genuinely gone (garbage elements, insane writePtr, ENGINE
+ * reset, or a new probe writing fresh libos args to MAILBOX0/1).
+ */
+void gsp_emu_clear_boot_state(struct gsp_emu_context *ctx, const char *reason)
+{
+	nv_vgpu_info(ctx->vgpu, "GSP-BOOT: clearing boot state (%s)", reason);
+
+	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO, 0);
+	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0);
+	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0);
+	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX0, 0);
+	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX1, 0);
+
+	ctx->boot_stage = GSP_BOOT_STAGE_IDLE;
+	ctx->flcn_kick_count = 0;
+
+	/* Do NOT reset the GSP-RM queue state here - the driver is still
+	 * sending cleanup frees on it. Only reset on genuine queue death
+	 * (garbage elements, insane writePtr) or ENGINE reset. */
+}
+
+/*
+ * gsp_emu_poll_boot - drive the guest's physical GSP boot to completion.
+ *
+ * The guest driver kicks falcon ucodes (FWSEC for FRTS, then Booter Load for
+ * GSP-RM) by writing STARTCPU to the GSP falcon CPUCTL, then polls for HALTED
+ * plus stage-specific result registers. The auto-boot waits (GFW) are already
+ * satisfied by the state pre-set in nv_vgpu_bar_emul_init. Here we watch for
+ * STARTCPU kicks and complete each stage so the guest believes its falcon
+ * ucodes ran successfully.
+ */
+static void gsp_emu_poll_boot(struct gsp_emu_context *ctx)
+{
+	u32 gsp_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_CPUCTL);
+	u32 sec2_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_SEC2_CPUCTL);
+	u32 engine = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_ENGINE);
+	u32 sw_vector;
+
+	/* Software-interrupt trigger: the driver writes a vector to
+	 * CPU_INTR_LEAF_TRIGGER to fire it (the osVerifyInterrupts sanity
+	 * test uses the doorbell vector). Set the pending bits in the LEAF
+	 * and TOP interrupt-tree registers and inject MSI-X so the guest's
+	 * ISR runs - without this the sanity test times out and
+	 * osVerifySystemEnvironment fails, bailing RmInitAdapter. */
+	sw_vector = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER);
+	if (sw_vector) {
+		u32 leaf_idx = sw_vector / 32;
+		u32 leaf_bit = sw_vector % 32;
+		u32 subtree  = sw_vector / 64;
+		u32 top_idx  = subtree / 32;
+		u32 top_bit  = subtree % 32;
+		u32 leaf_val = gsp_emu_shadow_read32(ctx,
+				GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4);
+		u32 top_val  = gsp_emu_shadow_read32(ctx,
+				GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4);
+
+		gsp_emu_shadow_write32(ctx,
+			GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4,
+			leaf_val | (1U << leaf_bit));
+		gsp_emu_shadow_write32(ctx,
+			GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4,
+			top_val | (1U << top_bit));
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER, 0);
+
+		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: SW interrupt vector %u -> LEAF[%u]|=0x%x TOP[%u]|=0x%x, MSI-X",
+			     sw_vector, leaf_idx, 1U << leaf_bit, top_idx,
+			     1U << top_bit);
+		gsp_emu_inject_completion_interrupt(ctx, 0);
+	}
+
+	/* TLB invalidate auto-complete: the driver sets TRIGGER then spins
+	 * polling for it to clear (kgmmuCheckPendingInvalidates_TU102 reads
+	 * GPU_VREG_RD32(NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE) =
+	 * BAR0 + 0xB80000 + 0x30B0; the physical path uses the plain alias).
+	 * Report instant completion or the driver soft-locks spinning inside
+	 * osDevReadReg032. */
+	{
+		static const u32 tlb_inv_regs[] = {
+			GSP_EMU_REG_MMU_INVALIDATE,
+			GSP_EMU_REG_MMU_INVALIDATE_WIN,
+		};
+		unsigned int i;
+
+		for (i = 0; i < ARRAY_SIZE(tlb_inv_regs); i++) {
+			u32 val = gsp_emu_shadow_read32(ctx, tlb_inv_regs[i]);
+
+			if (!(val & GSP_EMU_MMU_INVALIDATE_TRIGGER))
+				continue;
+			gsp_emu_shadow_write32(ctx, tlb_inv_regs[i],
+					       val & ~GSP_EMU_MMU_INVALIDATE_TRIGGER);
+			pr_info_ratelimited("GSP-BOOT: MMU_INVALIDATE[0x%x] trigger auto-cleared (val=0x%x)\n",
+					    tlb_inv_regs[i], val);
+		}
+	}
+
+	/* L2 cache-op auto-complete: kmemsysDoCacheOp_GM107 writes PENDING
+	 * (bit 0) into the NV_UFLUSH_* / VF L2 invalidate registers and spins
+	 * until the polled bits read back 0 (FB flush, L2 sysmem/peermem
+	 * invalidate, L2 clean, L2 flush-dirty, L2 wait-for-sys-reads). On
+	 * Turing the sysmem/peermem invalidate registers go through the
+	 * GPU_VREG_* window (BAR0 + 0xB80000 + NV_VIRTUAL_FUNCTION_PRIV_L2_*),
+	 * so cover both the plain and windowed aliases. The emulated L2 has
+	 * nothing to do, so clear them as soon as they are observed. */
+	{
+		static const u32 uflush_regs[] = {
+			GSP_EMU_REG_UFLUSH_FB_FLUSH,
+			GSP_EMU_REG_UFLUSH_L2_SYSMEM_INV,
+			GSP_EMU_REG_UFLUSH_L2_PEERMEM_INV,
+			GSP_EMU_REG_UFLUSH_L2_CLEAN_COMPTAGS,
+			GSP_EMU_REG_UFLUSH_L2_FLUSH_DIRTY,
+			GSP_EMU_REG_UFLUSH_L2_WAIT_SYS_READS,
+			GSP_EMU_REG_VF_L2_SYSMEM_INV,
+			GSP_EMU_REG_VF_L2_PEERMEM_INV,
+			GSP_EMU_REG_VF_L2_SYSMEM_INV_WIN,
+			GSP_EMU_REG_VF_L2_PEERMEM_INV_WIN,
+		};
+		unsigned int i;
+
+		for (i = 0; i < ARRAY_SIZE(uflush_regs); i++) {
+			u32 val = gsp_emu_shadow_read32(ctx,
+							uflush_regs[i]);
+
+			if (!val)
+				continue;
+			gsp_emu_shadow_write32(ctx, uflush_regs[i], 0);
+			pr_info_ratelimited("GSP-BOOT: UFLUSH[0x%x] 0x%x auto-cleared\n",
+					    uflush_regs[i], val);
+		}
+	}
+
+	/* CrashCat wayfinder suppression: if the GSP falcon's DEBUGINFO
+	 * register holds a valid wayfinder-L0 signature (0xdead), the driver's
+	 * kgspHealthCheck (run on every RPC drain) reads "reports" from
+	 * garbage and loops forever in nv_open_q, which then can't poll RPCs
+	 * (fn-10 FREE timeouts). Force it invalid - our emulated GSP has no
+	 * crashcat reports to offer. */
+	{
+		u32 wfl0 = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO);
+
+		if ((wfl0 & 0xffff) == 0xdead) {
+			gsp_emu_shadow_write32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO, 0);
+			nv_vgpu_info(ctx->vgpu, "GSP-BOOT: cleared CrashCat wayfinder (was 0x%x)",
+				     wfl0);
+		}
+	}
+
+	/* Temporary: watch PCFG / link-cap mirror offsets for guest writes */
+	{
+		static const u32 watch[] = {
+			0x000004, 0x000084, 0x880000, 0x880004, 0x880080,
+			0x880084, 0x880088, 0x88008c, 0xb80084, 0xb88084,
+			0xc08084,
+		};
+		static u32 last[ARRAY_SIZE(watch)];
+		unsigned int i;
+
+		for (i = 0; i < ARRAY_SIZE(watch); i++) {
+			u32 v = gsp_emu_shadow_read32(ctx, watch[i]);
+
+			if (v != last[i]) {
+				nv_vgpu_info(ctx->vgpu, "GSP-BOOT: PCFG[0x%x] = 0x%x (was 0x%x)",
+					     watch[i], v, last[i]);
+				last[i] = v;
+			}
+		}
+	}
+
+	/* GSP ENGINE reset asserted -> falcon rebooted, restart stage tracking */
+	if ((engine & 0x1) && ctx->boot_stage != GSP_BOOT_STAGE_IDLE) {
+		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: falcon reset, back to IDLE");
+		/* ENGINE reset is a full GPU reset: the GSP-RM queue state is
+		 * stale, reset it too. */
+		gsp_emu_clear_boot_state(ctx, "ENGINE reset");
+		gsp_rm_emu_reset(ctx);
+		return;
+	}
+
+	/*
+	 * PRAMIN window translation: the guest programs NV_PBUS_BAR0_WINDOW
+	 * (BAR0+0x1700) to select which vidmem page the PRAMIN window
+	 * (BAR0+0x700000) maps to. On real hardware the PRAMIN window is a
+	 * CPU aperture into vidmem at (BASE << 16) + offset. In our shadow
+	 * the PRAMIN window is just part of the shadow buffer - it does NOT
+	 * alias the emulated BAR1 (vidmem).
+	 *
+	 * The BAR2 verify test writes through the vBar2 aperture (PRAMIN)
+	 * and reads back through the BAR0 window (also PRAMIN), expecting
+	 * the same data. Both paths hit the same shadow location, so the
+	 * test data IS consistent. We do NOT sync the PRAMIN window from
+	 * the emulated BAR1 - the sync would overwrite the test data
+	 * written by the driver through the vBar2 aperture.
+	 *
+	 * The BAR0_WINDOW_BASE register is captured for observability only.
+	 */
+	{
+		u32 bar0_win = gsp_emu_shadow_read32(ctx, 0x1700);
+		u32 base = bar0_win & 0xFFFFFF;
+		u32 target = (bar0_win >> 24) & 0x3;
+
+		if (!ctx->bar0_window_valid ||
+		    base != ctx->bar0_window_base ||
+		    target != ctx->bar0_window_target) {
+			static u32 last_pramin_val;
+			u32 pramin_val = gsp_emu_shadow_read32(ctx, 0x70e000);
+
+			ctx->bar0_window_base = base;
+			ctx->bar0_window_target = target;
+			ctx->bar0_window_valid = true;
+			nv_vgpu_info(ctx->vgpu,
+				"GSP-BOOT: BAR0_WINDOW base=0x%x target=%u, PRAMIN[0xe000]=0x%08x (was 0x%08x)",
+				base, target, pramin_val, last_pramin_val);
+			last_pramin_val = pramin_val;
+		}
+	}
+
+	/*
+	 * FWSEC kick on the GSP falcon (FRTS setup). Complete it:
+	 * - CPUCTL: clear STARTCPU, set HALTED
+	 * - VBIOS_SCRATCH[0x0E] FRTS_ERR_CODE = NONE (0)
+	 * - WPR2_ADDR_HI != 0, WPR2_ADDR_LO = frtsOffset >> 0xc (in _VAL bits31:4)
+	 */
+	if ((gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU) &&
+	    ctx->boot_stage == GSP_BOOT_STAGE_IDLE) {
+		/* A fresh FWSEC kick means a new GSP boot: drop any GSP-RM
+		 * queue state left from the previous probe (the driver has
+		 * already torn down its old queue). Without this the next
+		 * Booter completion would skip the chain re-parse
+		 * (rm->parsed still set) and GspStatusQueueInit would time
+		 * out on re-probe. */
+		gsp_rm_emu_reset(ctx);
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_VBIOS_SCRATCH_0E, 0);
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0x100000);
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO,
+				       ctx->wpr2_addr_lo);
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL,
+				       GSP_EMU_CPUCTL_HALTED);
+		ctx->boot_stage = GSP_BOOT_STAGE_FWSEC_DONE;
+		ctx->flcn_kick_count++;
+		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: FWSEC kick completed "
+			     "(WPR2_LO=0x%x)", ctx->wpr2_addr_lo);
+		return;
+	}
+
+	/*
+	 * Booter Load kick on the SEC2 falcon (loads/authenticates the GSP-RM
+	 * image and starts the GSP RISCV). Complete it:
+	 * - SEC2 CPUCTL: clear STARTCPU, set HALTED
+	 * - SEC2 MAILBOX0 = 0 (booter error code, 0 = success)
+	 * - GSP RISCV_STATUS = ACTIVE (Booter started the RISCV;
+	 *   kflcnIsRiscvActive passes)
+	 */
+	if ((sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) &&
+	    ctx->boot_stage == GSP_BOOT_STAGE_FWSEC_DONE) {
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0);
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0x1);
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL,
+				       GSP_EMU_CPUCTL_HALTED);
+		ctx->boot_stage = GSP_BOOT_STAGE_BOOTER_DONE;
+		ctx->flcn_kick_count++;
+		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: Booter kick completed, "
+			     "RISCV active");
+		/* The driver has programmed the libos init args address into
+		 * GSP MAILBOX0/1 by now; parse the queue chain, link the
+		 * status queue and emit GSP_INIT_DONE. */
+		gsp_rm_emu_on_booter_done(ctx);
+		return;
+	}
+
+	/* Any other STARTCPU kick (resume/unload/etc.): complete it.
+	 * SEC2 kicks report their ucode result in SEC2_MAILBOX0 (0=success) -
+	 * e.g. Booter Unload during GPU shutdown. Without clearing it the
+	 * driver reads a stale/garbage error code (0xff) and the unload
+	 * fails, killing the subsequent re-init. A SEC2 kick after a
+	 * completed boot is a GSP shutdown/reload: drop the boot state
+	 * (GSP no longer active, queue dead). Clear boot state FIRST so
+	 * WPR2 is already down when the driver sees CPUCTL=HALTED and
+	 * checks kgspIsWpr2Up (avoids "Booter Unload: WPR2 is still up"). */
+	if (gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU)
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL,
+				       GSP_EMU_CPUCTL_HALTED);
+	if (sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) {
+		if (ctx->boot_stage == GSP_BOOT_STAGE_BOOTER_DONE)
+			gsp_emu_clear_boot_state(ctx, "Booter unload/reload kick");
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0);
+		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL,
+				       GSP_EMU_CPUCTL_HALTED);
+	}
+}
+
+/* ============================================================
+ * RPC Processing
+ * ============================================================ */
+
+static int gsp_emu_dispatch_rpc(struct gsp_emu_context *ctx,
+                                 struct gsp_rpc_msg *req,
+                                 struct gsp_rpc_response *rsp)
+{
+    int ret = 0;
+    ktime_t start = ktime_get();
+
+    memset(rsp, 0, sizeof(*rsp));
+    rsp->type = req->type;
+    rsp->seq_id = req->seq_id;
+
+    switch (req->type) {
+    case GSP_RPC_ALLOC: {
+        struct gsp_alloc_req *areq = (struct gsp_alloc_req *)req->payload;
+        struct gsp_alloc_rsp arsp = {0};
+        u64 gpu_va = 0, handle = 0, actual_size = 0;
+
+        ret = gsp_rpc_alloc(ctx->rpc_client,
+                            areq->size, areq->alignment, areq->mem_type,
+                            areq->flags, areq->addr_hint, areq->pid,
+                            &gpu_va, &handle, &actual_size);
+        if (!ret) {
+            arsp.gpu_va = gpu_va;
+            arsp.handle = handle;
+            arsp.size = actual_size;
+            memcpy(rsp->result, &arsp, sizeof(arsp));
+        }
+        break;
+    }
+
+    case GSP_RPC_FREE: {
+        u64 handle = req->payload[0];
+        ret = gsp_rpc_free(ctx->rpc_client, handle);
+        break;
+    }
+
+    case GSP_RPC_MAP: {
+        struct gsp_map_req *mreq = (struct gsp_map_req *)req->payload;
+        struct gsp_map_rsp mrsp = {0};
+        u64 mapped_va = 0;
+
+        ret = gsp_rpc_map(ctx->rpc_client,
+                          mreq->handle, mreq->gpu_va, mreq->size,
+                          mreq->prot, mreq->flags, mreq->offset,
+                          &mapped_va);
+        if (!ret) {
+            mrsp.gpu_va = mapped_va;
+            memcpy(rsp->result, &mrsp, sizeof(mrsp));
+        }
+        break;
+    }
+
+    case GSP_RPC_UNMAP: {
+        u64 handle = req->payload[0];
+        u64 gpu_va = req->payload[1];
+        u64 size = req->payload[2];
+        ret = gsp_rpc_unmap(ctx->rpc_client, handle, gpu_va, size);
+        break;
+    }
+
+    case GSP_RPC_PIN: {
+        struct gsp_pin_req *preq = (struct gsp_pin_req *)req->payload;
+        struct gsp_pin_rsp prsp = {0};
+        u64 dma_addr = 0;
+        u32 num_pages = 0;
+
+        ret = gsp_rpc_pin(ctx->rpc_client,
+                          preq->handle, preq->offset, preq->size,
+                          preq->flags, &dma_addr, &num_pages);
+        if (!ret) {
+            prsp.dma_addr = dma_addr;
+            prsp.num_pages = num_pages;
+            memcpy(rsp->result, &prsp, sizeof(prsp));
+        }
+        break;
+    }
+
+    case GSP_RPC_UNPIN: {
+        u64 handle = req->payload[0];
+        u64 offset = req->payload[1];
+        u64 size = req->payload[2];
+        ret = gsp_rpc_unpin(ctx->rpc_client, handle, offset, size);
+        break;
+    }
+
+    case GSP_RPC_SCHEDULE: {
+        struct gsp_sched_req *sreq = (struct gsp_sched_req *)req->payload;
+        ret = gsp_rpc_schedule(ctx->rpc_client,
+                               sreq->work_handle, sreq->callback, sreq->arg,
+                               sreq->priority, sreq->flags);
+        break;
+    }
+
+    case GSP_RPC_YIELD: {
+        ret = gsp_rpc_yield(ctx->rpc_client);
+        break;
+    }
+
+    case GSP_RPC_NOTIFY: {
+        /* NOTIFY is handled via registration, not direct RPC */
+        nv_vgpu_info(ctx->vgpu, "NOTIFY RPC received (type=0x%x)", req->subcmd);
+        ret = 0;
+        break;
+    }
+
+    case GSP_RPC_BAR_TRAP: {
+        struct gsp_bar_trap_req *breq = (struct gsp_bar_trap_req *)req->payload;
+        struct gsp_bar_trap_rsp brsp = {0};
+        u64 ret_value = 0;
+
+        ret = gsp_rpc_bar_trap(ctx->rpc_client,
+                               breq->bar_addr, breq->access_type, breq->size,
+                               breq->value, breq->bar_num, &ret_value);
+        if (!ret) {
+            brsp.value = ret_value;
+            brsp.handled = 1;
+            memcpy(rsp->result, &brsp, sizeof(brsp));
+        }
+        break;
+    }
+
+    case GSP_RPC_MIGRATE: {
+        struct gsp_migrate_req *mreq = (struct gsp_migrate_req *)req->payload;
+        struct gsp_migrate_rsp mrsp = {0};
+        u64 checkpoint = 0;
+
+        if (mreq->flags == GSP_MIGRATE_SUBCMD_SAVE) {
+            ret = gsp_rpc_migrate_save(ctx->rpc_client,
+                                       mreq->src_gpu_id, mreq->dst_gpu_id,
+                                       mreq->memory_bitmap, mreq->page_table_ptr,
+                                       &checkpoint);
+            if (!ret) {
+                mrsp.checkpoint = checkpoint;
+                memcpy(rsp->result, &mrsp, sizeof(mrsp));
+            }
+        } else if (mreq->flags == GSP_MIGRATE_SUBCMD_RESTORE) {
+            ret = gsp_rpc_migrate_restore(ctx->rpc_client,
+                                          mreq->src_gpu_id, mreq->dst_gpu_id,
+                                          mreq->memory_bitmap, mreq->page_table_ptr,
+                                          mreq->context);
+        }
+        break;
+    }
+
+    case GSP_RPC_DEBUG: {
+        nv_vgpu_info(ctx->vgpu, "DEBUG RPC received (seq=%u)", req->seq_id);
+        ret = 0;
+        break;
+    }
+
+    default:
+        nv_vgpu_warn(ctx->vgpu, "Unknown RPC type: 0x%x", req->type);
+        rsp->status = -ENOTSUPP;
+        ret = -ENOTSUPP;
+        break;
+    }
+
+    if (ret)
+        rsp->status = ret;
+
+    ctx->total_latency_ns += ktime_to_ns(ktime_sub(ktime_get(), start));
+    return ret;
+}
+
+/* ============================================================
+ * MSI-X Interrupt Injection
+ * ============================================================ */
+
+void gsp_emu_inject_completion_interrupt(struct gsp_emu_context *ctx, u32 seq_id)
+{
+    struct eventfd_ctx *trigger = NULL;
+
+    /* Write to MSI-X PBA (Pending Bit Array) at BAR0+0x3000 */
+    if (ctx->bar0_vaddr) {
+        u32 *pba = (u32 *)(ctx->bar0_vaddr + GSP_EMU_MSIX_PBA_OFFSET);
+        writel(1, pba); /* Set pending bit for vector 0 */
+    }
+
+    /* Look up the vector-0 trigger LIVE: QEMU wires eventfds via
+     * VFIO_DEVICE_SET_IRQS after gsp_emu_init runs, so an init-time copy
+     * would be stale (NULL). */
+    if (ctx->vgpu)
+        trigger = ctx->vgpu->irqs[0].trigger;
+
+    /* Signal eventfd for VFIO interrupt delivery */
+    if (trigger)
+        eventfd_signal(trigger);
+}
+
+/* ============================================================
+ * Emulation Thread
+ * ============================================================ */
+
+static int gsp_emu_thread_fn(void *data)
+{
+    struct gsp_emu_context *ctx = data;
+    struct gsp_rpc_msg req;
+    struct gsp_rpc_response rsp;
+    u32 doorbell_val;
+    int ret;
+
+    nv_vgpu_info(ctx->vgpu, "GSP emulation thread started");
+
+    while (!kthread_should_stop()) {
+        /* Read doorbell */
+        doorbell_val = gsp_emu_read_doorbell(ctx);
+
+        /* Watch GSP falcon boot registers (physical GSP boot tracing) */
+        gsp_emu_poll_falcon_watch(ctx);
+
+        /* Drive the guest's physical GSP boot to completion */
+        gsp_emu_poll_boot(ctx);
+
+        /* GSP-RM msgq emulation (physical path: sysmem cmd/status queues) */
+        gsp_rm_emu_poll(ctx);
+
+        if (doorbell_val != 0) {
+            /* ACK doorbell immediately */
+            gsp_emu_write_doorbell(ctx, 0);
+
+            /* Copy request from shared memory */
+            gsp_emu_copy_request_from_shared(ctx, &req);
+
+            /* Validate sequence ID */
+            if (req.seq_id == 0) {
+                nv_vgpu_warn(ctx->vgpu, "Invalid seq_id=0, ignoring RPC");
+                continue;
+            }
+
+            /* Process RPC */
+            ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp);
+
+            /* Write response to shared memory */
+            gsp_emu_copy_response_to_shared(ctx, &rsp);
+
+            /* Inject completion interrupt */
+            gsp_emu_inject_completion_interrupt(ctx, req.seq_id);
+
+            if (ret)
+                ctx->failed_rpcs++;
+            else
+                ctx->processed_rpcs++;
+        } else {
+            /* Adaptive polling: busy-wait first, then sleep (interruptible for kthread_stop) */
+            static int poll_count = 0;
+            if (poll_count < GSP_EMU_POLL_BUSY_US) {
+                udelay(1);
+                poll_count++;
+            } else {
+                /* Use interruptible sleep so kthread_stop can wake us */
+                schedule_timeout_interruptible(usecs_to_jiffies(GSP_EMU_POLL_SLEEP_MAX_US));
+                poll_count = 0;
+            }
+        }
+    }
+
+    nv_vgpu_info(ctx->vgpu, "GSP emulation thread stopped (processed=%llu, failed=%llu)",
+                 ctx->processed_rpcs, ctx->failed_rpcs);
+    complete(&ctx->thread_stopped);
+    return 0;
+}
+
+/* ============================================================
+ * Initialization & Cleanup
+ * ============================================================ */
+
+int gsp_emu_init(struct nv_vgpu_device *vgpu)
+{
+    struct nv_vgpu_bar_emul_ctx *bar_ctx;
+    struct gsp_emu_context *ctx;
+    int ret = 0;
+
+    nv_vgpu_info(vgpu, "Initializing GSP emulation engine");
+
+    ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+    if (!ctx)
+        return -ENOMEM;
+
+    ctx->vgpu = vgpu;
+    ctx->running = false;
+    ctx->initialized = false;
+    ctx->stopping = false;
+    init_completion(&ctx->thread_stopped);
+    spin_lock_init(&ctx->lock);
+
+    /* Physical GSP boot emulation state. wpr2_addr_lo matches what the
+     * guest driver expects WPR2_ADDR_LO to read after FWSEC FRTS:
+     * frtsOffset = fbSize - reservedFromEnd. With fbSize = 256MB
+     * (LOCAL_MEMORY_RANGE) and the 570.144 image's 2MB reservation,
+     * frtsOffset = 0x10000000 - 0x200000 = 0xfe00000; the register's
+     * _VAL field (bits31:4) must equal frtsOffset >> 12 = 0xfe00, so the
+     * raw register value is 0xfe000. */
+    ctx->boot_stage = 0;    /* GSP_BOOT_STAGE_IDLE */
+    ctx->flcn_kick_count = 0;
+    ctx->wpr2_addr_lo = 0x000fe000;
+
+    /* The guest's BAR0 is the emulation shadow (vzalloc'd by
+     * nv_vgpu_bar_emul_init, mapped to the guest via VFIO mmap) - NOT the
+     * host physical BAR0 (vgpu->bars[0].vaddr, which belongs to the real GPU
+     * and the host nvidia driver). All guest doorbell writes, RPC messages,
+     * MSI-X table writes and PBA reads land in the shadow, so the emulation
+     * engine must operate on the shadow exclusively. Polling the physical
+     * BAR0 would never see guest traffic and would corrupt host GPU state. */
+    bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
+    if (!bar_ctx || !bar_ctx->bar0_shadow) {
+        nv_vgpu_err(vgpu, "BAR0 emulation shadow not available");
+        ret = -ENODEV;
+        goto err_free_ctx;
+    }
+
+    ctx->bar0_vaddr = (void __iomem *)bar_ctx->bar0_shadow;
+
+    /* Doorbell register at BAR0+0x110C00 (guest-visible shadow) */
+    ctx->doorbell_reg = ctx->bar0_vaddr + GSP_EMU_DOORBELL_OFFSET;
+
+    /* RPC shared memory at BAR0+0x1000. This ALIASES the guest-visible
+     * shadow - it is NOT a separate allocation. shared_mem_dma stays 0 to
+     * mark it as aliased so cleanup does not dma_free it. */
+    ctx->shared_mem_size = GSP_EMU_SHARED_MEM_SIZE;
+    ctx->shared_mem = ctx->bar0_vaddr + GSP_EMU_SHARED_MEM_OFFSET;
+    ctx->shared_mem_virt = (void *)ctx->shared_mem;
+    ctx->shared_mem_dma = 0;
+
+    /* Initialize RPC client with the shadow BAR0 base and doorbell */
+    ctx->rpc_client = gsp_client_init_emulation(&vgpu->dev, ctx->bar0_vaddr, ctx->doorbell_reg);
+    if (IS_ERR(ctx->rpc_client)) {
+        ret = PTR_ERR(ctx->rpc_client);
+        nv_vgpu_err(vgpu, "Failed to init GSP RPC client: %d", ret);
+        goto err_free_ctx;
+    }
+
+    /* Enable emulation mode in RPC client */
+    gsp_rpc_emulation_set(true);
+
+    /* Initialize GSP-RM msgq emulation (physical path) */
+    ret = gsp_rm_emu_init(vgpu, ctx);
+    if (ret) {
+        nv_vgpu_err(vgpu, "Failed to init GSP-RM emulation: %d", ret);
+        goto err_fini_client;
+    }
+
+    /* NOTE: MSI-X eventfd triggers are NOT copied here - QEMU wires them
+     * via VFIO_DEVICE_SET_IRQS later, so gsp_emu_inject_completion_interrupt
+     * looks them up live from vgpu->irqs[]. */
+
+    /* Start emulation thread */
+    ctx->thread = kthread_run(gsp_emu_thread_fn, ctx, "gsp_emu_%u", vgpu->vgpu_id);
+    if (IS_ERR(ctx->thread)) {
+        ret = PTR_ERR(ctx->thread);
+        nv_vgpu_err(vgpu, "Failed to start GSP emulation thread: %d", ret);
+        goto err_fini_client;
+    }
+    /* Hold a reference on the thread's task_struct so it stays valid
+     * even if the thread crashes (e.g. fortify_panic). Without this,
+     * cleanup could dereference a freed task_struct. */
+    get_task_struct(ctx->thread);
+
+    ctx->running = true;
+    ctx->initialized = true;
+    vgpu->gsp_comm_ctx = ctx; /* Reuse existing field for emulation context */
+
+    nv_vgpu_info(vgpu, "GSP emulation engine initialized successfully");
+    return 0;
+
+err_fini_client:
+    gsp_client_fini(ctx->rpc_client);
+err_free_ctx:
+    /* shared_mem_virt aliases the BAR0 shadow (owned by bar_emulation),
+     * nothing to free here */
+    kfree(ctx);
+    return ret;
+}
+
+void gsp_emu_cleanup(struct nv_vgpu_device *vgpu)
+{
+    struct gsp_emu_context *ctx = vgpu->gsp_comm_ctx;
+
+    if (!ctx)
+        return;
+
+    nv_vgpu_info(vgpu, "Cleaning up GSP emulation engine");
+
+    /* Stop emulation thread with timeout */
+    if (ctx->thread) {
+        ctx->stopping = true;
+        /* Only call kthread_stop() if the thread is still alive.
+         * If the thread crashed (e.g. fortify_panic), the kernel already
+         * killed it and kthread_stop() would hit a refcount saturation
+         * warning on the dead task_struct. */
+        if (pid_alive(ctx->thread)) {
+            kthread_stop(ctx->thread);
+            /* Wait for thread to actually stop with timeout (5 seconds) */
+            if (!wait_for_completion_timeout(&ctx->thread_stopped, msecs_to_jiffies(5000))) {
+                nv_vgpu_warn(vgpu, "GSP emulation thread did not stop in time, forcing cleanup");
+            }
+        } else {
+            nv_vgpu_warn(vgpu, "GSP emulation thread already dead (crashed), skipping kthread_stop");
+        }
+        put_task_struct(ctx->thread);
+        ctx->thread = NULL;
+    }
+    ctx->running = false;
+
+    /* Teardown GSP-RM msgq emulation */
+    gsp_rm_emu_fini(ctx);
+
+    /* Disable emulation mode */
+    gsp_rpc_emulation_set(false);
+
+    /* Cleanup RPC client */
+    if (ctx->rpc_client) {
+        gsp_client_fini(ctx->rpc_client);
+        ctx->rpc_client = NULL;
+    }
+
+    /* Shared memory aliases the BAR0 shadow (owned by bar_emulation) -
+     * only free it if it was a standalone DMA allocation (dma != 0) */
+    if (ctx->shared_mem_virt && ctx->shared_mem_dma) {
+        dma_free_wc(&vgpu->pdev->dev, ctx->shared_mem_size,
+                    ctx->shared_mem_virt, ctx->shared_mem_dma);
+    }
+    ctx->shared_mem_virt = NULL;
+
+    /* BAR0 shadow is managed by nv_vgpu_bar_emul_init/cleanup */
+    ctx->bar0_vaddr = NULL;
+
+    vgpu->gsp_comm_ctx = NULL;
+    kfree(ctx);
+
+    nv_vgpu_info(vgpu, "GSP emulation engine cleaned up");
+}
+
+/* ============================================================
+ * Public RPC Processing (for external callers)
+ * ============================================================ */
+
+int gsp_emu_process_rpc(struct gsp_emu_context *ctx)
+{
+    struct gsp_rpc_msg req;
+    struct gsp_rpc_response rsp;
+    int ret;
+
+    if (!ctx || !ctx->initialized)
+        return -ENODEV;
+
+    gsp_emu_copy_request_from_shared(ctx, &req);
+
+    if (req.seq_id == 0)
+        return -EINVAL;
+
+    ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp);
+
+    gsp_emu_copy_response_to_shared(ctx, &rsp);
+    gsp_emu_inject_completion_interrupt(ctx, req.seq_id);
+
+    return ret;
+}
+
+/* ============================================================
+ * Statistics
+ * ============================================================ */
+
+void gsp_emu_get_stats(struct gsp_emu_context *ctx,
+                       u64 *processed, u64 *failed, u64 *total_latency_ns)
+{
+    if (!ctx)
+        return;
+
+    if (processed)
+        *processed = ctx->processed_rpcs;
+    if (failed)
+        *failed = ctx->failed_rpcs;
+    if (total_latency_ns)
+        *total_latency_ns = ctx->total_latency_ns;
+}
+
+EXPORT_SYMBOL(gsp_emu_process_rpc);
+EXPORT_SYMBOL(gsp_emu_get_stats);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Open-IOV Project");
+MODULE_DESCRIPTION("GSP Emulation Engine for Host-Side vGPU");
\ No newline at end of file