Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 578 insertions, 0 deletions

diff --git a/kernel/bar_emulation.c b/kernel/bar_emulation.c
new file mode 100644
index 0000000..685831a
--- /dev/null
+++ b/kernel/bar_emulation.c
@@ -0,0 +1,578 @@
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/io.h>
+#include <linux/vmalloc.h>
+#include <linux/pci.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#include <linux/eventfd.h>
+#include <linux/firmware.h>
+
+#include "nvidia-vgpu-vfio.h"
+#include "bar_emulation.h"
+#include "gsp_comm.h"
+#include "gsp_emulation.h"
+
+#define NV_VGPU_BAR0_TRAP_REGIONS 64
+
+/* VBIOS (PROM window) emulation - see nv_vgpu_bar_emul_init */
+#define NV_VGPU_VBIOS_FW_NAME		"tu104_vbios.bin"
+#define NV_VGPU_PROM_WINDOW_OFFSET	0x300000
+#define NV_VGPU_VBIOS_MAX_SIZE		0x100000  /* 1MB (s_getBaseBiosMaxSize_TU102) */
+
+/* MSI-X Table entry size (16 bytes: addr_lo, addr_hi, data, vector_control) */
+#define NV_VGPU_MSIX_ENTRY_SIZE 16
+#define NV_VGPU_MSIX_TABLE_ENTRIES 32
+#define NV_VGPU_MSIX_TABLE_SIZE (NV_VGPU_MSIX_ENTRY_SIZE * NV_VGPU_MSIX_TABLE_ENTRIES)
+#define NV_VGPU_MSIX_PBA_SIZE 8  /* 64 bits for 32 vectors */
+
+static const struct {
+	u32 offset;
+	u32 size;
+	const char *name;
+} nv_vgpu_bar0_traps[] = {
+	{ 0x000000, 0x1000, "PMC" },
+	{ 0x001000, 0x1000, "PRAMIN" },
+	{ 0x002000, 0x1000, "PCR" },
+	{ 0x003000, 0x1000, "PRAMDAC" },
+	{ 0x004000, 0x1000, "PVIO" },
+	{ 0x005000, 0x1000, "PFIFO" },
+	{ 0x006000, 0x1000, "PGRAPH" },
+	{ 0x007000, 0x1000, "PTIMER" },
+	{ 0x008000, 0x1000, "PFUSE" },
+	{ 0x009000, 0x1000, "PBUS" },
+	{ 0x00A000, 0x1000, "PCOP" },
+	{ 0x00B000, 0x1000, "PVP" },
+	{ 0x00C000, 0x1000, "PCE" },
+	{ 0x00D000, 0x1000, "PDISP" },
+	{ 0x00E000, 0x1000, "PNVDEC" },
+	{ 0x00F000, 0x1000, "PNVENC" },
+	{ 0x010000, 0x1000, "PSEC2" },
+	{ 0x011000, 0x1000, "PMU" },
+	{ 0x012000, 0x1000, "PGFX" },
+	{ 0x013000, 0x1000, "PXVE" },
+	{ 0x020000, 0x10000, "FIFO" },
+	{ 0x040000, 0x10000, "GR" },
+	{ 0x050000, 0x10000, "CE" },
+	{ 0x060000, 0x10000, "NVDEC" },
+	{ 0x070000, 0x10000, "NVENC" },
+	{ 0x080000, 0x10000, "NVJPG" },
+	{ 0x090000, 0x10000, "OFA" },
+	{ 0x100000, 0x10000, "FB" },
+	{ 0x110000, 0x10000, "LTC" },
+	{ 0x120000, 0x10000, "GPC" },
+	{ 0x130000, 0x10000, "TPC" },
+	{ 0x140000, 0x10000, "ROP" },
+	{ 0x150000, 0x10000, "HUB" },
+	{ 0x160000, 0x10000, "NVLINK" },
+	{ 0x170000, 0x10000, "GSP" },
+	{ 0x180000, 0x10000, "RAP" },
+	{ 0x190000, 0x10000, "SEC" },
+	{ 0x1A0000, 0x10000, "PMU_RISCV" },
+	{ 0x1B0000, 0x10000, "FALCON" },
+	{ 0x1C0000, 0x10000, "XVE" },
+	{ 0x1D0000, 0x10000, "DMCE" },
+	{ 0x1E0000, 0x10000, "P2P" },
+	{ 0x1F0000, 0x10000, "MISC" },
+};
+
+static void nv_vgpu_msix_init(struct nv_vgpu_bar_emul_ctx *ctx)
+{
+	memset(ctx->msix_table, 0, sizeof(ctx->msix_table));
+	ctx->msix_pba = 0;
+	spin_lock_init(&ctx->msix_lock);
+	ctx->msix_enabled = false;
+}
+
+static void nv_vgpu_msix_write_table(struct nv_vgpu_bar_emul_ctx *ctx,
+				      u32 offset, u64 val, u32 size)
+{
+	u32 entry_idx = offset / NV_VGPU_MSIX_ENTRY_SIZE;
+	u32 entry_offset = offset % NV_VGPU_MSIX_ENTRY_SIZE;
+	
+	if (entry_idx >= NV_VGPU_MSIX_TABLE_ENTRIES)
+		return;
+	
+	spin_lock(&ctx->msix_lock);
+	
+	switch (entry_offset) {
+	case 0x0:  /* Message Address Low */
+		ctx->msix_table[entry_idx].addr = (ctx->msix_table[entry_idx].addr & 0xFFFFFFFF00000000ULL) | val;
+		break;
+	case 0x4:  /* Message Address High */
+		ctx->msix_table[entry_idx].addr = (ctx->msix_table[entry_idx].addr & 0x00000000FFFFFFFFULL) | (val << 32);
+		break;
+	case 0x8:  /* Message Data */
+		ctx->msix_table[entry_idx].data = (u32)val;
+		break;
+	case 0xC:  /* Vector Control */
+		ctx->msix_table[entry_idx].vector_control = (u32)val;
+		break;
+	}
+	
+	spin_unlock(&ctx->msix_lock);
+}
+
+static u64 nv_vgpu_msix_read_table(struct nv_vgpu_bar_emul_ctx *ctx,
+				    u32 offset, u32 size)
+{
+	u32 entry_idx = offset / NV_VGPU_MSIX_ENTRY_SIZE;
+	u32 entry_offset = offset % NV_VGPU_MSIX_ENTRY_SIZE;
+	u64 val = 0;
+	
+	if (entry_idx >= NV_VGPU_MSIX_TABLE_ENTRIES)
+		return 0;
+	
+	spin_lock(&ctx->msix_lock);
+	
+	switch (entry_offset) {
+	case 0x0:  /* Message Address Low */
+		val = ctx->msix_table[entry_idx].addr & 0xFFFFFFFF;
+		break;
+	case 0x4:  /* Message Address High */
+		val = (ctx->msix_table[entry_idx].addr >> 32) & 0xFFFFFFFF;
+		break;
+	case 0x8:  /* Message Data */
+		val = ctx->msix_table[entry_idx].data;
+		break;
+	case 0xC:  /* Vector Control */
+		val = ctx->msix_table[entry_idx].vector_control;
+		break;
+	}
+	
+	spin_unlock(&ctx->msix_lock);
+	return val;
+}
+
+static u64 nv_vgpu_msix_read_pba(struct nv_vgpu_bar_emul_ctx *ctx, u32 size)
+{
+	u64 val;
+	spin_lock(&ctx->msix_lock);
+	val = ctx->msix_pba;
+	spin_unlock(&ctx->msix_lock);
+	return val;
+}
+
+void nv_vgpu_msix_set_pending(struct nv_vgpu_device *vgpu, int vector)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+	
+	if (!ctx || vector < 0 || vector >= NV_VGPU_MSIX_TABLE_ENTRIES)
+		return;
+	
+	spin_lock(&ctx->msix_lock);
+	ctx->msix_pba |= (1ULL << vector);
+	spin_unlock(&ctx->msix_lock);
+}
+
+void nv_vgpu_msix_clear_pending(struct nv_vgpu_device *vgpu, int vector)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+	
+	if (!ctx || vector < 0 || vector >= NV_VGPU_MSIX_TABLE_ENTRIES)
+		return;
+	
+	spin_lock(&ctx->msix_lock);
+	ctx->msix_pba &= ~(1ULL << vector);
+	spin_unlock(&ctx->msix_lock);
+}
+
+int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx;
+	int i, ret;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->vgpu = vgpu;
+	ctx->bar0_shadow_size = NV_VGPU_BAR0_SIZE;
+	ctx->bar0_shadow = vzalloc(ctx->bar0_shadow_size);
+	if (!ctx->bar0_shadow) {
+		ret = -ENOMEM;
+		goto err_free;
+	}
+
+	spin_lock_init(&ctx->shadow_lock);
+	ctx->trap_count = 0;
+
+	for (i = 0; i < ARRAY_SIZE(nv_vgpu_bar0_traps); i++) {
+		if (ctx->trap_count >= 256)
+			break;
+		ctx->trap_regs[ctx->trap_count].offset = nv_vgpu_bar0_traps[i].offset;
+		ctx->trap_regs[ctx->trap_count].trapped = true;
+		ctx->trap_count++;
+	}
+
+	/* Initialize MSI-X emulation */
+	nv_vgpu_msix_init(ctx);
+
+	/*
+	 * Seed PMC boot registers so the guest driver sees a REAL,
+	 * PHYSICAL, non-virtual TU104 GPU. Values captured from the host's
+	 * physical TU104 (RTX 2080 SUPER, PCI 0000:01:00.0):
+	 *   NV_PMC_BOOT_0  (BAR0+0x000) = 0x164000a1  (arch TU104, rev A1)
+	 *   NV_PMC_BOOT_1  (BAR0+0x004) = 0x00000000  (VGPU field = REAL,
+	 *                                            no PV/VF - the guest
+	 *                                            must take the physical
+	 *                                            probe path, never a
+	 *                                            para-virtual vGPU path)
+	 *   NV_PMC_BOOT_42 (BAR0+0xa00) = 0x164a1000  (chip id / revision)
+	 *
+	 * The guest reads these via the VFIO-mmap'd shadow (and via VFIO
+	 * region reads, which hit the same shadow). Without them the open
+	 * driver's halmgrGetHalForGpu() fails and probe aborts with the
+	 * "does not include the required GPU System Processor (GSP)" error.
+	 */
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x000) = 0x164000a1;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x004) = 0x00000000;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xa00) = 0x164a1000;
+
+	/*
+	 * Pre-set the GSP secure-boot (GFW) completion state.
+	 *
+	 * During RmInitAdapter the guest driver waits for the GPU's
+	 * post-FLR secure boot (kgspWaitForGfwBootOk -> gpuWaitForGfwBootComplete
+	 * -> _gpuIsGfwBootCompleted_TU102 + kflcnWaitForHalt). On real hardware
+	 * the boot ROM runs this automatically after reset; the driver only
+	 * *waits* for it (it never kicks the falcon here). We present the
+	 * "already booted" state so the wait succeeds immediately:
+	 *   0x118128  NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK
+	 *             bit0 READ_PROTECTION_LEVEL0 = ENABLE (FWSEC lowered PLM)
+	 *   0x118234  NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT
+	 *             bits7:0 PROGRESS = 0xFF (COMPLETED)
+	 *   0x110100  GSP falcon CPUCTL bit4 = HALTED
+	 *
+	 * Driver-kicked falcon runs later (FWSEC/Booter) clear CPUCTL via
+	 * STARTCPU; the GSP emulation thread re-completes those (see
+	 * gsp_emu_poll_boot in gsp_emulation.c).
+	 */
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x118128) = 0x00000001;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x118234) = 0x000000ff;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x110100) = 0x00000010;
+
+	/*
+	 * More post-GFW-boot state the driver validates before/while booting
+	 * GSP-RM:
+	 *   0x1100f4  GSP falcon HWCFG2 bit10 RISCV = ENABLE. Checked by
+	 *             kflcnIsRiscvCpuEnabled in kgspPrepareForBootstrap;
+	 *             without it probe dies with "RISC-V core is not enabled".
+	 *   0x009430  NV_PTIMER_TIME_PRIV_LEVEL_MASK bit4
+	 *             WRITE_PROTECTION_LEVEL0 = ENABLE. FWSEC lowers PLMs
+	 *             after GFW boot; driver asserts "Write to PTIMER
+	 *             attempted even though Level 0 PLM is disabled" when
+	 *             setting the GPU clock otherwise.
+	 */
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x1100f4) = 0x00000400;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x009430) = 0x00000010;
+
+	/*
+	 * 0x100ce0  NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE - usable FB size.
+	 * Encoded as fbSize = LOWER_MAG << (LOWER_SCALE + 20). Set to
+	 * 256MB (matches our BAR1): mag=1, scale=8 -> 1 << 28. On real
+	 * hardware VBIOS devinit programs this after GFW boot; the guest
+	 * driver's kgspPopulateWprMeta reads it via kmemsysGetUsableFbSize
+	 * and uses it to compute the FRTS offset (fbSize - reservedFromEnd).
+	 * With it 0, the FRTS offset underflows and FWSEC verification
+	 * fails with "WPR2 initialized at an unexpected location".
+	 */
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x100ce0) = 0x00000018;
+
+	/*
+	 * NV_PCFG (BAR0+0x88000) - PCI config space mirror the driver reads
+	 * via gpuReadBusConfigReg_HAL (GPU_BUS_CFG_RD32):
+	 *   0x88084 NV_XVE_LINK_CAPABILITIES: MAX_LINK_SPEED 3:0 = 3 (8 GT/s,
+	 *           Gen3), MAX_LINK_WIDTH 9:4 = 0x10 (x16).
+	 *   0x88088 NV_XVE_LINK_CONTROL_STATUS: negotiated speed 19:16 = 3,
+	 *           negotiated width 25:20 = 0x10 (x16).
+	 * UVM's nvGpuOpsDeviceCreate -> getPCIELinkRateMBps ->
+	 * kbifControlGetPCIEInfo -> kbifGetGpuLinkCapabilities reads 0x88084;
+	 * zeros make calculatePCIELinkRateMBps log "Unknown PCIe speed" and
+	 * return NV_ERR_INVALID_STATE, which fails cuInit
+	 * (CUDA_ERROR_NOT_INITIALIZED in the guest).
+	 *
+	 * DEVICE_BASE(NV_PCFG) expands to 0 in this driver (nvmisc.h:
+	 * "#define DEVICE_BASE(d) (0?d)"), so the read is actually
+	 * gpuNvAddr + 0x84 - mirror the values at the low alias as well.
+	 */
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x88084) = 0x00000103;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x88088) = 0x01030000;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x00084) = 0x00000103;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x00088) = 0x01030000;
+	/* Same values under the VF register window aliases, in case this
+	 * driver reaches the config mirror through virtualRegPhysOffset. */
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB80084) = 0x00000103;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB80088) = 0x01030000;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB88084) = 0x00000103;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB88088) = 0x01030000;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xC08084) = 0x00000103;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xC08088) = 0x01030000;
+
+	/*
+	 * Per-VF static sizes the driver reads through the VF register window
+	 * (GPU_VREG_RD32 = BAR0 + sriovState.virtualRegPhysOffset, 0xB80000
+	 * for a non-SRIOV virtual GPU). Both are read-only SIZE registers on
+	 * real hardware; we pre-fill them so the local (non-RPC) paths agree
+	 * with the GSP_RM_CONTROL replies in gsp_rm_emulation.c.
+	 *
+	 *  0x3010/0x3030 NV_VIRTUAL_FUNCTION_PRIV_MMU_FAULT_BUFFER_SIZE(0/1):
+	 *      _VAL is in NVC369_BUF_SIZE (32B) units. 0x800 -> 64KB
+	 *      replayable, 0x80 -> 4KB legacy; matches TU104_GMMU_*_SIZE.
+	 *      kgmmuSetAndGetDefaultFaultBufferSize_TU102 writes SET_DEFAULT
+	 *      (bit30) and reads _VAL back, so a pre-filled _VAL is all it
+	 *      needs.
+	 *  0x3110 NV_VIRTUAL_FUNCTION_PRIV_ACCESS_COUNTER_NOTIFY_BUFFER_SIZE:
+	 *      UVM's uvmGetAccessCounterBufferSize_TU102 multiplies it by
+	 *      NV_ACCESS_COUNTER_NOTIFY_BUF_SIZE (32B). 0x1000 -> 128KB, the
+	 *      value the host's physical TU104 reports. Zero makes the
+	 *      access-counter buffer a 0-byte allocation and cuInit fails in
+	 *      uvmInitializeAccessCntrBuffer.
+	 */
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83010) = 0x00000800;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83030) = 0x00000080;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x003010) = 0x00000800;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x003030) = 0x00000080;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83110) = 0x00001000;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0x003110) = 0x00001000;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB8311C) = 0x00000000;
+	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83118) = 0x00000000;
+
+
+	/*
+	 * Serve a valid VBIOS at the PROM window (BAR0+0x300000).
+	 *
+	 * The guest driver's physical GSP boot (kgspInitRm -> TU102 path)
+	 * reads the VBIOS via NV_PROM_DATA (BAR0+0x300000) to extract the
+	 * FWSEC ucode (mandatory on Turing: kgspGetFrtsSize is hardwired
+	 * to 1MB, so FWSEC/FRTS cannot be skipped). Without a valid ROM
+	 * signature the probe aborts with:
+	 *   "kgspExtractVbiosFromRom_TU102: did not find valid ROM signature"
+	 *
+	 * The image is a dump of the host's physical TU104 VBIOS (read via
+	 * its own PROM window), loaded from /lib/firmware/tu104_vbios.bin.
+	 * Non-fatal if absent: probe then fails later in FWSEC, not here.
+	 */
+	{
+		const struct firmware *vbios_fw;
+		int fw_ret = request_firmware(&vbios_fw, NV_VGPU_VBIOS_FW_NAME,
+					      &vgpu->pdev->dev);
+		if (fw_ret == 0) {
+			u32 copy_size = min_t(u32, vbios_fw->size,
+					      NV_VGPU_VBIOS_MAX_SIZE);
+
+			if (NV_VGPU_PROM_WINDOW_OFFSET + copy_size <= ctx->bar0_shadow_size) {
+				memcpy((u8 *)ctx->bar0_shadow + NV_VGPU_PROM_WINDOW_OFFSET,
+				       vbios_fw->data, copy_size);
+				nv_vgpu_info(vgpu, "VBIOS loaded at BAR0+0x%x (%u bytes)",
+					     NV_VGPU_PROM_WINDOW_OFFSET, copy_size);
+			} else {
+				nv_vgpu_warn(vgpu, "VBIOS too large for PROM window (%u bytes)",
+					     copy_size);
+			}
+			release_firmware(vbios_fw);
+		} else {
+			nv_vgpu_warn(vgpu, "VBIOS firmware %s not found (%d); "
+				     "guest GSP boot will fail at ROM parse",
+				     NV_VGPU_VBIOS_FW_NAME, fw_ret);
+		}
+	}
+
+	vgpu->bar_emul_ctx = ctx;
+	return 0;
+
+err_free:
+	kfree(ctx);
+	return ret;
+}
+
+void nv_vgpu_bar_emul_cleanup(struct nv_vgpu_device *vgpu)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+
+	if (!ctx)
+		return;
+
+	if (ctx->bar0_shadow)
+		vfree(ctx->bar0_shadow);
+
+	kfree(ctx);
+	vgpu->bar_emul_ctx = NULL;
+}
+
+int nv_vgpu_bar_trap_register(struct nv_vgpu_device *vgpu, int bar_idx,
+			       u32 offset, u32 size)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+	unsigned long flags;
+	int i;
+
+	if (!ctx || bar_idx != 0)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ctx->shadow_lock, flags);
+
+	for (i = 0; i < ctx->trap_count; i++) {
+		if (ctx->trap_regs[i].offset == offset) {
+			ctx->trap_regs[i].trapped = true;
+			spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+			return 0;
+		}
+	}
+
+	if (ctx->trap_count >= 256) {
+		spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+		return -ENOSPC;
+	}
+
+	ctx->trap_regs[ctx->trap_count].offset = offset;
+	ctx->trap_regs[ctx->trap_count].trapped = true;
+	ctx->trap_count++;
+
+	spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+	return 0;
+}
+
+int nv_vgpu_bar_untrap_register(struct nv_vgpu_device *vgpu, int bar_idx,
+				 u32 offset)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+	unsigned long flags;
+	int i;
+
+	if (!ctx || bar_idx != 0)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ctx->shadow_lock, flags);
+
+	for (i = 0; i < ctx->trap_count; i++) {
+		if (ctx->trap_regs[i].offset == offset) {
+			ctx->trap_regs[i].trapped = false;
+			spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+			return 0;
+		}
+	}
+
+	spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+	return -ENOENT;
+}
+
+int nv_vgpu_bar_emul_read(struct nv_vgpu_device *vgpu, int bar_idx,
+			   u64 offset, void *val, size_t size)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+	unsigned long flags;
+	u32 *val32 = val;
+	u64 *val64 = val;
+
+	if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size)
+		return -EINVAL;
+
+	/* Handle MSI-X Table reads (at BAR0+0x2000) */
+	if (offset >= GSP_EMU_MSIX_TABLE_OFFSET && 
+	    offset < GSP_EMU_MSIX_TABLE_OFFSET + NV_VGPU_MSIX_TABLE_SIZE) {
+		u32 msix_offset = offset - GSP_EMU_MSIX_TABLE_OFFSET;
+		*val64 = nv_vgpu_msix_read_table(ctx, msix_offset, size);
+		return 0;
+	}
+
+	/* Handle MSI-X PBA reads (at BAR0+0x3000) */
+	if (offset >= GSP_EMU_MSIX_PBA_OFFSET && 
+	    offset < GSP_EMU_MSIX_PBA_OFFSET + NV_VGPU_MSIX_PBA_SIZE) {
+		*val64 = nv_vgpu_msix_read_pba(ctx, size);
+		return 0;
+	}
+
+	spin_lock_irqsave(&ctx->shadow_lock, flags);
+
+	switch (size) {
+	case 4:
+		*val32 = *(u32 *)((u8 *)ctx->bar0_shadow + offset);
+		break;
+	case 8:
+		*val64 = *(u64 *)((u8 *)ctx->bar0_shadow + offset);
+		break;
+	default:
+		spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+		return -EINVAL;
+	}
+
+	spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+	return 0;
+}
+
+int nv_vgpu_bar_emul_write(struct nv_vgpu_device *vgpu, int bar_idx,
+			    u64 offset, void *val, size_t size)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+	unsigned long flags;
+	u32 *val32 = val;
+	u64 *val64 = val;
+	bool trapped = false;
+	int i;
+
+	if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size)
+		return -EINVAL;
+
+	/* Handle MSI-X Table writes (at BAR0+0x2000) */
+	if (offset >= GSP_EMU_MSIX_TABLE_OFFSET && 
+	    offset < GSP_EMU_MSIX_TABLE_OFFSET + NV_VGPU_MSIX_TABLE_SIZE) {
+		u32 msix_offset = offset - GSP_EMU_MSIX_TABLE_OFFSET;
+		nv_vgpu_msix_write_table(ctx, msix_offset, *val64, size);
+		return 0;
+	}
+
+	spin_lock_irqsave(&ctx->shadow_lock, flags);
+
+	for (i = 0; i < ctx->trap_count; i++) {
+		if (ctx->trap_regs[i].offset == offset && ctx->trap_regs[i].trapped) {
+			trapped = true;
+			ctx->trap_regs[i].value = (size == 4) ? *val32 : (u32)*val64;
+			ctx->trap_regs[i].is_write = true;
+			break;
+		}
+	}
+
+	switch (size) {
+	case 4:
+		*(u32 *)((u8 *)ctx->bar0_shadow + offset) = *val32;
+		break;
+	case 8:
+		*(u64 *)((u8 *)ctx->bar0_shadow + offset) = *val64;
+		break;
+	default:
+		spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+		return -EINVAL;
+	}
+
+	spin_unlock_irqrestore(&ctx->shadow_lock, flags);
+
+	if (trapped) {
+		nv_vgpu_dbg(vgpu, "BAR0 trap: offset=0x%llx, size=%zu, write=0x%x",
+			    offset, size, (size == 4) ? *val32 : (u32)*val64);
+		nv_vgpu_gsp_rpc_bar_trap(vgpu, offset, 1, size, *(u64 *)val, 0, NULL);
+	}
+
+	return 0;
+}
+
+int nv_vgpu_bar_emul_handle_fault(struct nv_vgpu_device *vgpu,
+				   unsigned long addr, bool write)
+{
+	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
+	struct nv_vgpu_bar *bar = &vgpu->bars[0];
+	u64 offset = addr - (unsigned long)bar->vaddr;
+	u32 val = 0;
+	int ret;
+
+	if (!ctx || !bar->mapped || !bar->trapped)
+		return -EINVAL;
+
+	if (write) {
+		ret = nv_vgpu_bar_emul_write(vgpu, 0, offset, &val, 4);
+	} else {
+		ret = nv_vgpu_bar_emul_read(vgpu, 0, offset, &val, 4);
+	}
+
+	return ret;
+}
\ No newline at end of file