path: root/kernel/bar_emulation.c

#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/io.h>
#include <linux/vmalloc.h>
#include <linux/pci.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/eventfd.h>
#include <linux/firmware.h>

#include "nvidia-vgpu-vfio.h"
#include "bar_emulation.h"
#include "gsp_comm.h"
#include "gsp_emulation.h"

#define NV_VGPU_BAR0_TRAP_REGIONS 64

/* VBIOS (PROM window) emulation - see nv_vgpu_bar_emul_init */
#define NV_VGPU_VBIOS_FW_NAME		"tu104_vbios.bin"
#define NV_VGPU_PROM_WINDOW_OFFSET	0x300000
#define NV_VGPU_VBIOS_MAX_SIZE		0x100000  /* 1MB (s_getBaseBiosMaxSize_TU102) */

/* MSI-X Table entry size (16 bytes: addr_lo, addr_hi, data, vector_control) */
#define NV_VGPU_MSIX_ENTRY_SIZE 16
#define NV_VGPU_MSIX_TABLE_ENTRIES 32
#define NV_VGPU_MSIX_TABLE_SIZE (NV_VGPU_MSIX_ENTRY_SIZE * NV_VGPU_MSIX_TABLE_ENTRIES)
#define NV_VGPU_MSIX_PBA_SIZE 8  /* 64 bits for 32 vectors */

static const struct {
	u32 offset;
	u32 size;
	const char *name;
} nv_vgpu_bar0_traps[] = {
	{ 0x000000, 0x1000, "PMC" },
	{ 0x001000, 0x1000, "PRAMIN" },
	{ 0x002000, 0x1000, "PCR" },
	{ 0x003000, 0x1000, "PRAMDAC" },
	{ 0x004000, 0x1000, "PVIO" },
	{ 0x005000, 0x1000, "PFIFO" },
	{ 0x006000, 0x1000, "PGRAPH" },
	{ 0x007000, 0x1000, "PTIMER" },
	{ 0x008000, 0x1000, "PFUSE" },
	{ 0x009000, 0x1000, "PBUS" },
	{ 0x00A000, 0x1000, "PCOP" },
	{ 0x00B000, 0x1000, "PVP" },
	{ 0x00C000, 0x1000, "PCE" },
	{ 0x00D000, 0x1000, "PDISP" },
	{ 0x00E000, 0x1000, "PNVDEC" },
	{ 0x00F000, 0x1000, "PNVENC" },
	{ 0x010000, 0x1000, "PSEC2" },
	{ 0x011000, 0x1000, "PMU" },
	{ 0x012000, 0x1000, "PGFX" },
	{ 0x013000, 0x1000, "PXVE" },
	{ 0x020000, 0x10000, "FIFO" },
	{ 0x040000, 0x10000, "GR" },
	{ 0x050000, 0x10000, "CE" },
	{ 0x060000, 0x10000, "NVDEC" },
	{ 0x070000, 0x10000, "NVENC" },
	{ 0x080000, 0x10000, "NVJPG" },
	{ 0x090000, 0x10000, "OFA" },
	{ 0x100000, 0x10000, "FB" },
	{ 0x110000, 0x10000, "LTC" },
	{ 0x120000, 0x10000, "GPC" },
	{ 0x130000, 0x10000, "TPC" },
	{ 0x140000, 0x10000, "ROP" },
	{ 0x150000, 0x10000, "HUB" },
	{ 0x160000, 0x10000, "NVLINK" },
	{ 0x170000, 0x10000, "GSP" },
	{ 0x180000, 0x10000, "RAP" },
	{ 0x190000, 0x10000, "SEC" },
	{ 0x1A0000, 0x10000, "PMU_RISCV" },
	{ 0x1B0000, 0x10000, "FALCON" },
	{ 0x1C0000, 0x10000, "XVE" },
	{ 0x1D0000, 0x10000, "DMCE" },
	{ 0x1E0000, 0x10000, "P2P" },
	{ 0x1F0000, 0x10000, "MISC" },
};

static void nv_vgpu_msix_init(struct nv_vgpu_bar_emul_ctx *ctx)
{
	memset(ctx->msix_table, 0, sizeof(ctx->msix_table));
	ctx->msix_pba = 0;
	spin_lock_init(&ctx->msix_lock);
	ctx->msix_enabled = false;
}

static void nv_vgpu_msix_write_table(struct nv_vgpu_bar_emul_ctx *ctx,
				      u32 offset, u64 val, u32 size)
{
	u32 entry_idx = offset / NV_VGPU_MSIX_ENTRY_SIZE;
	u32 entry_offset = offset % NV_VGPU_MSIX_ENTRY_SIZE;
	
	if (entry_idx >= NV_VGPU_MSIX_TABLE_ENTRIES)
		return;
	
	spin_lock(&ctx->msix_lock);
	
	switch (entry_offset) {
	case 0x0:  /* Message Address Low */
		ctx->msix_table[entry_idx].addr = (ctx->msix_table[entry_idx].addr & 0xFFFFFFFF00000000ULL) | val;
		break;
	case 0x4:  /* Message Address High */
		ctx->msix_table[entry_idx].addr = (ctx->msix_table[entry_idx].addr & 0x00000000FFFFFFFFULL) | (val << 32);
		break;
	case 0x8:  /* Message Data */
		ctx->msix_table[entry_idx].data = (u32)val;
		break;
	case 0xC:  /* Vector Control */
		ctx->msix_table[entry_idx].vector_control = (u32)val;
		break;
	}
	
	spin_unlock(&ctx->msix_lock);
}

static u64 nv_vgpu_msix_read_table(struct nv_vgpu_bar_emul_ctx *ctx,
				    u32 offset, u32 size)
{
	u32 entry_idx = offset / NV_VGPU_MSIX_ENTRY_SIZE;
	u32 entry_offset = offset % NV_VGPU_MSIX_ENTRY_SIZE;
	u64 val = 0;
	
	if (entry_idx >= NV_VGPU_MSIX_TABLE_ENTRIES)
		return 0;
	
	spin_lock(&ctx->msix_lock);
	
	switch (entry_offset) {
	case 0x0:  /* Message Address Low */
		val = ctx->msix_table[entry_idx].addr & 0xFFFFFFFF;
		break;
	case 0x4:  /* Message Address High */
		val = (ctx->msix_table[entry_idx].addr >> 32) & 0xFFFFFFFF;
		break;
	case 0x8:  /* Message Data */
		val = ctx->msix_table[entry_idx].data;
		break;
	case 0xC:  /* Vector Control */
		val = ctx->msix_table[entry_idx].vector_control;
		break;
	}
	
	spin_unlock(&ctx->msix_lock);
	return val;
}

static u64 nv_vgpu_msix_read_pba(struct nv_vgpu_bar_emul_ctx *ctx, u32 size)
{
	u64 val;
	spin_lock(&ctx->msix_lock);
	val = ctx->msix_pba;
	spin_unlock(&ctx->msix_lock);
	return val;
}

void nv_vgpu_msix_set_pending(struct nv_vgpu_device *vgpu, int vector)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
	
	if (!ctx || vector < 0 || vector >= NV_VGPU_MSIX_TABLE_ENTRIES)
		return;
	
	spin_lock(&ctx->msix_lock);
	ctx->msix_pba |= (1ULL << vector);
	spin_unlock(&ctx->msix_lock);
}

void nv_vgpu_msix_clear_pending(struct nv_vgpu_device *vgpu, int vector)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
	
	if (!ctx || vector < 0 || vector >= NV_VGPU_MSIX_TABLE_ENTRIES)
		return;
	
	spin_lock(&ctx->msix_lock);
	ctx->msix_pba &= ~(1ULL << vector);
	spin_unlock(&ctx->msix_lock);
}

int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu)
{
	struct nv_vgpu_bar_emul_ctx *ctx;
	int i, ret;

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return -ENOMEM;

	ctx->vgpu = vgpu;
	ctx->bar0_shadow_size = NV_VGPU_BAR0_SIZE;
	ctx->bar0_shadow = vzalloc(ctx->bar0_shadow_size);
	if (!ctx->bar0_shadow) {
		ret = -ENOMEM;
		goto err_free;
	}

	spin_lock_init(&ctx->shadow_lock);
	ctx->trap_count = 0;

	for (i = 0; i < ARRAY_SIZE(nv_vgpu_bar0_traps); i++) {
		if (ctx->trap_count >= 256)
			break;
		ctx->trap_regs[ctx->trap_count].offset = nv_vgpu_bar0_traps[i].offset;
		ctx->trap_regs[ctx->trap_count].trapped = true;
		ctx->trap_count++;
	}

	/* Initialize MSI-X emulation */
	nv_vgpu_msix_init(ctx);

	/*
	 * Seed PMC boot registers so the guest driver sees a REAL,
	 * PHYSICAL, non-virtual TU104 GPU. Values captured from the host's
	 * physical TU104 (RTX 2080 SUPER, PCI 0000:01:00.0):
	 *   NV_PMC_BOOT_0  (BAR0+0x000) = 0x164000a1  (arch TU104, rev A1)
	 *   NV_PMC_BOOT_1  (BAR0+0x004) = 0x00000000  (VGPU field = REAL,
	 *                                            no PV/VF - the guest
	 *                                            must take the physical
	 *                                            probe path, never a
	 *                                            para-virtual vGPU path)
	 *   NV_PMC_BOOT_42 (BAR0+0xa00) = 0x164a1000  (chip id / revision)
	 *
	 * The guest reads these via the VFIO-mmap'd shadow (and via VFIO
	 * region reads, which hit the same shadow). Without them the open
	 * driver's halmgrGetHalForGpu() fails and probe aborts with the
	 * "does not include the required GPU System Processor (GSP)" error.
	 */
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x000) = 0x164000a1;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x004) = 0x00000000;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xa00) = 0x164a1000;

	/*
	 * Pre-set the GSP secure-boot (GFW) completion state.
	 *
	 * During RmInitAdapter the guest driver waits for the GPU's
	 * post-FLR secure boot (kgspWaitForGfwBootOk -> gpuWaitForGfwBootComplete
	 * -> _gpuIsGfwBootCompleted_TU102 + kflcnWaitForHalt). On real hardware
	 * the boot ROM runs this automatically after reset; the driver only
	 * *waits* for it (it never kicks the falcon here). We present the
	 * "already booted" state so the wait succeeds immediately:
	 *   0x118128  NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK
	 *             bit0 READ_PROTECTION_LEVEL0 = ENABLE (FWSEC lowered PLM)
	 *   0x118234  NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT
	 *             bits7:0 PROGRESS = 0xFF (COMPLETED)
	 *   0x110100  GSP falcon CPUCTL bit4 = HALTED
	 *
	 * Driver-kicked falcon runs later (FWSEC/Booter) clear CPUCTL via
	 * STARTCPU; the GSP emulation thread re-completes those (see
	 * gsp_emu_poll_boot in gsp_emulation.c).
	 */
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x118128) = 0x00000001;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x118234) = 0x000000ff;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x110100) = 0x00000010;

	/*
	 * More post-GFW-boot state the driver validates before/while booting
	 * GSP-RM:
	 *   0x1100f4  GSP falcon HWCFG2 bit10 RISCV = ENABLE. Checked by
	 *             kflcnIsRiscvCpuEnabled in kgspPrepareForBootstrap;
	 *             without it probe dies with "RISC-V core is not enabled".
	 *   0x009430  NV_PTIMER_TIME_PRIV_LEVEL_MASK bit4
	 *             WRITE_PROTECTION_LEVEL0 = ENABLE. FWSEC lowers PLMs
	 *             after GFW boot; driver asserts "Write to PTIMER
	 *             attempted even though Level 0 PLM is disabled" when
	 *             setting the GPU clock otherwise.
	 */
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x1100f4) = 0x00000400;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x009430) = 0x00000010;

	/*
	 * 0x100ce0  NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE - usable FB size.
	 * Encoded as fbSize = LOWER_MAG << (LOWER_SCALE + 20) (LOWER_SCALE
	 * bits 3:0, LOWER_MAG bits 9:4; kmemsysReadUsableFbSize_GP102).
	 * Set to 512MB (mag=1, scale=9 -> 1 << 29): on real hardware VBIOS
	 * devinit programs this after GFW boot; the guest driver's
	 * kgspPopulateWprMeta reads it via kmemsysGetUsableFbSize and uses
	 * it to compute the FRTS offset (fbSize - reservedFromEnd). With it
	 * 0, the FRTS offset underflows and FWSEC verification fails with
	 * "WPR2 initialized at an unexpected location". Must stay in
	 * lockstep with GSCI fb_length/frtsOffset and wpr2_addr_lo.
	 */
	{
		/*
		 * Derive the encoding from the mdev's emulated FB size:
		 * fbSize = MAG << (SCALE + 20), SCALE bits 3:0, MAG bits 9:4.
		 * 512 MiB -> mag=1 scale=9 -> 0x19 (the historical value);
		 * 1 GiB -> 0x1a; 2 GiB -> 0x1b. Sizes that are not exactly
		 * representable round down; the emulation keeps GSCI
		 * fb_length/frtsOffset derived from the same value, so the
		 * guest's fbSize - reserved arithmetic stays consistent.
		 */
		u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu);
		u32 scale = 0;

		while (scale < 15 && (fb >> (scale + 20)) > 0x3f)
			scale++;
		*(u32 *)((u8 *)ctx->bar0_shadow + 0x100ce0) =
			((((u32)(fb >> (scale + 20))) & 0x3f) << 4) |
			(scale & 0xf);
	}

	/*
	 * NV_PCFG (BAR0+0x88000) - PCI config space mirror the driver reads
	 * via gpuReadBusConfigReg_HAL (GPU_BUS_CFG_RD32):
	 *   0x88084 NV_XVE_LINK_CAPABILITIES: MAX_LINK_SPEED 3:0 = 3 (8 GT/s,
	 *           Gen3), MAX_LINK_WIDTH 9:4 = 0x10 (x16).
	 *   0x88088 NV_XVE_LINK_CONTROL_STATUS: negotiated speed 19:16 = 3,
	 *           negotiated width 25:20 = 0x10 (x16).
	 * UVM's nvGpuOpsDeviceCreate -> getPCIELinkRateMBps ->
	 * kbifControlGetPCIEInfo -> kbifGetGpuLinkCapabilities reads 0x88084;
	 * zeros make calculatePCIELinkRateMBps log "Unknown PCIe speed" and
	 * return NV_ERR_INVALID_STATE, which fails cuInit
	 * (CUDA_ERROR_NOT_INITIALIZED in the guest).
	 *
	 * DEVICE_BASE(NV_PCFG) expands to 0 in this driver (nvmisc.h:
	 * "#define DEVICE_BASE(d) (0?d)"), so the read is actually
	 * gpuNvAddr + 0x84 - mirror the values at the low alias as well.
	 */
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x88084) = 0x00000103;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x88088) = 0x01030000;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x00084) = 0x00000103;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x00088) = 0x01030000;
	/* Same values under the VF register window aliases, in case this
	 * driver reaches the config mirror through virtualRegPhysOffset. */
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB80084) = 0x00000103;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB80088) = 0x01030000;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB88084) = 0x00000103;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB88088) = 0x01030000;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xC08084) = 0x00000103;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xC08088) = 0x01030000;

	/*
	 * Per-VF static sizes the driver reads through the VF register window
	 * (GPU_VREG_RD32 = BAR0 + sriovState.virtualRegPhysOffset, 0xB80000
	 * for a non-SRIOV virtual GPU). Both are read-only SIZE registers on
	 * real hardware; we pre-fill them so the local (non-RPC) paths agree
	 * with the GSP_RM_CONTROL replies in gsp_rm_emulation.c.
	 *
	 *  0x3010/0x3030 NV_VIRTUAL_FUNCTION_PRIV_MMU_FAULT_BUFFER_SIZE(0/1):
	 *      _VAL is in NVC369_BUF_SIZE (32B) units. 0x800 -> 64KB
	 *      replayable, 0x80 -> 4KB legacy; matches TU104_GMMU_*_SIZE.
	 *      kgmmuSetAndGetDefaultFaultBufferSize_TU102 writes SET_DEFAULT
	 *      (bit30) and reads _VAL back, so a pre-filled _VAL is all it
	 *      needs.
	 *  0x3110 NV_VIRTUAL_FUNCTION_PRIV_ACCESS_COUNTER_NOTIFY_BUFFER_SIZE:
	 *      UVM's uvmGetAccessCounterBufferSize_TU102 multiplies it by
	 *      NV_ACCESS_COUNTER_NOTIFY_BUF_SIZE (32B). 0x1000 -> 128KB, the
	 *      value the host's physical TU104 reports. Zero makes the
	 *      access-counter buffer a 0-byte allocation and cuInit fails in
	 *      uvmInitializeAccessCntrBuffer.
	 */
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83010) = 0x00000800;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83030) = 0x00000080;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x003010) = 0x00000800;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x003030) = 0x00000080;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83110) = 0x00001000;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0x003110) = 0x00001000;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB8311C) = 0x00000000;
	*(u32 *)((u8 *)ctx->bar0_shadow + 0xB83118) = 0x00000000;


	/*
	 * Serve a valid VBIOS at the PROM window (BAR0+0x300000).
	 *
	 * The guest driver's physical GSP boot (kgspInitRm -> TU102 path)
	 * reads the VBIOS via NV_PROM_DATA (BAR0+0x300000) to extract the
	 * FWSEC ucode (mandatory on Turing: kgspGetFrtsSize is hardwired
	 * to 1MB, so FWSEC/FRTS cannot be skipped). Without a valid ROM
	 * signature the probe aborts with:
	 *   "kgspExtractVbiosFromRom_TU102: did not find valid ROM signature"
	 *
	 * The image is a dump of the host's physical TU104 VBIOS (read via
	 * its own PROM window), loaded from /lib/firmware/tu104_vbios.bin.
	 * Non-fatal if absent: probe then fails later in FWSEC, not here.
	 */
	{
		const struct firmware *vbios_fw;
		int fw_ret = request_firmware(&vbios_fw, NV_VGPU_VBIOS_FW_NAME,
					      &vgpu->pdev->dev);
		if (fw_ret == 0) {
			u32 copy_size = min_t(u32, vbios_fw->size,
					      NV_VGPU_VBIOS_MAX_SIZE);

			if (NV_VGPU_PROM_WINDOW_OFFSET + copy_size <= ctx->bar0_shadow_size) {
				memcpy((u8 *)ctx->bar0_shadow + NV_VGPU_PROM_WINDOW_OFFSET,
				       vbios_fw->data, copy_size);
				nv_vgpu_info(vgpu, "VBIOS loaded at BAR0+0x%x (%u bytes)",
					     NV_VGPU_PROM_WINDOW_OFFSET, copy_size);
			} else {
				nv_vgpu_warn(vgpu, "VBIOS too large for PROM window (%u bytes)",
					     copy_size);
			}
			release_firmware(vbios_fw);
		} else {
			nv_vgpu_warn(vgpu, "VBIOS firmware %s not found (%d); "
				     "guest GSP boot will fail at ROM parse",
				     NV_VGPU_VBIOS_FW_NAME, fw_ret);
		}
	}

	vgpu->bar_emul_ctx = ctx;
	return 0;

err_free:
	kfree(ctx);
	return ret;
}

void nv_vgpu_bar_emul_cleanup(struct nv_vgpu_device *vgpu)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;

	if (!ctx)
		return;

	if (ctx->bar0_shadow)
		vfree(ctx->bar0_shadow);

	kfree(ctx);
	vgpu->bar_emul_ctx = NULL;
}

int nv_vgpu_bar_trap_register(struct nv_vgpu_device *vgpu, int bar_idx,
			       u32 offset, u32 size)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
	unsigned long flags;
	int i;

	if (!ctx || bar_idx != 0)
		return -EINVAL;

	spin_lock_irqsave(&ctx->shadow_lock, flags);

	for (i = 0; i < ctx->trap_count; i++) {
		if (ctx->trap_regs[i].offset == offset) {
			ctx->trap_regs[i].trapped = true;
			spin_unlock_irqrestore(&ctx->shadow_lock, flags);
			return 0;
		}
	}

	if (ctx->trap_count >= 256) {
		spin_unlock_irqrestore(&ctx->shadow_lock, flags);
		return -ENOSPC;
	}

	ctx->trap_regs[ctx->trap_count].offset = offset;
	ctx->trap_regs[ctx->trap_count].trapped = true;
	ctx->trap_count++;

	spin_unlock_irqrestore(&ctx->shadow_lock, flags);
	return 0;
}

int nv_vgpu_bar_untrap_register(struct nv_vgpu_device *vgpu, int bar_idx,
				 u32 offset)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
	unsigned long flags;
	int i;

	if (!ctx || bar_idx != 0)
		return -EINVAL;

	spin_lock_irqsave(&ctx->shadow_lock, flags);

	for (i = 0; i < ctx->trap_count; i++) {
		if (ctx->trap_regs[i].offset == offset) {
			ctx->trap_regs[i].trapped = false;
			spin_unlock_irqrestore(&ctx->shadow_lock, flags);
			return 0;
		}
	}

	spin_unlock_irqrestore(&ctx->shadow_lock, flags);
	return -ENOENT;
}

int nv_vgpu_bar_emul_read(struct nv_vgpu_device *vgpu, int bar_idx,
			   u64 offset, void *val, size_t size)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
	unsigned long flags;
	u32 *val32 = val;
	u64 *val64 = val;

	if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size)
		return -EINVAL;

	/* Handle MSI-X Table reads (at BAR0+0x2000) */
	if (offset >= GSP_EMU_MSIX_TABLE_OFFSET && 
	    offset < GSP_EMU_MSIX_TABLE_OFFSET + NV_VGPU_MSIX_TABLE_SIZE) {
		u32 msix_offset = offset - GSP_EMU_MSIX_TABLE_OFFSET;
		*val64 = nv_vgpu_msix_read_table(ctx, msix_offset, size);
		return 0;
	}

	/* Handle MSI-X PBA reads (at BAR0+0x3000) */
	if (offset >= GSP_EMU_MSIX_PBA_OFFSET && 
	    offset < GSP_EMU_MSIX_PBA_OFFSET + NV_VGPU_MSIX_PBA_SIZE) {
		*val64 = nv_vgpu_msix_read_pba(ctx, size);
		return 0;
	}

	spin_lock_irqsave(&ctx->shadow_lock, flags);

	switch (size) {
	case 4:
		*val32 = *(u32 *)((u8 *)ctx->bar0_shadow + offset);
		break;
	case 8:
		*val64 = *(u64 *)((u8 *)ctx->bar0_shadow + offset);
		break;
	default:
		spin_unlock_irqrestore(&ctx->shadow_lock, flags);
		return -EINVAL;
	}

	spin_unlock_irqrestore(&ctx->shadow_lock, flags);
	return 0;
}

int nv_vgpu_bar_emul_write(struct nv_vgpu_device *vgpu, int bar_idx,
			    u64 offset, void *val, size_t size)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
	unsigned long flags;
	u32 *val32 = val;
	u64 *val64 = val;
	bool trapped = false;
	int i;

	if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size)
		return -EINVAL;

	/* Handle MSI-X Table writes (at BAR0+0x2000) */
	if (offset >= GSP_EMU_MSIX_TABLE_OFFSET && 
	    offset < GSP_EMU_MSIX_TABLE_OFFSET + NV_VGPU_MSIX_TABLE_SIZE) {
		u32 msix_offset = offset - GSP_EMU_MSIX_TABLE_OFFSET;
		nv_vgpu_msix_write_table(ctx, msix_offset, *val64, size);
		return 0;
	}

	spin_lock_irqsave(&ctx->shadow_lock, flags);

	for (i = 0; i < ctx->trap_count; i++) {
		if (ctx->trap_regs[i].offset == offset && ctx->trap_regs[i].trapped) {
			trapped = true;
			ctx->trap_regs[i].value = (size == 4) ? *val32 : (u32)*val64;
			ctx->trap_regs[i].is_write = true;
			break;
		}
	}

	switch (size) {
	case 4:
		*(u32 *)((u8 *)ctx->bar0_shadow + offset) = *val32;
		break;
	case 8:
		*(u64 *)((u8 *)ctx->bar0_shadow + offset) = *val64;
		break;
	default:
		spin_unlock_irqrestore(&ctx->shadow_lock, flags);
		return -EINVAL;
	}

	spin_unlock_irqrestore(&ctx->shadow_lock, flags);

	if (trapped) {
		nv_vgpu_dbg(vgpu, "BAR0 trap: offset=0x%llx, size=%zu, write=0x%x",
			    offset, size, (size == 4) ? *val32 : (u32)*val64);
		nv_vgpu_gsp_rpc_bar_trap(vgpu, offset, 1, size, *(u64 *)val, 0, NULL);
	}

	return 0;
}

int nv_vgpu_bar_emul_handle_fault(struct nv_vgpu_device *vgpu,
				   unsigned long addr, bool write)
{
	struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx;
	struct nv_vgpu_bar *bar = &vgpu->bars[0];
	u64 offset = addr - (unsigned long)bar->vaddr;
	u32 val = 0;
	int ret;

	if (!ctx || !bar->mapped || !bar->trapped)
		return -EINVAL;

	if (write) {
		ret = nv_vgpu_bar_emul_write(vgpu, 0, offset, &val, 4);
	} else {
		ret = nv_vgpu_bar_emul_read(vgpu, 0, offset, &val, 4);
	}

	return ret;
}