path: root/kernel/gsp_emulation.c

/*
 * GSP Emulation Engine Implementation
 * Host-side GSP emulation thread that processes RPCs from guest via shared memory
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/ktime.h>
#include <linux/eventfd.h>
#include <asm/io.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_emulation.h"
#include "gsp_rm_emulation.h"
#include "gsp_rpc_client.h"
#include "bar_emulation.h"
#include "irq.h"
#include "memory.h"

/* ============================================================
 * Constants
 * ============================================================ */

#define GSP_EMU_DOORBELL_OFFSET     0x110C00
#define GSP_EMU_SHARED_MEM_OFFSET   0x1000
#define GSP_EMU_SHARED_MEM_SIZE     4096
#define GSP_EMU_MSIX_TABLE_OFFSET   0x2000
#define GSP_EMU_MSIX_PBA_OFFSET     0x3000
#define GSP_EMU_POLL_BUSY_US        100
#define GSP_EMU_POLL_SLEEP_MIN_US   50
#define GSP_EMU_POLL_SLEEP_MAX_US   100

/* ============================================================
 * Helper Functions
 * ============================================================ */

static inline u32 gsp_emu_read_doorbell(struct gsp_emu_context *ctx)
{
    return readl(ctx->doorbell_reg);
}

static inline void gsp_emu_write_doorbell(struct gsp_emu_context *ctx, u32 val)
{
    writel(val, ctx->doorbell_reg);
}

static inline void gsp_emu_copy_request_from_shared(struct gsp_emu_context *ctx,
                                                     struct gsp_rpc_msg *req)
{
    memcpy(req, ctx->shared_mem_virt, sizeof(*req));
}

static inline void gsp_emu_copy_response_to_shared(struct gsp_emu_context *ctx,
                                                    struct gsp_rpc_response *rsp)
{
    memcpy(ctx->shared_mem_virt + 0x80, rsp, sizeof(*rsp));
}

/* ============================================================
 * GSP Falcon Boot-Watch (observability for physical GSP boot)
 * ============================================================ */

/*
 * The guest's physical GSP boot (FWSEC -> Booter -> RISCV) pokes the GSP
 * falcon block via the mmap'd shadow, which the host cannot trap. Poll these
 * BAR0 offsets and log changes so the boot sequence is visible in host dmesg.
 */
static const struct {
	u32 offset;
	const char *name;
} gsp_emu_falcon_watch[GSP_EMU_FALCON_WATCH_COUNT] = {
	{ 0x110040, "GSP_MAILBOX0" },
	{ 0x110044, "GSP_MAILBOX1" },
	{ 0x110080, "GSP_FALCON_OS" },
	{ 0x110100, "GSP_CPUCTL" },
	{ 0x110180, "GSP_IMEMC" },
	{ 0x110184, "GSP_IMEMD" },
	{ 0x1101c0, "GSP_DMEMC" },
	{ 0x1101c4, "GSP_DMEMD" },
	{ 0x1103c0, "GSP_ENGINE" },
	{ 0x110c04, "GSP_QUEUE_HEAD1" },
	{ 0x110c08, "GSP_QUEUE_TAIL0" },
	{ 0x110c0c, "GSP_QUEUE_TAIL1" },
	{ 0x118128, "PGC6_PLM_GFW" },
	{ 0x118234, "PGC6_GFW_BOOT" },
	{ 0x1fa824, "WPR2_ADDR_LO" },
	{ 0x1fa828, "WPR2_ADDR_HI" },
	{ 0x001438, "VBIOS_SCRATCH_0E" },
	{ 0x840040, "SEC2_MAILBOX0" },
	{ 0x840100, "SEC2_CPUCTL" },
	{ 0x8403c0, "SEC2_ENGINE" },
};

static void gsp_emu_poll_falcon_watch(struct gsp_emu_context *ctx)
{
	int i;

	for (i = 0; i < GSP_EMU_FALCON_WATCH_COUNT; i++) {
		u32 val = readl(ctx->bar0_vaddr + gsp_emu_falcon_watch[i].offset);

		if (!ctx->falcon_watch_valid) {
			ctx->falcon_watch_last[i] = val;
			continue;
		}

		if (val != ctx->falcon_watch_last[i]) {
			nv_vgpu_info(ctx->vgpu, "FALCON %s (0x%x): 0x%08x -> 0x%08x",
				     gsp_emu_falcon_watch[i].name,
				     gsp_emu_falcon_watch[i].offset,
				     ctx->falcon_watch_last[i], val);
			ctx->falcon_watch_last[i] = val;
		}
	}
	ctx->falcon_watch_valid = true;
}

/* ============================================================
 * Physical GSP Boot Emulation (FWSEC / Booter falcon auto-complete)
 * ============================================================ */

/* Register offsets + CPUCTL bits live in gsp_emulation.h (shared with
 * gsp_rm_emulation.c), as are gsp_emu_shadow_read32/write32. */

enum gsp_emu_boot_stage {
	GSP_BOOT_STAGE_IDLE = 0,	/* waiting for first kick (FWSEC) */
	GSP_BOOT_STAGE_FWSEC_DONE,	/* FWSEC FRTS completed */
	GSP_BOOT_STAGE_BOOTER_DONE,	/* Booter Load completed */
};

/*
 * gsp_emu_clear_boot_state - return the emulated GSP to a cold state.
 *
 * On real hardware, driver unload tears down WPR2 (Booter Unload) so the
 * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce
 * no observable BAR0 writes on unload, so callers must trigger this on
 * indirect death signals (ENGINE reset, GSP-RM queue going stale).
 * Clears WPR2, RISCV active status and the falcon stage tracker, and
 * drops the parsed GSP-RM queue state.
 */
/*
 * gsp_emu_clear_boot_state - return the emulated GSP to a cold state.
 *
 * On real hardware, driver unload tears down WPR2 (Booter Unload) so the
 * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce
 * no observable BAR0 writes on unload, so callers must trigger this on
 * indirect death signals (ENGINE reset, GSP-RM queue going stale).
 *
 * IMPORTANT: on the Booter-unload kick we clear the FALCON boot state
 * (WPR2/RISCV/stages) but PRESERVE the GSP-RM queue state. The driver
 * sends cleanup FREEs AFTER the Booter-unload kick; if we reset the queue
 * here, those frees time out (fn-10). The queue state is only cleared when
 * the queue is genuinely gone (garbage elements, insane writePtr, ENGINE
 * reset, or a new probe writing fresh libos args to MAILBOX0/1).
 */
void gsp_emu_clear_boot_state(struct gsp_emu_context *ctx, const char *reason)
{
	nv_vgpu_info(ctx->vgpu, "GSP-BOOT: clearing boot state (%s)", reason);

	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO, 0);
	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0);
	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0);
	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX0, 0);
	gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX1, 0);

	ctx->boot_stage = GSP_BOOT_STAGE_IDLE;
	ctx->flcn_kick_count = 0;

	/* Do NOT reset the GSP-RM queue state here - the driver is still
	 * sending cleanup frees on it. Only reset on genuine queue death
	 * (garbage elements, insane writePtr) or ENGINE reset. */
}

/*
 * gsp_emu_poll_boot - drive the guest's physical GSP boot to completion.
 *
 * The guest driver kicks falcon ucodes (FWSEC for FRTS, then Booter Load for
 * GSP-RM) by writing STARTCPU to the GSP falcon CPUCTL, then polls for HALTED
 * plus stage-specific result registers. The auto-boot waits (GFW) are already
 * satisfied by the state pre-set in nv_vgpu_bar_emul_init. Here we watch for
 * STARTCPU kicks and complete each stage so the guest believes its falcon
 * ucodes ran successfully.
 */
static void gsp_emu_poll_boot(struct gsp_emu_context *ctx)
{
	u32 gsp_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_CPUCTL);
	u32 sec2_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_SEC2_CPUCTL);
	u32 engine = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_ENGINE);
	u32 sw_vector;

	/* Software-interrupt trigger: the driver writes a vector to
	 * CPU_INTR_LEAF_TRIGGER to fire it (the osVerifyInterrupts sanity
	 * test uses the doorbell vector). Set the pending bits in the LEAF
	 * and TOP interrupt-tree registers and inject MSI-X so the guest's
	 * ISR runs - without this the sanity test times out and
	 * osVerifySystemEnvironment fails, bailing RmInitAdapter. */
	sw_vector = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER);
	if (sw_vector) {
		u32 leaf_idx = sw_vector / 32;
		u32 leaf_bit = sw_vector % 32;
		u32 subtree  = sw_vector / 64;
		u32 top_idx  = subtree / 32;
		u32 top_bit  = subtree % 32;
		u32 leaf_val = gsp_emu_shadow_read32(ctx,
				GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4);
		u32 top_val  = gsp_emu_shadow_read32(ctx,
				GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4);

		gsp_emu_shadow_write32(ctx,
			GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4,
			leaf_val | (1U << leaf_bit));
		gsp_emu_shadow_write32(ctx,
			GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4,
			top_val | (1U << top_bit));
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER, 0);

		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: SW interrupt vector %u -> LEAF[%u]|=0x%x TOP[%u]|=0x%x, MSI-X",
			     sw_vector, leaf_idx, 1U << leaf_bit, top_idx,
			     1U << top_bit);
		gsp_emu_inject_completion_interrupt(ctx, 0);
	}

	/* TLB invalidate auto-complete: the driver sets TRIGGER then spins
	 * polling for it to clear (kgmmuCheckPendingInvalidates_TU102 reads
	 * GPU_VREG_RD32(NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE) =
	 * BAR0 + 0xB80000 + 0x30B0; the physical path uses the plain alias).
	 * Report instant completion or the driver soft-locks spinning inside
	 * osDevReadReg032. */
	{
		static const u32 tlb_inv_regs[] = {
			GSP_EMU_REG_MMU_INVALIDATE,
			GSP_EMU_REG_MMU_INVALIDATE_WIN,
		};
		unsigned int i;

		for (i = 0; i < ARRAY_SIZE(tlb_inv_regs); i++) {
			u32 val = gsp_emu_shadow_read32(ctx, tlb_inv_regs[i]);

			if (!(val & GSP_EMU_MMU_INVALIDATE_TRIGGER))
				continue;
			gsp_emu_shadow_write32(ctx, tlb_inv_regs[i],
					       val & ~GSP_EMU_MMU_INVALIDATE_TRIGGER);
			pr_info_ratelimited("vgsp-boot: MMU_INVALIDATE[0x%x] trigger auto-cleared (val=0x%x)\n",
					    tlb_inv_regs[i], val);
		}
	}

	/* L2 cache-op auto-complete: kmemsysDoCacheOp_GM107 writes PENDING
	 * (bit 0) into the NV_UFLUSH_* / VF L2 invalidate registers and spins
	 * until the polled bits read back 0 (FB flush, L2 sysmem/peermem
	 * invalidate, L2 clean, L2 flush-dirty, L2 wait-for-sys-reads). On
	 * Turing the sysmem/peermem invalidate registers go through the
	 * GPU_VREG_* window (BAR0 + 0xB80000 + NV_VIRTUAL_FUNCTION_PRIV_L2_*),
	 * so cover both the plain and windowed aliases. The emulated L2 has
	 * nothing to do, so clear them as soon as they are observed. */
	{
		static const u32 uflush_regs[] = {
			GSP_EMU_REG_UFLUSH_FB_FLUSH,
			GSP_EMU_REG_UFLUSH_L2_SYSMEM_INV,
			GSP_EMU_REG_UFLUSH_L2_PEERMEM_INV,
			GSP_EMU_REG_UFLUSH_L2_CLEAN_COMPTAGS,
			GSP_EMU_REG_UFLUSH_L2_FLUSH_DIRTY,
			GSP_EMU_REG_UFLUSH_L2_WAIT_SYS_READS,
			GSP_EMU_REG_VF_L2_SYSMEM_INV,
			GSP_EMU_REG_VF_L2_PEERMEM_INV,
			GSP_EMU_REG_VF_L2_SYSMEM_INV_WIN,
			GSP_EMU_REG_VF_L2_PEERMEM_INV_WIN,
		};
		unsigned int i;

		for (i = 0; i < ARRAY_SIZE(uflush_regs); i++) {
			u32 val = gsp_emu_shadow_read32(ctx,
							uflush_regs[i]);

			if (!val)
				continue;
			gsp_emu_shadow_write32(ctx, uflush_regs[i], 0);
			pr_info_ratelimited("vgsp-boot: UFLUSH[0x%x] 0x%x auto-cleared\n",
					    uflush_regs[i], val);
		}
	}

	/* CrashCat wayfinder suppression: if the GSP falcon's DEBUGINFO
	 * register holds a valid wayfinder-L0 signature (0xdead), the driver's
	 * kgspHealthCheck (run on every RPC drain) reads "reports" from
	 * garbage and loops forever in nv_open_q, which then can't poll RPCs
	 * (fn-10 FREE timeouts). Force it invalid - our emulated GSP has no
	 * crashcat reports to offer. */
	{
		u32 wfl0 = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO);

		if ((wfl0 & 0xffff) == 0xdead) {
			gsp_emu_shadow_write32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO, 0);
			nv_vgpu_info(ctx->vgpu, "GSP-BOOT: cleared CrashCat wayfinder (was 0x%x)",
				     wfl0);
		} else if (wfl0) {
			pr_info_ratelimited("vgsp-boot: DEBUGINFO nonzero (0x%x), not a wayfinder\n",
					    wfl0);
		}
	}

	/* Temporary: watch PCFG / link-cap mirror offsets for guest writes */
	{
		static const u32 watch[] = {
			0x000004, 0x000084, 0x880000, 0x880004, 0x880080,
			0x880084, 0x880088, 0x88008c, 0xb80084, 0xb88084,
			0xc08084,
		};
		static u32 last[ARRAY_SIZE(watch)];
		unsigned int i;

		for (i = 0; i < ARRAY_SIZE(watch); i++) {
			u32 v = gsp_emu_shadow_read32(ctx, watch[i]);

			if (v != last[i]) {
				nv_vgpu_info(ctx->vgpu, "GSP-BOOT: PCFG[0x%x] = 0x%x (was 0x%x)",
					     watch[i], v, last[i]);
				last[i] = v;
			}
		}
	}

	/* GSP ENGINE reset asserted -> falcon rebooted, restart stage tracking */
	if ((engine & 0x1) && ctx->boot_stage != GSP_BOOT_STAGE_IDLE) {
		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: falcon reset, back to IDLE");
		/* ENGINE reset is a full GPU reset: the GSP-RM queue state is
		 * stale, reset it too. */
		gsp_emu_clear_boot_state(ctx, "ENGINE reset");
		gsp_rm_emu_reset(ctx);
		return;
	}

	/*
	 * PRAMIN window translation: the guest programs NV_PBUS_BAR0_WINDOW
	 * (BAR0+0x1700) to select which vidmem page the PRAMIN window
	 * (BAR0+0x700000) maps to. On real hardware the PRAMIN window is a
	 * CPU aperture into vidmem at (BASE << 16) + offset. In our shadow
	 * the PRAMIN window is just part of the shadow buffer - it does NOT
	 * alias the emulated BAR1 (vidmem).
	 *
	 * The BAR2 verify test writes through the vBar2 aperture (PRAMIN)
	 * and reads back through the BAR0 window (also PRAMIN), expecting
	 * the same data. Both paths hit the same shadow location, so the
	 * test data IS consistent. We do NOT sync the PRAMIN window from
	 * the emulated BAR1 - the sync would overwrite the test data
	 * written by the driver through the vBar2 aperture.
	 *
	 * The BAR0_WINDOW_BASE register is captured for observability only.
	 */
	{
		u32 bar0_win = gsp_emu_shadow_read32(ctx, 0x1700);
		u32 base = bar0_win & 0xFFFFFF;
		u32 target = (bar0_win >> 24) & 0x3;

		if (!ctx->bar0_window_valid ||
		    base != ctx->bar0_window_base ||
		    target != ctx->bar0_window_target) {
			static u32 last_pramin_val;
			u32 pramin_val = gsp_emu_shadow_read32(ctx, 0x70e000);

			ctx->bar0_window_base = base;
			ctx->bar0_window_target = target;
			ctx->bar0_window_valid = true;
			nv_vgpu_info(ctx->vgpu,
				"GSP-BOOT: BAR0_WINDOW base=0x%x target=%u, PRAMIN[0xe000]=0x%08x (was 0x%08x)",
				base, target, pramin_val, last_pramin_val);
			last_pramin_val = pramin_val;
		}
	}

	/*
	 * FWSEC kick on the GSP falcon (FRTS setup). Complete it:
	 * - CPUCTL: clear STARTCPU, set HALTED
	 * - VBIOS_SCRATCH[0x0E] FRTS_ERR_CODE = NONE (0)
	 * - WPR2_ADDR_HI != 0, WPR2_ADDR_LO = frtsOffset >> 0xc (in _VAL bits31:4)
	 */
	if ((gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU) &&
	    ctx->boot_stage == GSP_BOOT_STAGE_IDLE) {
		/* A fresh FWSEC kick means a new GSP boot: drop any GSP-RM
		 * queue state left from the previous probe (the driver has
		 * already torn down its old queue). Without this the next
		 * Booter completion would skip the chain re-parse
		 * (rm->parsed still set) and GspStatusQueueInit would time
		 * out on re-probe. */
		gsp_rm_emu_reset(ctx);
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_VBIOS_SCRATCH_0E, 0);
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0x100000);
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO,
				       ctx->wpr2_addr_lo);
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL,
				       GSP_EMU_CPUCTL_HALTED);
		ctx->boot_stage = GSP_BOOT_STAGE_FWSEC_DONE;
		ctx->flcn_kick_count++;
		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: FWSEC kick completed "
			     "(WPR2_LO=0x%x)", ctx->wpr2_addr_lo);
		return;
	}

	/*
	 * Booter Load kick on the SEC2 falcon (loads/authenticates the GSP-RM
	 * image and starts the GSP RISCV). Complete it:
	 * - SEC2 CPUCTL: clear STARTCPU, set HALTED
	 * - SEC2 MAILBOX0 = 0 (booter error code, 0 = success)
	 * - GSP RISCV_STATUS = ACTIVE (Booter started the RISCV;
	 *   kflcnIsRiscvActive passes)
	 */
	if ((sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) &&
	    ctx->boot_stage == GSP_BOOT_STAGE_FWSEC_DONE) {
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0);
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0x1);
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL,
				       GSP_EMU_CPUCTL_HALTED);
		ctx->boot_stage = GSP_BOOT_STAGE_BOOTER_DONE;
		ctx->flcn_kick_count++;
		nv_vgpu_info(ctx->vgpu, "GSP-BOOT: Booter kick completed, "
			     "RISCV active");
		/* The driver has programmed the libos init args address into
		 * GSP MAILBOX0/1 by now; parse the queue chain, link the
		 * status queue and emit GSP_INIT_DONE. */
		gsp_rm_emu_on_booter_done(ctx);
		return;
	}

	/* Any other STARTCPU kick (resume/unload/etc.): complete it.
	 * SEC2 kicks report their ucode result in SEC2_MAILBOX0 (0=success) -
	 * e.g. Booter Unload during GPU shutdown. Without clearing it the
	 * driver reads a stale/garbage error code (0xff) and the unload
	 * fails, killing the subsequent re-init. A SEC2 kick after a
	 * completed boot is a GSP shutdown/reload: drop the boot state
	 * (GSP no longer active, queue dead). Clear boot state FIRST so
	 * WPR2 is already down when the driver sees CPUCTL=HALTED and
	 * checks kgspIsWpr2Up (avoids "Booter Unload: WPR2 is still up"). */
	if (gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU)
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL,
				       GSP_EMU_CPUCTL_HALTED);
	if (sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) {
		if (ctx->boot_stage == GSP_BOOT_STAGE_BOOTER_DONE)
			gsp_emu_clear_boot_state(ctx, "Booter unload/reload kick");
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0);
		gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL,
				       GSP_EMU_CPUCTL_HALTED);
	}
}

/* ============================================================
 * RPC Processing
 * ============================================================ */

static int gsp_emu_dispatch_rpc(struct gsp_emu_context *ctx,
                                 struct gsp_rpc_msg *req,
                                 struct gsp_rpc_response *rsp)
{
    int ret = 0;
    ktime_t start = ktime_get();

    memset(rsp, 0, sizeof(*rsp));
    rsp->type = req->type;
    rsp->seq_id = req->seq_id;

    switch (req->type) {
    case GSP_RPC_ALLOC: {
        struct gsp_alloc_req *areq = (struct gsp_alloc_req *)req->payload;
        struct gsp_alloc_rsp arsp = {0};
        u64 gpu_va = 0, handle = 0, actual_size = 0;

        ret = gsp_rpc_alloc(ctx->rpc_client,
                            areq->size, areq->alignment, areq->mem_type,
                            areq->flags, areq->addr_hint, areq->pid,
                            &gpu_va, &handle, &actual_size);
        if (!ret) {
            arsp.gpu_va = gpu_va;
            arsp.handle = handle;
            arsp.size = actual_size;
            memcpy(rsp->result, &arsp, sizeof(arsp));
        }
        break;
    }

    case GSP_RPC_FREE: {
        u64 handle = req->payload[0];
        ret = gsp_rpc_free(ctx->rpc_client, handle);
        break;
    }

    case GSP_RPC_MAP: {
        struct gsp_map_req *mreq = (struct gsp_map_req *)req->payload;
        struct gsp_map_rsp mrsp = {0};
        u64 mapped_va = 0;

        ret = gsp_rpc_map(ctx->rpc_client,
                          mreq->handle, mreq->gpu_va, mreq->size,
                          mreq->prot, mreq->flags, mreq->offset,
                          &mapped_va);
        if (!ret) {
            mrsp.gpu_va = mapped_va;
            memcpy(rsp->result, &mrsp, sizeof(mrsp));
        }
        break;
    }

    case GSP_RPC_UNMAP: {
        u64 handle = req->payload[0];
        u64 gpu_va = req->payload[1];
        u64 size = req->payload[2];
        ret = gsp_rpc_unmap(ctx->rpc_client, handle, gpu_va, size);
        break;
    }

    case GSP_RPC_PIN: {
        struct gsp_pin_req *preq = (struct gsp_pin_req *)req->payload;
        struct gsp_pin_rsp prsp = {0};
        u64 dma_addr = 0;
        u32 num_pages = 0;

        ret = gsp_rpc_pin(ctx->rpc_client,
                          preq->handle, preq->offset, preq->size,
                          preq->flags, &dma_addr, &num_pages);
        if (!ret) {
            prsp.dma_addr = dma_addr;
            prsp.num_pages = num_pages;
            memcpy(rsp->result, &prsp, sizeof(prsp));
        }
        break;
    }

    case GSP_RPC_UNPIN: {
        u64 handle = req->payload[0];
        u64 offset = req->payload[1];
        u64 size = req->payload[2];
        ret = gsp_rpc_unpin(ctx->rpc_client, handle, offset, size);
        break;
    }

    case GSP_RPC_SCHEDULE: {
        struct gsp_sched_req *sreq = (struct gsp_sched_req *)req->payload;
        ret = gsp_rpc_schedule(ctx->rpc_client,
                               sreq->work_handle, sreq->callback, sreq->arg,
                               sreq->priority, sreq->flags);
        break;
    }

    case GSP_RPC_YIELD: {
        ret = gsp_rpc_yield(ctx->rpc_client);
        break;
    }

    case GSP_RPC_NOTIFY: {
        /* NOTIFY is handled via registration, not direct RPC */
        nv_vgpu_info(ctx->vgpu, "NOTIFY RPC received (type=0x%x)", req->subcmd);
        ret = 0;
        break;
    }

    case GSP_RPC_BAR_TRAP: {
        struct gsp_bar_trap_req *breq = (struct gsp_bar_trap_req *)req->payload;
        struct gsp_bar_trap_rsp brsp = {0};
        u64 ret_value = 0;

        ret = gsp_rpc_bar_trap(ctx->rpc_client,
                               breq->bar_addr, breq->access_type, breq->size,
                               breq->value, breq->bar_num, &ret_value);
        if (!ret) {
            brsp.value = ret_value;
            brsp.handled = 1;
            memcpy(rsp->result, &brsp, sizeof(brsp));
        }
        break;
    }

    case GSP_RPC_MIGRATE: {
        struct gsp_migrate_req *mreq = (struct gsp_migrate_req *)req->payload;
        struct gsp_migrate_rsp mrsp = {0};
        u64 checkpoint = 0;

        if (mreq->flags == GSP_MIGRATE_SUBCMD_SAVE) {
            ret = gsp_rpc_migrate_save(ctx->rpc_client,
                                       mreq->src_gpu_id, mreq->dst_gpu_id,
                                       mreq->memory_bitmap, mreq->page_table_ptr,
                                       &checkpoint);
            if (!ret) {
                mrsp.checkpoint = checkpoint;
                memcpy(rsp->result, &mrsp, sizeof(mrsp));
            }
        } else if (mreq->flags == GSP_MIGRATE_SUBCMD_RESTORE) {
            ret = gsp_rpc_migrate_restore(ctx->rpc_client,
                                          mreq->src_gpu_id, mreq->dst_gpu_id,
                                          mreq->memory_bitmap, mreq->page_table_ptr,
                                          mreq->context);
        }
        break;
    }

    case GSP_RPC_DEBUG: {
        nv_vgpu_info(ctx->vgpu, "DEBUG RPC received (seq=%u)", req->seq_id);
        ret = 0;
        break;
    }

    default:
        nv_vgpu_warn(ctx->vgpu, "Unknown RPC type: 0x%x", req->type);
        rsp->status = -ENOTSUPP;
        ret = -ENOTSUPP;
        break;
    }

    if (ret)
        rsp->status = ret;

    ctx->total_latency_ns += ktime_to_ns(ktime_sub(ktime_get(), start));
    return ret;
}

/* ============================================================
 * MSI-X Interrupt Injection
 * ============================================================ */

void gsp_emu_inject_completion_interrupt(struct gsp_emu_context *ctx, u32 seq_id)
{
    struct eventfd_ctx *trigger = NULL;

    /* Write to MSI-X PBA (Pending Bit Array) at BAR0+0x3000 */
    if (ctx->bar0_vaddr) {
        u32 *pba = (u32 *)(ctx->bar0_vaddr + GSP_EMU_MSIX_PBA_OFFSET);
        writel(1, pba); /* Set pending bit for vector 0 */
    }

    /* Look up the vector-0 trigger LIVE: QEMU wires eventfds via
     * VFIO_DEVICE_SET_IRQS after gsp_emu_init runs, so an init-time copy
     * would be stale (NULL). */
    if (ctx->vgpu)
        trigger = ctx->vgpu->irqs[0].trigger;

    /* Signal eventfd for VFIO interrupt delivery */
    if (trigger)
        eventfd_signal(trigger);
}

/* ============================================================
 * Emulation Thread
 * ============================================================ */

static int gsp_emu_thread_fn(void *data)
{
    struct gsp_emu_context *ctx = data;
    struct gsp_rpc_msg req;
    struct gsp_rpc_response rsp;
    u32 doorbell_val;
    int ret;

    nv_vgpu_info(ctx->vgpu, "GSP emulation thread started");

    while (!kthread_should_stop()) {
        /* Read doorbell */
        doorbell_val = gsp_emu_read_doorbell(ctx);

        /* Watch GSP falcon boot registers (physical GSP boot tracing) */
        gsp_emu_poll_falcon_watch(ctx);

        /* Drive the guest's physical GSP boot to completion */
        gsp_emu_poll_boot(ctx);

        /* GSP-RM msgq emulation (physical path: sysmem cmd/status queues) */
        gsp_rm_emu_poll(ctx);

        if (doorbell_val != 0) {
            /* ACK doorbell immediately */
            gsp_emu_write_doorbell(ctx, 0);

            /* Copy request from shared memory */
            gsp_emu_copy_request_from_shared(ctx, &req);

            /* Validate sequence ID */
            if (req.seq_id == 0) {
                nv_vgpu_warn(ctx->vgpu, "Invalid seq_id=0, ignoring RPC");
                continue;
            }

            /* Process RPC */
            ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp);

            /* Write response to shared memory */
            gsp_emu_copy_response_to_shared(ctx, &rsp);

            /* Inject completion interrupt */
            gsp_emu_inject_completion_interrupt(ctx, req.seq_id);

            if (ret)
                ctx->failed_rpcs++;
            else
                ctx->processed_rpcs++;
        } else {
            /* Adaptive polling: busy-wait first, then sleep (interruptible for kthread_stop) */
            static int poll_count = 0;
            if (poll_count < GSP_EMU_POLL_BUSY_US) {
                udelay(1);
                poll_count++;
            } else {
                /* Use interruptible sleep so kthread_stop can wake us */
                schedule_timeout_interruptible(usecs_to_jiffies(GSP_EMU_POLL_SLEEP_MAX_US));
                poll_count = 0;
            }
        }
    }

    nv_vgpu_info(ctx->vgpu, "GSP emulation thread stopped (processed=%llu, failed=%llu)",
                 ctx->processed_rpcs, ctx->failed_rpcs);
    complete(&ctx->thread_stopped);
    return 0;
}

/* ============================================================
 * Initialization & Cleanup
 * ============================================================ */

int gsp_emu_init(struct nv_vgpu_device *vgpu)
{
    struct nv_vgpu_bar_emul_ctx *bar_ctx;
    struct gsp_emu_context *ctx;
    int ret = 0;

    nv_vgpu_info(vgpu, "Initializing GSP emulation engine");

    ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
    if (!ctx)
        return -ENOMEM;

    ctx->vgpu = vgpu;
    ctx->running = false;
    ctx->initialized = false;
    ctx->stopping = false;
    init_completion(&ctx->thread_stopped);
    spin_lock_init(&ctx->lock);

    /* Physical GSP boot emulation state. wpr2_addr_lo matches what the
     * guest driver expects WPR2_ADDR_LO to read after FWSEC FRTS:
     * frtsOffset = fbSize - reservedFromEnd. With fbSize = 512MB
     * (LOCAL_MEMORY_RANGE / GSCI fb_length) and the 570.144 image's 2MB
     * reservation, frtsOffset = 0x20000000 - 0x200000 = 0x1fe00000; the
     * register's _VAL field (bits31:4) must equal frtsOffset >> 12 =
     * 0x1fe00, so the raw register value is 0x1fe000. Kept in lockstep
     * with GSCI fb_length/frtsOffset and NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE. */
    ctx->boot_stage = 0;    /* GSP_BOOT_STAGE_IDLE */
    ctx->flcn_kick_count = 0;
    ctx->wpr2_addr_lo = 0x001fe000;

    /* The guest's BAR0 is the emulation shadow (vzalloc'd by
     * nv_vgpu_bar_emul_init, mapped to the guest via VFIO mmap) - NOT the
     * host physical BAR0 (vgpu->bars[0].vaddr, which belongs to the real GPU
     * and the host nvidia driver). All guest doorbell writes, RPC messages,
     * MSI-X table writes and PBA reads land in the shadow, so the emulation
     * engine must operate on the shadow exclusively. Polling the physical
     * BAR0 would never see guest traffic and would corrupt host GPU state. */
    bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
    if (!bar_ctx || !bar_ctx->bar0_shadow) {
        nv_vgpu_err(vgpu, "BAR0 emulation shadow not available");
        ret = -ENODEV;
        goto err_free_ctx;
    }

    ctx->bar0_vaddr = (void __iomem *)bar_ctx->bar0_shadow;

    /* Doorbell register at BAR0+0x110C00 (guest-visible shadow) */
    ctx->doorbell_reg = ctx->bar0_vaddr + GSP_EMU_DOORBELL_OFFSET;

    /* RPC shared memory at BAR0+0x1000. This ALIASES the guest-visible
     * shadow - it is NOT a separate allocation. shared_mem_dma stays 0 to
     * mark it as aliased so cleanup does not dma_free it. */
    ctx->shared_mem_size = GSP_EMU_SHARED_MEM_SIZE;
    ctx->shared_mem = ctx->bar0_vaddr + GSP_EMU_SHARED_MEM_OFFSET;
    ctx->shared_mem_virt = (void *)ctx->shared_mem;
    ctx->shared_mem_dma = 0;

    /* Initialize RPC client with the shadow BAR0 base and doorbell */
    ctx->rpc_client = gsp_client_init_emulation(&vgpu->dev, ctx->bar0_vaddr, ctx->doorbell_reg);
    if (IS_ERR(ctx->rpc_client)) {
        ret = PTR_ERR(ctx->rpc_client);
        nv_vgpu_err(vgpu, "Failed to init GSP RPC client: %d", ret);
        goto err_free_ctx;
    }

    /* Enable emulation mode in RPC client */
    gsp_rpc_emulation_set(true);

    /* Initialize GSP-RM msgq emulation (physical path) */
    ret = gsp_rm_emu_init(vgpu, ctx);
    if (ret) {
        nv_vgpu_err(vgpu, "Failed to init GSP-RM emulation: %d", ret);
        goto err_fini_client;
    }

    /* NOTE: MSI-X eventfd triggers are NOT copied here - QEMU wires them
     * via VFIO_DEVICE_SET_IRQS later, so gsp_emu_inject_completion_interrupt
     * looks them up live from vgpu->irqs[]. */

    /* Start emulation thread */
    ctx->thread = kthread_run(gsp_emu_thread_fn, ctx, "gsp_emu_%u", vgpu->vgpu_id);
    if (IS_ERR(ctx->thread)) {
        ret = PTR_ERR(ctx->thread);
        nv_vgpu_err(vgpu, "Failed to start GSP emulation thread: %d", ret);
        goto err_fini_client;
    }
    /* Hold a reference on the thread's task_struct so it stays valid
     * even if the thread crashes (e.g. fortify_panic). Without this,
     * cleanup could dereference a freed task_struct. */
    get_task_struct(ctx->thread);

    ctx->running = true;
    ctx->initialized = true;
    vgpu->gsp_comm_ctx = ctx; /* Reuse existing field for emulation context */

    nv_vgpu_info(vgpu, "GSP emulation engine initialized successfully");
    return 0;

err_fini_client:
    gsp_client_fini(ctx->rpc_client);
err_free_ctx:
    /* shared_mem_virt aliases the BAR0 shadow (owned by bar_emulation),
     * nothing to free here */
    kfree(ctx);
    return ret;
}

void gsp_emu_cleanup(struct nv_vgpu_device *vgpu)
{
    struct gsp_emu_context *ctx = vgpu->gsp_comm_ctx;

    if (!ctx)
        return;

    nv_vgpu_info(vgpu, "Cleaning up GSP emulation engine");

    /* Stop emulation thread with timeout */
    if (ctx->thread) {
        ctx->stopping = true;
        /* Only call kthread_stop() if the thread is still alive.
         * If the thread crashed (e.g. fortify_panic), the kernel already
         * killed it and kthread_stop() would hit a refcount saturation
         * warning on the dead task_struct. */
        if (pid_alive(ctx->thread)) {
            kthread_stop(ctx->thread);
            /* Wait for thread to actually stop with timeout (5 seconds) */
            if (!wait_for_completion_timeout(&ctx->thread_stopped, msecs_to_jiffies(5000))) {
                nv_vgpu_warn(vgpu, "GSP emulation thread did not stop in time, forcing cleanup");
            }
        } else {
            nv_vgpu_warn(vgpu, "GSP emulation thread already dead (crashed), skipping kthread_stop");
        }
        put_task_struct(ctx->thread);
        ctx->thread = NULL;
    }
    ctx->running = false;

    /* Teardown GSP-RM msgq emulation */
    gsp_rm_emu_fini(ctx);

    /* Disable emulation mode */
    gsp_rpc_emulation_set(false);

    /* Cleanup RPC client */
    if (ctx->rpc_client) {
        gsp_client_fini(ctx->rpc_client);
        ctx->rpc_client = NULL;
    }

    /* Shared memory aliases the BAR0 shadow (owned by bar_emulation) -
     * only free it if it was a standalone DMA allocation (dma != 0) */
    if (ctx->shared_mem_virt && ctx->shared_mem_dma) {
        dma_free_wc(&vgpu->pdev->dev, ctx->shared_mem_size,
                    ctx->shared_mem_virt, ctx->shared_mem_dma);
    }
    ctx->shared_mem_virt = NULL;

    /* BAR0 shadow is managed by nv_vgpu_bar_emul_init/cleanup */
    ctx->bar0_vaddr = NULL;

    vgpu->gsp_comm_ctx = NULL;
    kfree(ctx);

    nv_vgpu_info(vgpu, "GSP emulation engine cleaned up");
}

/* ============================================================
 * Public RPC Processing (for external callers)
 * ============================================================ */

int gsp_emu_process_rpc(struct gsp_emu_context *ctx)
{
    struct gsp_rpc_msg req;
    struct gsp_rpc_response rsp;
    int ret;

    if (!ctx || !ctx->initialized)
        return -ENODEV;

    gsp_emu_copy_request_from_shared(ctx, &req);

    if (req.seq_id == 0)
        return -EINVAL;

    ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp);

    gsp_emu_copy_response_to_shared(ctx, &rsp);
    gsp_emu_inject_completion_interrupt(ctx, req.seq_id);

    return ret;
}

/* ============================================================
 * Statistics
 * ============================================================ */

void gsp_emu_get_stats(struct gsp_emu_context *ctx,
                       u64 *processed, u64 *failed, u64 *total_latency_ns)
{
    if (!ctx)
        return;

    if (processed)
        *processed = ctx->processed_rpcs;
    if (failed)
        *failed = ctx->failed_rpcs;
    if (total_latency_ns)
        *total_latency_ns = ctx->total_latency_ns;
}

EXPORT_SYMBOL(gsp_emu_process_rpc);
EXPORT_SYMBOL(gsp_emu_get_stats);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Open-IOV Project");
MODULE_DESCRIPTION("GSP Emulation Engine for Host-Side vGPU");