/* * GSP Emulation Engine Implementation * Host-side GSP emulation thread that processes RPCs from guest via shared memory */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_emulation.h" #include "gsp_rm_emulation.h" #include "gsp_rpc_client.h" #include "bar_emulation.h" #include "irq.h" #include "memory.h" /* ============================================================ * Constants * ============================================================ */ #define GSP_EMU_DOORBELL_OFFSET 0x110C00 #define GSP_EMU_SHARED_MEM_OFFSET 0x1000 #define GSP_EMU_SHARED_MEM_SIZE 4096 #define GSP_EMU_MSIX_TABLE_OFFSET 0x2000 #define GSP_EMU_MSIX_PBA_OFFSET 0x3000 #define GSP_EMU_POLL_BUSY_US 100 #define GSP_EMU_POLL_SLEEP_MIN_US 50 #define GSP_EMU_POLL_SLEEP_MAX_US 100 /* ============================================================ * Helper Functions * ============================================================ */ static inline u32 gsp_emu_read_doorbell(struct gsp_emu_context *ctx) { return readl(ctx->doorbell_reg); } static inline void gsp_emu_write_doorbell(struct gsp_emu_context *ctx, u32 val) { writel(val, ctx->doorbell_reg); } static inline void gsp_emu_copy_request_from_shared(struct gsp_emu_context *ctx, struct gsp_rpc_msg *req) { memcpy(req, ctx->shared_mem_virt, sizeof(*req)); } static inline void gsp_emu_copy_response_to_shared(struct gsp_emu_context *ctx, struct gsp_rpc_response *rsp) { memcpy(ctx->shared_mem_virt + 0x80, rsp, sizeof(*rsp)); } /* ============================================================ * GSP Falcon Boot-Watch (observability for physical GSP boot) * ============================================================ */ /* * The guest's physical GSP boot (FWSEC -> Booter -> RISCV) pokes the GSP * falcon block via the mmap'd shadow, which the host cannot trap. Poll these * BAR0 offsets and log changes so the boot sequence is visible in host dmesg. */ static const struct { u32 offset; const char *name; } gsp_emu_falcon_watch[GSP_EMU_FALCON_WATCH_COUNT] = { { 0x110040, "GSP_MAILBOX0" }, { 0x110044, "GSP_MAILBOX1" }, { 0x110080, "GSP_FALCON_OS" }, { 0x110100, "GSP_CPUCTL" }, { 0x110180, "GSP_IMEMC" }, { 0x110184, "GSP_IMEMD" }, { 0x1101c0, "GSP_DMEMC" }, { 0x1101c4, "GSP_DMEMD" }, { 0x1103c0, "GSP_ENGINE" }, { 0x110c04, "GSP_QUEUE_HEAD1" }, { 0x110c08, "GSP_QUEUE_TAIL0" }, { 0x110c0c, "GSP_QUEUE_TAIL1" }, { 0x118128, "PGC6_PLM_GFW" }, { 0x118234, "PGC6_GFW_BOOT" }, { 0x1fa824, "WPR2_ADDR_LO" }, { 0x1fa828, "WPR2_ADDR_HI" }, { 0x001438, "VBIOS_SCRATCH_0E" }, { 0x840040, "SEC2_MAILBOX0" }, { 0x840100, "SEC2_CPUCTL" }, { 0x8403c0, "SEC2_ENGINE" }, }; static void gsp_emu_poll_falcon_watch(struct gsp_emu_context *ctx) { int i; for (i = 0; i < GSP_EMU_FALCON_WATCH_COUNT; i++) { u32 val = readl(ctx->bar0_vaddr + gsp_emu_falcon_watch[i].offset); if (!ctx->falcon_watch_valid) { ctx->falcon_watch_last[i] = val; continue; } if (val != ctx->falcon_watch_last[i]) { nv_vgpu_info(ctx->vgpu, "FALCON %s (0x%x): 0x%08x -> 0x%08x", gsp_emu_falcon_watch[i].name, gsp_emu_falcon_watch[i].offset, ctx->falcon_watch_last[i], val); ctx->falcon_watch_last[i] = val; } } ctx->falcon_watch_valid = true; } /* ============================================================ * Physical GSP Boot Emulation (FWSEC / Booter falcon auto-complete) * ============================================================ */ /* Register offsets + CPUCTL bits live in gsp_emulation.h (shared with * gsp_rm_emulation.c), as are gsp_emu_shadow_read32/write32. */ enum gsp_emu_boot_stage { GSP_BOOT_STAGE_IDLE = 0, /* waiting for first kick (FWSEC) */ GSP_BOOT_STAGE_FWSEC_DONE, /* FWSEC FRTS completed */ GSP_BOOT_STAGE_BOOTER_DONE, /* Booter Load completed */ }; /* * gsp_emu_clear_boot_state - return the emulated GSP to a cold state. * * On real hardware, driver unload tears down WPR2 (Booter Unload) so the * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce * no observable BAR0 writes on unload, so callers must trigger this on * indirect death signals (ENGINE reset, GSP-RM queue going stale). * Clears WPR2, RISCV active status and the falcon stage tracker, and * drops the parsed GSP-RM queue state. */ /* * gsp_emu_clear_boot_state - return the emulated GSP to a cold state. * * On real hardware, driver unload tears down WPR2 (Booter Unload) so the * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce * no observable BAR0 writes on unload, so callers must trigger this on * indirect death signals (ENGINE reset, GSP-RM queue going stale). * * IMPORTANT: on the Booter-unload kick we clear the FALCON boot state * (WPR2/RISCV/stages) but PRESERVE the GSP-RM queue state. The driver * sends cleanup FREEs AFTER the Booter-unload kick; if we reset the queue * here, those frees time out (fn-10). The queue state is only cleared when * the queue is genuinely gone (garbage elements, insane writePtr, ENGINE * reset, or a new probe writing fresh libos args to MAILBOX0/1). */ void gsp_emu_clear_boot_state(struct gsp_emu_context *ctx, const char *reason) { nv_vgpu_info(ctx->vgpu, "GSP-BOOT: clearing boot state (%s)", reason); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO, 0); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX0, 0); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX1, 0); ctx->boot_stage = GSP_BOOT_STAGE_IDLE; ctx->flcn_kick_count = 0; /* Do NOT reset the GSP-RM queue state here - the driver is still * sending cleanup frees on it. Only reset on genuine queue death * (garbage elements, insane writePtr) or ENGINE reset. */ } /* * gsp_emu_poll_boot - drive the guest's physical GSP boot to completion. * * The guest driver kicks falcon ucodes (FWSEC for FRTS, then Booter Load for * GSP-RM) by writing STARTCPU to the GSP falcon CPUCTL, then polls for HALTED * plus stage-specific result registers. The auto-boot waits (GFW) are already * satisfied by the state pre-set in nv_vgpu_bar_emul_init. Here we watch for * STARTCPU kicks and complete each stage so the guest believes its falcon * ucodes ran successfully. */ static void gsp_emu_poll_boot(struct gsp_emu_context *ctx) { u32 gsp_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_CPUCTL); u32 sec2_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_SEC2_CPUCTL); u32 engine = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_ENGINE); u32 sw_vector; /* Software-interrupt trigger: the driver writes a vector to * CPU_INTR_LEAF_TRIGGER to fire it (the osVerifyInterrupts sanity * test uses the doorbell vector). Set the pending bits in the LEAF * and TOP interrupt-tree registers and inject MSI-X so the guest's * ISR runs - without this the sanity test times out and * osVerifySystemEnvironment fails, bailing RmInitAdapter. */ sw_vector = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER); if (sw_vector) { u32 leaf_idx = sw_vector / 32; u32 leaf_bit = sw_vector % 32; u32 subtree = sw_vector / 64; u32 top_idx = subtree / 32; u32 top_bit = subtree % 32; u32 leaf_val = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4); u32 top_val = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4, leaf_val | (1U << leaf_bit)); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4, top_val | (1U << top_bit)); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER, 0); nv_vgpu_info(ctx->vgpu, "GSP-BOOT: SW interrupt vector %u -> LEAF[%u]|=0x%x TOP[%u]|=0x%x, MSI-X", sw_vector, leaf_idx, 1U << leaf_bit, top_idx, 1U << top_bit); gsp_emu_inject_completion_interrupt(ctx, 0); } /* TLB invalidate auto-complete: the driver sets TRIGGER then spins * polling for it to clear (kgmmuCheckPendingInvalidates_TU102 reads * GPU_VREG_RD32(NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE) = * BAR0 + 0xB80000 + 0x30B0; the physical path uses the plain alias). * Report instant completion or the driver soft-locks spinning inside * osDevReadReg032. */ { static const u32 tlb_inv_regs[] = { GSP_EMU_REG_MMU_INVALIDATE, GSP_EMU_REG_MMU_INVALIDATE_WIN, }; unsigned int i; for (i = 0; i < ARRAY_SIZE(tlb_inv_regs); i++) { u32 val = gsp_emu_shadow_read32(ctx, tlb_inv_regs[i]); if (!(val & GSP_EMU_MMU_INVALIDATE_TRIGGER)) continue; gsp_emu_shadow_write32(ctx, tlb_inv_regs[i], val & ~GSP_EMU_MMU_INVALIDATE_TRIGGER); pr_info_ratelimited("vgsp-boot: MMU_INVALIDATE[0x%x] trigger auto-cleared (val=0x%x)\n", tlb_inv_regs[i], val); } } /* L2 cache-op auto-complete: kmemsysDoCacheOp_GM107 writes PENDING * (bit 0) into the NV_UFLUSH_* / VF L2 invalidate registers and spins * until the polled bits read back 0 (FB flush, L2 sysmem/peermem * invalidate, L2 clean, L2 flush-dirty, L2 wait-for-sys-reads). On * Turing the sysmem/peermem invalidate registers go through the * GPU_VREG_* window (BAR0 + 0xB80000 + NV_VIRTUAL_FUNCTION_PRIV_L2_*), * so cover both the plain and windowed aliases. The emulated L2 has * nothing to do, so clear them as soon as they are observed. */ { static const u32 uflush_regs[] = { GSP_EMU_REG_UFLUSH_FB_FLUSH, GSP_EMU_REG_UFLUSH_L2_SYSMEM_INV, GSP_EMU_REG_UFLUSH_L2_PEERMEM_INV, GSP_EMU_REG_UFLUSH_L2_CLEAN_COMPTAGS, GSP_EMU_REG_UFLUSH_L2_FLUSH_DIRTY, GSP_EMU_REG_UFLUSH_L2_WAIT_SYS_READS, GSP_EMU_REG_VF_L2_SYSMEM_INV, GSP_EMU_REG_VF_L2_PEERMEM_INV, GSP_EMU_REG_VF_L2_SYSMEM_INV_WIN, GSP_EMU_REG_VF_L2_PEERMEM_INV_WIN, }; unsigned int i; for (i = 0; i < ARRAY_SIZE(uflush_regs); i++) { u32 val = gsp_emu_shadow_read32(ctx, uflush_regs[i]); if (!val) continue; gsp_emu_shadow_write32(ctx, uflush_regs[i], 0); pr_info_ratelimited("vgsp-boot: UFLUSH[0x%x] 0x%x auto-cleared\n", uflush_regs[i], val); } } /* CrashCat wayfinder suppression: if the GSP falcon's DEBUGINFO * register holds a valid wayfinder-L0 signature (0xdead), the driver's * kgspHealthCheck (run on every RPC drain) reads "reports" from * garbage and loops forever in nv_open_q, which then can't poll RPCs * (fn-10 FREE timeouts). Force it invalid - our emulated GSP has no * crashcat reports to offer. */ { u32 wfl0 = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO); if ((wfl0 & 0xffff) == 0xdead) { gsp_emu_shadow_write32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO, 0); nv_vgpu_info(ctx->vgpu, "GSP-BOOT: cleared CrashCat wayfinder (was 0x%x)", wfl0); } else if (wfl0) { pr_info_ratelimited("vgsp-boot: DEBUGINFO nonzero (0x%x), not a wayfinder\n", wfl0); } } /* Temporary: watch PCFG / link-cap mirror offsets for guest writes */ { static const u32 watch[] = { 0x000004, 0x000084, 0x880000, 0x880004, 0x880080, 0x880084, 0x880088, 0x88008c, 0xb80084, 0xb88084, 0xc08084, }; static u32 last[ARRAY_SIZE(watch)]; unsigned int i; for (i = 0; i < ARRAY_SIZE(watch); i++) { u32 v = gsp_emu_shadow_read32(ctx, watch[i]); if (v != last[i]) { nv_vgpu_info(ctx->vgpu, "GSP-BOOT: PCFG[0x%x] = 0x%x (was 0x%x)", watch[i], v, last[i]); last[i] = v; } } } /* GSP ENGINE reset asserted -> falcon rebooted, restart stage tracking */ if ((engine & 0x1) && ctx->boot_stage != GSP_BOOT_STAGE_IDLE) { nv_vgpu_info(ctx->vgpu, "GSP-BOOT: falcon reset, back to IDLE"); /* ENGINE reset is a full GPU reset: the GSP-RM queue state is * stale, reset it too. */ gsp_emu_clear_boot_state(ctx, "ENGINE reset"); gsp_rm_emu_reset(ctx); return; } /* * PRAMIN window translation: the guest programs NV_PBUS_BAR0_WINDOW * (BAR0+0x1700) to select which vidmem page the PRAMIN window * (BAR0+0x700000) maps to. On real hardware the PRAMIN window is a * CPU aperture into vidmem at (BASE << 16) + offset. In our shadow * the PRAMIN window is just part of the shadow buffer - it does NOT * alias the emulated BAR1 (vidmem). * * The BAR2 verify test writes through the vBar2 aperture (PRAMIN) * and reads back through the BAR0 window (also PRAMIN), expecting * the same data. Both paths hit the same shadow location, so the * test data IS consistent. We do NOT sync the PRAMIN window from * the emulated BAR1 - the sync would overwrite the test data * written by the driver through the vBar2 aperture. * * The BAR0_WINDOW_BASE register is captured for observability only. */ { u32 bar0_win = gsp_emu_shadow_read32(ctx, 0x1700); u32 base = bar0_win & 0xFFFFFF; u32 target = (bar0_win >> 24) & 0x3; if (!ctx->bar0_window_valid || base != ctx->bar0_window_base || target != ctx->bar0_window_target) { static u32 last_pramin_val; u32 pramin_val = gsp_emu_shadow_read32(ctx, 0x70e000); ctx->bar0_window_base = base; ctx->bar0_window_target = target; ctx->bar0_window_valid = true; nv_vgpu_info(ctx->vgpu, "GSP-BOOT: BAR0_WINDOW base=0x%x target=%u, PRAMIN[0xe000]=0x%08x (was 0x%08x)", base, target, pramin_val, last_pramin_val); last_pramin_val = pramin_val; } } /* * FWSEC kick on the GSP falcon (FRTS setup). Complete it: * - CPUCTL: clear STARTCPU, set HALTED * - VBIOS_SCRATCH[0x0E] FRTS_ERR_CODE = NONE (0) * - WPR2_ADDR_HI != 0, WPR2_ADDR_LO = frtsOffset >> 0xc (in _VAL bits31:4) */ if ((gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU) && ctx->boot_stage == GSP_BOOT_STAGE_IDLE) { /* A fresh FWSEC kick means a new GSP boot: drop any GSP-RM * queue state left from the previous probe (the driver has * already torn down its old queue). Without this the next * Booter completion would skip the chain re-parse * (rm->parsed still set) and GspStatusQueueInit would time * out on re-probe. */ gsp_rm_emu_reset(ctx); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_VBIOS_SCRATCH_0E, 0); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0x100000); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO, ctx->wpr2_addr_lo); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL, GSP_EMU_CPUCTL_HALTED); ctx->boot_stage = GSP_BOOT_STAGE_FWSEC_DONE; ctx->flcn_kick_count++; nv_vgpu_info(ctx->vgpu, "GSP-BOOT: FWSEC kick completed " "(WPR2_LO=0x%x)", ctx->wpr2_addr_lo); return; } /* * Booter Load kick on the SEC2 falcon (loads/authenticates the GSP-RM * image and starts the GSP RISCV). Complete it: * - SEC2 CPUCTL: clear STARTCPU, set HALTED * - SEC2 MAILBOX0 = 0 (booter error code, 0 = success) * - GSP RISCV_STATUS = ACTIVE (Booter started the RISCV; * kflcnIsRiscvActive passes) */ if ((sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) && ctx->boot_stage == GSP_BOOT_STAGE_FWSEC_DONE) { gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0x1); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL, GSP_EMU_CPUCTL_HALTED); ctx->boot_stage = GSP_BOOT_STAGE_BOOTER_DONE; ctx->flcn_kick_count++; nv_vgpu_info(ctx->vgpu, "GSP-BOOT: Booter kick completed, " "RISCV active"); /* The driver has programmed the libos init args address into * GSP MAILBOX0/1 by now; parse the queue chain, link the * status queue and emit GSP_INIT_DONE. */ gsp_rm_emu_on_booter_done(ctx); return; } /* Any other STARTCPU kick (resume/unload/etc.): complete it. * SEC2 kicks report their ucode result in SEC2_MAILBOX0 (0=success) - * e.g. Booter Unload during GPU shutdown. Without clearing it the * driver reads a stale/garbage error code (0xff) and the unload * fails, killing the subsequent re-init. A SEC2 kick after a * completed boot is a GSP shutdown/reload: drop the boot state * (GSP no longer active, queue dead). Clear boot state FIRST so * WPR2 is already down when the driver sees CPUCTL=HALTED and * checks kgspIsWpr2Up (avoids "Booter Unload: WPR2 is still up"). */ if (gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU) gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL, GSP_EMU_CPUCTL_HALTED); if (sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) { if (ctx->boot_stage == GSP_BOOT_STAGE_BOOTER_DONE) gsp_emu_clear_boot_state(ctx, "Booter unload/reload kick"); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0); gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL, GSP_EMU_CPUCTL_HALTED); } } /* ============================================================ * RPC Processing * ============================================================ */ static int gsp_emu_dispatch_rpc(struct gsp_emu_context *ctx, struct gsp_rpc_msg *req, struct gsp_rpc_response *rsp) { int ret = 0; ktime_t start = ktime_get(); memset(rsp, 0, sizeof(*rsp)); rsp->type = req->type; rsp->seq_id = req->seq_id; switch (req->type) { case GSP_RPC_ALLOC: { struct gsp_alloc_req *areq = (struct gsp_alloc_req *)req->payload; struct gsp_alloc_rsp arsp = {0}; u64 gpu_va = 0, handle = 0, actual_size = 0; ret = gsp_rpc_alloc(ctx->rpc_client, areq->size, areq->alignment, areq->mem_type, areq->flags, areq->addr_hint, areq->pid, &gpu_va, &handle, &actual_size); if (!ret) { arsp.gpu_va = gpu_va; arsp.handle = handle; arsp.size = actual_size; memcpy(rsp->result, &arsp, sizeof(arsp)); } break; } case GSP_RPC_FREE: { u64 handle = req->payload[0]; ret = gsp_rpc_free(ctx->rpc_client, handle); break; } case GSP_RPC_MAP: { struct gsp_map_req *mreq = (struct gsp_map_req *)req->payload; struct gsp_map_rsp mrsp = {0}; u64 mapped_va = 0; ret = gsp_rpc_map(ctx->rpc_client, mreq->handle, mreq->gpu_va, mreq->size, mreq->prot, mreq->flags, mreq->offset, &mapped_va); if (!ret) { mrsp.gpu_va = mapped_va; memcpy(rsp->result, &mrsp, sizeof(mrsp)); } break; } case GSP_RPC_UNMAP: { u64 handle = req->payload[0]; u64 gpu_va = req->payload[1]; u64 size = req->payload[2]; ret = gsp_rpc_unmap(ctx->rpc_client, handle, gpu_va, size); break; } case GSP_RPC_PIN: { struct gsp_pin_req *preq = (struct gsp_pin_req *)req->payload; struct gsp_pin_rsp prsp = {0}; u64 dma_addr = 0; u32 num_pages = 0; ret = gsp_rpc_pin(ctx->rpc_client, preq->handle, preq->offset, preq->size, preq->flags, &dma_addr, &num_pages); if (!ret) { prsp.dma_addr = dma_addr; prsp.num_pages = num_pages; memcpy(rsp->result, &prsp, sizeof(prsp)); } break; } case GSP_RPC_UNPIN: { u64 handle = req->payload[0]; u64 offset = req->payload[1]; u64 size = req->payload[2]; ret = gsp_rpc_unpin(ctx->rpc_client, handle, offset, size); break; } case GSP_RPC_SCHEDULE: { struct gsp_sched_req *sreq = (struct gsp_sched_req *)req->payload; ret = gsp_rpc_schedule(ctx->rpc_client, sreq->work_handle, sreq->callback, sreq->arg, sreq->priority, sreq->flags); break; } case GSP_RPC_YIELD: { ret = gsp_rpc_yield(ctx->rpc_client); break; } case GSP_RPC_NOTIFY: { /* NOTIFY is handled via registration, not direct RPC */ nv_vgpu_info(ctx->vgpu, "NOTIFY RPC received (type=0x%x)", req->subcmd); ret = 0; break; } case GSP_RPC_BAR_TRAP: { struct gsp_bar_trap_req *breq = (struct gsp_bar_trap_req *)req->payload; struct gsp_bar_trap_rsp brsp = {0}; u64 ret_value = 0; ret = gsp_rpc_bar_trap(ctx->rpc_client, breq->bar_addr, breq->access_type, breq->size, breq->value, breq->bar_num, &ret_value); if (!ret) { brsp.value = ret_value; brsp.handled = 1; memcpy(rsp->result, &brsp, sizeof(brsp)); } break; } case GSP_RPC_MIGRATE: { struct gsp_migrate_req *mreq = (struct gsp_migrate_req *)req->payload; struct gsp_migrate_rsp mrsp = {0}; u64 checkpoint = 0; if (mreq->flags == GSP_MIGRATE_SUBCMD_SAVE) { ret = gsp_rpc_migrate_save(ctx->rpc_client, mreq->src_gpu_id, mreq->dst_gpu_id, mreq->memory_bitmap, mreq->page_table_ptr, &checkpoint); if (!ret) { mrsp.checkpoint = checkpoint; memcpy(rsp->result, &mrsp, sizeof(mrsp)); } } else if (mreq->flags == GSP_MIGRATE_SUBCMD_RESTORE) { ret = gsp_rpc_migrate_restore(ctx->rpc_client, mreq->src_gpu_id, mreq->dst_gpu_id, mreq->memory_bitmap, mreq->page_table_ptr, mreq->context); } break; } case GSP_RPC_DEBUG: { nv_vgpu_info(ctx->vgpu, "DEBUG RPC received (seq=%u)", req->seq_id); ret = 0; break; } default: nv_vgpu_warn(ctx->vgpu, "Unknown RPC type: 0x%x", req->type); rsp->status = -ENOTSUPP; ret = -ENOTSUPP; break; } if (ret) rsp->status = ret; ctx->total_latency_ns += ktime_to_ns(ktime_sub(ktime_get(), start)); return ret; } /* ============================================================ * MSI-X Interrupt Injection * ============================================================ */ void gsp_emu_inject_completion_interrupt(struct gsp_emu_context *ctx, u32 seq_id) { struct eventfd_ctx *trigger = NULL; /* Write to MSI-X PBA (Pending Bit Array) at BAR0+0x3000 */ if (ctx->bar0_vaddr) { u32 *pba = (u32 *)(ctx->bar0_vaddr + GSP_EMU_MSIX_PBA_OFFSET); writel(1, pba); /* Set pending bit for vector 0 */ } /* Look up the vector-0 trigger LIVE: QEMU wires eventfds via * VFIO_DEVICE_SET_IRQS after gsp_emu_init runs, so an init-time copy * would be stale (NULL). */ if (ctx->vgpu) trigger = ctx->vgpu->irqs[0].trigger; /* Signal eventfd for VFIO interrupt delivery */ if (trigger) eventfd_signal(trigger); } /* ============================================================ * Emulation Thread * ============================================================ */ static int gsp_emu_thread_fn(void *data) { struct gsp_emu_context *ctx = data; struct gsp_rpc_msg req; struct gsp_rpc_response rsp; u32 doorbell_val; int ret; nv_vgpu_info(ctx->vgpu, "GSP emulation thread started"); while (!kthread_should_stop()) { /* Read doorbell */ doorbell_val = gsp_emu_read_doorbell(ctx); /* Watch GSP falcon boot registers (physical GSP boot tracing) */ gsp_emu_poll_falcon_watch(ctx); /* Drive the guest's physical GSP boot to completion */ gsp_emu_poll_boot(ctx); /* GSP-RM msgq emulation (physical path: sysmem cmd/status queues) */ gsp_rm_emu_poll(ctx); if (doorbell_val != 0) { /* ACK doorbell immediately */ gsp_emu_write_doorbell(ctx, 0); /* Copy request from shared memory */ gsp_emu_copy_request_from_shared(ctx, &req); /* Validate sequence ID */ if (req.seq_id == 0) { nv_vgpu_warn(ctx->vgpu, "Invalid seq_id=0, ignoring RPC"); continue; } /* Process RPC */ ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp); /* Write response to shared memory */ gsp_emu_copy_response_to_shared(ctx, &rsp); /* Inject completion interrupt */ gsp_emu_inject_completion_interrupt(ctx, req.seq_id); if (ret) ctx->failed_rpcs++; else ctx->processed_rpcs++; } else { /* Adaptive polling: busy-wait first, then sleep (interruptible for kthread_stop) */ static int poll_count = 0; if (poll_count < GSP_EMU_POLL_BUSY_US) { udelay(1); poll_count++; } else { /* Use interruptible sleep so kthread_stop can wake us */ schedule_timeout_interruptible(usecs_to_jiffies(GSP_EMU_POLL_SLEEP_MAX_US)); poll_count = 0; } } } nv_vgpu_info(ctx->vgpu, "GSP emulation thread stopped (processed=%llu, failed=%llu)", ctx->processed_rpcs, ctx->failed_rpcs); complete(&ctx->thread_stopped); return 0; } /* ============================================================ * Initialization & Cleanup * ============================================================ */ int gsp_emu_init(struct nv_vgpu_device *vgpu) { struct nv_vgpu_bar_emul_ctx *bar_ctx; struct gsp_emu_context *ctx; int ret = 0; nv_vgpu_info(vgpu, "Initializing GSP emulation engine"); ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; ctx->vgpu = vgpu; ctx->running = false; ctx->initialized = false; ctx->stopping = false; init_completion(&ctx->thread_stopped); spin_lock_init(&ctx->lock); /* Physical GSP boot emulation state. wpr2_addr_lo matches what the * guest driver expects WPR2_ADDR_LO to read after FWSEC FRTS: * frtsOffset = fbSize - reservedFromEnd. With fbSize = 256MB * (LOCAL_MEMORY_RANGE) and the 570.144 image's 2MB reservation, * frtsOffset = 0x10000000 - 0x200000 = 0xfe00000; the register's * _VAL field (bits31:4) must equal frtsOffset >> 12 = 0xfe00, so the * raw register value is 0xfe000. */ ctx->boot_stage = 0; /* GSP_BOOT_STAGE_IDLE */ ctx->flcn_kick_count = 0; ctx->wpr2_addr_lo = 0x000fe000; /* The guest's BAR0 is the emulation shadow (vzalloc'd by * nv_vgpu_bar_emul_init, mapped to the guest via VFIO mmap) - NOT the * host physical BAR0 (vgpu->bars[0].vaddr, which belongs to the real GPU * and the host nvidia driver). All guest doorbell writes, RPC messages, * MSI-X table writes and PBA reads land in the shadow, so the emulation * engine must operate on the shadow exclusively. Polling the physical * BAR0 would never see guest traffic and would corrupt host GPU state. */ bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx; if (!bar_ctx || !bar_ctx->bar0_shadow) { nv_vgpu_err(vgpu, "BAR0 emulation shadow not available"); ret = -ENODEV; goto err_free_ctx; } ctx->bar0_vaddr = (void __iomem *)bar_ctx->bar0_shadow; /* Doorbell register at BAR0+0x110C00 (guest-visible shadow) */ ctx->doorbell_reg = ctx->bar0_vaddr + GSP_EMU_DOORBELL_OFFSET; /* RPC shared memory at BAR0+0x1000. This ALIASES the guest-visible * shadow - it is NOT a separate allocation. shared_mem_dma stays 0 to * mark it as aliased so cleanup does not dma_free it. */ ctx->shared_mem_size = GSP_EMU_SHARED_MEM_SIZE; ctx->shared_mem = ctx->bar0_vaddr + GSP_EMU_SHARED_MEM_OFFSET; ctx->shared_mem_virt = (void *)ctx->shared_mem; ctx->shared_mem_dma = 0; /* Initialize RPC client with the shadow BAR0 base and doorbell */ ctx->rpc_client = gsp_client_init_emulation(&vgpu->dev, ctx->bar0_vaddr, ctx->doorbell_reg); if (IS_ERR(ctx->rpc_client)) { ret = PTR_ERR(ctx->rpc_client); nv_vgpu_err(vgpu, "Failed to init GSP RPC client: %d", ret); goto err_free_ctx; } /* Enable emulation mode in RPC client */ gsp_rpc_emulation_set(true); /* Initialize GSP-RM msgq emulation (physical path) */ ret = gsp_rm_emu_init(vgpu, ctx); if (ret) { nv_vgpu_err(vgpu, "Failed to init GSP-RM emulation: %d", ret); goto err_fini_client; } /* NOTE: MSI-X eventfd triggers are NOT copied here - QEMU wires them * via VFIO_DEVICE_SET_IRQS later, so gsp_emu_inject_completion_interrupt * looks them up live from vgpu->irqs[]. */ /* Start emulation thread */ ctx->thread = kthread_run(gsp_emu_thread_fn, ctx, "gsp_emu_%u", vgpu->vgpu_id); if (IS_ERR(ctx->thread)) { ret = PTR_ERR(ctx->thread); nv_vgpu_err(vgpu, "Failed to start GSP emulation thread: %d", ret); goto err_fini_client; } /* Hold a reference on the thread's task_struct so it stays valid * even if the thread crashes (e.g. fortify_panic). Without this, * cleanup could dereference a freed task_struct. */ get_task_struct(ctx->thread); ctx->running = true; ctx->initialized = true; vgpu->gsp_comm_ctx = ctx; /* Reuse existing field for emulation context */ nv_vgpu_info(vgpu, "GSP emulation engine initialized successfully"); return 0; err_fini_client: gsp_client_fini(ctx->rpc_client); err_free_ctx: /* shared_mem_virt aliases the BAR0 shadow (owned by bar_emulation), * nothing to free here */ kfree(ctx); return ret; } void gsp_emu_cleanup(struct nv_vgpu_device *vgpu) { struct gsp_emu_context *ctx = vgpu->gsp_comm_ctx; if (!ctx) return; nv_vgpu_info(vgpu, "Cleaning up GSP emulation engine"); /* Stop emulation thread with timeout */ if (ctx->thread) { ctx->stopping = true; /* Only call kthread_stop() if the thread is still alive. * If the thread crashed (e.g. fortify_panic), the kernel already * killed it and kthread_stop() would hit a refcount saturation * warning on the dead task_struct. */ if (pid_alive(ctx->thread)) { kthread_stop(ctx->thread); /* Wait for thread to actually stop with timeout (5 seconds) */ if (!wait_for_completion_timeout(&ctx->thread_stopped, msecs_to_jiffies(5000))) { nv_vgpu_warn(vgpu, "GSP emulation thread did not stop in time, forcing cleanup"); } } else { nv_vgpu_warn(vgpu, "GSP emulation thread already dead (crashed), skipping kthread_stop"); } put_task_struct(ctx->thread); ctx->thread = NULL; } ctx->running = false; /* Teardown GSP-RM msgq emulation */ gsp_rm_emu_fini(ctx); /* Disable emulation mode */ gsp_rpc_emulation_set(false); /* Cleanup RPC client */ if (ctx->rpc_client) { gsp_client_fini(ctx->rpc_client); ctx->rpc_client = NULL; } /* Shared memory aliases the BAR0 shadow (owned by bar_emulation) - * only free it if it was a standalone DMA allocation (dma != 0) */ if (ctx->shared_mem_virt && ctx->shared_mem_dma) { dma_free_wc(&vgpu->pdev->dev, ctx->shared_mem_size, ctx->shared_mem_virt, ctx->shared_mem_dma); } ctx->shared_mem_virt = NULL; /* BAR0 shadow is managed by nv_vgpu_bar_emul_init/cleanup */ ctx->bar0_vaddr = NULL; vgpu->gsp_comm_ctx = NULL; kfree(ctx); nv_vgpu_info(vgpu, "GSP emulation engine cleaned up"); } /* ============================================================ * Public RPC Processing (for external callers) * ============================================================ */ int gsp_emu_process_rpc(struct gsp_emu_context *ctx) { struct gsp_rpc_msg req; struct gsp_rpc_response rsp; int ret; if (!ctx || !ctx->initialized) return -ENODEV; gsp_emu_copy_request_from_shared(ctx, &req); if (req.seq_id == 0) return -EINVAL; ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp); gsp_emu_copy_response_to_shared(ctx, &rsp); gsp_emu_inject_completion_interrupt(ctx, req.seq_id); return ret; } /* ============================================================ * Statistics * ============================================================ */ void gsp_emu_get_stats(struct gsp_emu_context *ctx, u64 *processed, u64 *failed, u64 *total_latency_ns) { if (!ctx) return; if (processed) *processed = ctx->processed_rpcs; if (failed) *failed = ctx->failed_rpcs; if (total_latency_ns) *total_latency_ns = ctx->total_latency_ns; } EXPORT_SYMBOL(gsp_emu_process_rpc); EXPORT_SYMBOL(gsp_emu_get_stats); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Open-IOV Project"); MODULE_DESCRIPTION("GSP Emulation Engine for Host-Side vGPU");