From f6d450a8be075fdc19a874826e7649ae0149b099 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Tue, 22 Sep 2026 22:40:26 -0400 Subject: kernel: guest probe completes - nvidia-smi enumerates the emulated TU104 Root cause of the golden-image NV_ERR_NO_MEMORY: STATIC_KGR_GET_CONTEXT_ BUFFERS_INFO (0x20800a32) fell through to the generic zeroed reply, so kgraphicsInitializeDeferredStaticData() stored a 0-byte main ctx buffer size and kgrctxAllocMainCtxBuffer() did a 0-page allocation (mem_desc.c:1353). Serve the TU104 per-engine ctx buffer table (26 engine IDs x 8 GR engines; NV_U32_MAX = unsupported, which the client skips). Second blocker after that: L2 cache-op and TLB-invalidate completion registers. Turing reaches them through the virtual-register window (GPU_VREG_* = BAR0 + sriovState.virtualRegPhysOffset + offset, 0xB80000 for a non-SRIOV virtual GPU): NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE (0xB830B0, TRIGGER bit31) and the L2 invalidate/dirty-flush set (0x70000-0x70014 and 0xB80F00/0xB80F04). The driver soft-locked spinning in osDevReadReg032 on 0xB830B4 (kgmmuCheckPendingInvalidates_TU102) and timed out in kmemsysDoCacheOp_GM107; auto-clear all of them (both the plain and windowed aliases). Third: re-probe failed with GspStatusQueueInit msgqRxLink timeout because gsp_rm_emu_reset() was not called before a fresh boot, leaving rm->parsed set so the new libos args were never parsed. Reset GSP-RM state on the fresh FWSEC kick. Verified: guest nvidia-smi and nvidia-smi -L both exit 0 (full unload/reload cycle), GPU enumerated as "NVIDIA GeForce RTX 2080 SUPER" with 256MB, no NO_MEMORY / timeouts / soft lockups in the guest. Also add tools/rmq/grctx_oracle.c documenting that the host RM does not expose the ctx-buffer table to userspace (the real values are probed from the ctxsw ucode by GSP-RM), so the emulation serves synthesized sizes. --- kernel/gsp_emulation.h | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) (limited to 'kernel/gsp_emulation.h') diff --git a/kernel/gsp_emulation.h b/kernel/gsp_emulation.h index 801ea78..9a8f241 100644 --- a/kernel/gsp_emulation.h +++ b/kernel/gsp_emulation.h @@ -58,10 +58,42 @@ #define GSP_EMU_REG_SEC2_MAILBOX0 0x840040 #define GSP_EMU_REG_SEC2_ENGINE 0x8403c0 +/* Base of the virtual-function register window (NV_VIRTUAL_FUNCTION_FULL_PHYS_ + * OFFSET). The client RM reaches per-VF registers through GPU_VREG_* = + * BAR0 + sriovState.virtualRegPhysOffset + offset; for a non-SRIOV virtual GPU + * that offset is 0xB80000, and 0 for a VF, hence the plain + windowed aliases + * below. */ +#define GSP_EMU_REG_FULL_PHYS_OFFSET 0xB80000 + /* NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE: driver sets TRIGGER(bit31) to - * start a TLB invalidate, then polls for bit31==0 (done). We auto-clear it. */ + * start a TLB invalidate, then polls for bit31==0 (done). We auto-clear it. + * The client RM reaches it through the GPU_VREG_* window (BAR0 + 0xB80000 + + * 0x30B0) and the physical path pokes the plain 0x30B0 alias, so watch both. */ #define GSP_EMU_REG_MMU_INVALIDATE 0x0030b0 #define GSP_EMU_MMU_INVALIDATE_TRIGGER (1U << 31) +#define GSP_EMU_REG_MMU_INVALIDATE_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x0030b0) + +/* L2 cache-op completion registers (NV_UFLUSH_*): the driver writes PENDING + * (bit 0) to issue FB flush / L2 invalidate / L2 clean+dirty flush and then + * spins in kmemsysDoCacheOp_GM107 until the polled bits read back 0. On real + * hardware the L2 clears them when the operation completes; the emulated L2 + * completes instantly, so auto-clear any pending bits. + * + * On Turing the sysmem/peermem invalidate registers are reached through the + * virtual-register window (GPU_VREG_*: BAR0 + sriovState.virtualRegPhysOffset + * + NV_VIRTUAL_FUNCTION_PRIV_L2_*); the offset is 0xB80000 for a non-SRIOV + * virtual GPU and 0 for a VF, so watch both the plain and windowed aliases. */ +#define GSP_EMU_REG_UFLUSH_BASE 0x070000 +#define GSP_EMU_REG_UFLUSH_FB_FLUSH (GSP_EMU_REG_UFLUSH_BASE + 0x00) +#define GSP_EMU_REG_UFLUSH_L2_SYSMEM_INV (GSP_EMU_REG_UFLUSH_BASE + 0x04) +#define GSP_EMU_REG_UFLUSH_L2_PEERMEM_INV (GSP_EMU_REG_UFLUSH_BASE + 0x08) +#define GSP_EMU_REG_UFLUSH_L2_CLEAN_COMPTAGS (GSP_EMU_REG_UFLUSH_BASE + 0x0c) +#define GSP_EMU_REG_UFLUSH_L2_FLUSH_DIRTY (GSP_EMU_REG_UFLUSH_BASE + 0x10) +#define GSP_EMU_REG_UFLUSH_L2_WAIT_SYS_READS (GSP_EMU_REG_UFLUSH_BASE + 0x14) + +/* NV_VIRTUAL_FUNCTION_PRIV_L2_{SYSMEM,PEERMEM}_INVALIDATE (dev_vm.h) */ +#define GSP_EMU_REG_VF_L2_SYSMEM_INV 0x000f00 +#define GSP_EMU_REG_VF_L2_PEERMEM_INV 0x000f04 /* NV_PFALCON_FALCON_DEBUGINFO (GSP falcon BAR0+0x110094): holds the CrashCat * wayfinder-L0 descriptor. If its low 16 bits == 0xdead (NV_CRASHCAT_SIGNATURE), @@ -80,11 +112,14 @@ * _osVerifyInterrupts (osVerifySystemEnvironment) fires a doorbell software * interrupt and waits for its ISR; if we don't set the pending state and * inject MSI-X, the ISR never runs and RmInitAdapter bails. */ -#define GSP_EMU_REG_FULL_PHYS_OFFSET 0xB80000 #define GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1640) #define GSP_EMU_REG_CPU_INTR_LEAF_BASE (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1000) #define GSP_EMU_REG_CPU_INTR_TOP_BASE (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1600) +/* Windowed aliases of the VF L2 invalidate registers (GPU_VREG_* path). */ +#define GSP_EMU_REG_VF_L2_SYSMEM_INV_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x000f00) +#define GSP_EMU_REG_VF_L2_PEERMEM_INV_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x000f04) + #define GSP_EMU_CPUCTL_STARTCPU 0x2 #define GSP_EMU_CPUCTL_HALTED 0x10 -- cgit v1.2.3