From f6d450a8be075fdc19a874826e7649ae0149b099 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Tue, 22 Sep 2026 22:40:26 -0400 Subject: kernel: guest probe completes - nvidia-smi enumerates the emulated TU104 Root cause of the golden-image NV_ERR_NO_MEMORY: STATIC_KGR_GET_CONTEXT_ BUFFERS_INFO (0x20800a32) fell through to the generic zeroed reply, so kgraphicsInitializeDeferredStaticData() stored a 0-byte main ctx buffer size and kgrctxAllocMainCtxBuffer() did a 0-page allocation (mem_desc.c:1353). Serve the TU104 per-engine ctx buffer table (26 engine IDs x 8 GR engines; NV_U32_MAX = unsupported, which the client skips). Second blocker after that: L2 cache-op and TLB-invalidate completion registers. Turing reaches them through the virtual-register window (GPU_VREG_* = BAR0 + sriovState.virtualRegPhysOffset + offset, 0xB80000 for a non-SRIOV virtual GPU): NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE (0xB830B0, TRIGGER bit31) and the L2 invalidate/dirty-flush set (0x70000-0x70014 and 0xB80F00/0xB80F04). The driver soft-locked spinning in osDevReadReg032 on 0xB830B4 (kgmmuCheckPendingInvalidates_TU102) and timed out in kmemsysDoCacheOp_GM107; auto-clear all of them (both the plain and windowed aliases). Third: re-probe failed with GspStatusQueueInit msgqRxLink timeout because gsp_rm_emu_reset() was not called before a fresh boot, leaving rm->parsed set so the new libos args were never parsed. Reset GSP-RM state on the fresh FWSEC kick. Verified: guest nvidia-smi and nvidia-smi -L both exit 0 (full unload/reload cycle), GPU enumerated as "NVIDIA GeForce RTX 2080 SUPER" with 256MB, no NO_MEMORY / timeouts / soft lockups in the guest. Also add tools/rmq/grctx_oracle.c documenting that the host RM does not expose the ctx-buffer table to userspace (the real values are probed from the ctxsw ucode by GSP-RM), so the emulation serves synthesized sizes. --- kernel/gsp_rm_emulation.c | 90 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 16 deletions(-) (limited to 'kernel/gsp_rm_emulation.c') diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 07ba807..f18d17c 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -1201,6 +1201,74 @@ static void gsp_rm_handle_kgr_get_floorsweeping_masks( m->gpcMask, TU104_TPC_PER_GPC); } +/* + * STATIC_KGR_GET_CONTEXT_BUFFERS_INFO (0x20800a32) + * + * kgraphicsInitializeDeferredStaticData() asks the physical RM (GSP-RM) for + * the per-engine context buffer sizes; kgrctxAllocMainCtxBuffer() then sizes + * the golden image ctx buffer from engine[GRAPHICS].size. The zeroed generic + * reply left size == 0 -> a 0-page allocation -> NV_ERR_NO_MEMORY at + * mem_desc.c:1353 (kgrctxAllocMainCtxBuffer -> kgraphicsCreateGoldenImageChannel). + * + * The real physical RM probes these from the ctxsw ucode + * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and reports NV_U32_MAX + * for unsupported engine IDs. The client skips NV_U32_MAX entries in + * kgraphicsCreateGoldenImageChannel's VA reservation, so the emulated layout + * below mirrors that: page-aligned scratch sizes for the buffers the client + * allocates, NV_U32_MAX for the engine IDs Turing does not support. + */ +static void gsp_rm_handle_kgr_get_context_buffers_info( + struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + static const struct tu104_ctx_buffer_info sizes[TU104_CTX_ENGINE_ID_COUNT] = { + [TU104_CTX_ENG_GRAPHICS] = { TU104_CTX_SIZE_MAIN, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_ZCULL] = { TU104_CTX_SIZE_ZCULL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PM] = { TU104_CTX_SIZE_PM, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PREEMPT] = { TU104_CTX_SIZE_PREEMPT, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_SPILL] = { TU104_CTX_SIZE_SPILL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PAGEPOOL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_BETACB] = { TU104_CTX_SIZE_BETACB, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_RTV] = { TU104_CTX_SIZE_RTV, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PATCH] = { TU104_CTX_SIZE_PATCH, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_BUNDLE_CB] = { TU104_CTX_SIZE_BUNDLE_CB, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PAGEPOOL_GLOBAL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_ATTRIBUTE_CB] = { TU104_CTX_SIZE_ATTRIBUTE_CB, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_RTV_CB_GLOBAL] = { TU104_CTX_SIZE_RTV_CB_GLOBAL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_FECS_EVENT] = { TU104_CTX_SIZE_FECS_EVENT, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PRIV_ACCESS_MAP] = { TU104_CTX_SIZE_PRIV_ACCESS_MAP, 0x1000 }, + }; + struct tu104_gr_get_context_buffers_info_params *info = + (struct tu104_gr_get_context_buffers_info_params *)out_params; + u32 e, i; + + memset(out_params, 0, sizeof(*info)); + + for (e = 0; e < TU104_CTX_GPU_MAX_ENGINES; e++) { + for (i = 0; i < TU104_CTX_ENGINE_ID_COUNT; i++) { + /* engine IDs the chip does not support report + * NV_U32_MAX (the client skips those entries) */ + if (sizes[i].size == 0) { + info->engineContextBuffersInfo[e].engine[i].size = 0xFFFFFFFF; + info->engineContextBuffersInfo[e].engine[i].alignment = 0xFFFFFFFF; + continue; + } + info->engineContextBuffersInfo[e].engine[i].size = sizes[i].size; + info->engineContextBuffersInfo[e].engine[i].alignment = sizes[i].alignment; + } + } + + *out_params_size = sizeof(*info); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_CONTEXT_BUFFERS_INFO -> main ctx 0x%x, patch 0x%x, pm 0x%x (engines=%u x %u)", + TU104_CTX_SIZE_MAIN, TU104_CTX_SIZE_PATCH, TU104_CTX_SIZE_PM, + TU104_CTX_GPU_MAX_ENGINES, TU104_CTX_ENGINE_ID_COUNT); +} + static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { @@ -1301,6 +1369,12 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; + case TU104_CTRL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO: + gsp_rm_handle_kgr_get_context_buffers_info(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + case TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS: gsp_rm_handle_kgr_get_floorsweeping_masks(ctx, sequence, &out_status, @@ -1352,22 +1426,6 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, kfree(reply); } -/* - * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK. - * - * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the - * request's params back with status=0 so the driver's free rpc_message is a - * complete well-formed record. (The poll match is function-only; the echo - * keeps the post-poll result struct self-consistent.) - */ -/* - sizeof(struct tu104_gr_floorsweeping_masks); - *out_status = 0; - - nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_FLOORSWEEPING_MASKS -> gpcMask=0x%x tpc/gpc=%u", - m->gpcMask, TU104_TPC_PER_GPC); -} - /* * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK. * -- cgit v1.2.3