kernel: guest probe completes - nvidia-smi enumerates the emulated TU104

Root cause of the golden-image NV_ERR_NO_MEMORY: STATIC_KGR_GET_CONTEXT_
BUFFERS_INFO (0x20800a32) fell through to the generic zeroed reply, so
kgraphicsInitializeDeferredStaticData() stored a 0-byte main ctx buffer
size and kgrctxAllocMainCtxBuffer() did a 0-page allocation
(mem_desc.c:1353). Serve the TU104 per-engine ctx buffer table (26 engine
IDs x 8 GR engines; NV_U32_MAX = unsupported, which the client skips).

Second blocker after that: L2 cache-op and TLB-invalidate completion
registers. Turing reaches them through the virtual-register window
(GPU_VREG_* = BAR0 + sriovState.virtualRegPhysOffset + offset, 0xB80000
for a non-SRIOV virtual GPU): NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE
(0xB830B0, TRIGGER bit31) and the L2 invalidate/dirty-flush set
(0x70000-0x70014 and 0xB80F00/0xB80F04). The driver soft-locked spinning
in osDevReadReg032 on 0xB830B4 (kgmmuCheckPendingInvalidates_TU102) and
timed out in kmemsysDoCacheOp_GM107; auto-clear all of them (both the
plain and windowed aliases).

Third: re-probe failed with GspStatusQueueInit msgqRxLink timeout because
gsp_rm_emu_reset() was not called before a fresh boot, leaving rm->parsed
set so the new libos args were never parsed. Reset GSP-RM state on the
fresh FWSEC kick.

Verified: guest nvidia-smi and nvidia-smi -L both exit 0 (full
unload/reload cycle), GPU enumerated as "NVIDIA GeForce RTX 2080 SUPER"
with 256MB, no NO_MEMORY / timeouts / soft lockups in the guest.

Also add tools/rmq/grctx_oracle.c documenting that the host RM does not
expose the ctx-buffer table to userspace (the real values are probed from
the ctxsw ucode by GSP-RM), so the emulation serves synthesized sizes.

1 files changed, 189 insertions, 0 deletions

diff --git a/tools/rmq/grctx_oracle.c b/tools/rmq/grctx_oracle.c
new file mode 100644
index 0000000..a2fc5fa
--- /dev/null
+++ b/tools/rmq/grctx_oracle.c
@@ -0,0 +1,189 @@
+/* grctx_oracle - query the host NVIDIA RM for the engine context buffer sizes.
+ *
+ * Goal: recover the table that GSP-RM serves for
+ * NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO (0x20800a32),
+ * so the emulation layer does not have to guess the per-engine ctx buffer
+ * sizes (kernel/gsp_rm_tu104.h: TU104_CTX_SIZE_*).
+ *
+ * Outcome (2026-09-22): NOT REACHABLE from userspace on a bare-metal PF:
+ *   - 0x801707 NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES
+ *       hObject=hDevice  -> 0x57 NV_ERR_OBJECT_NOT_FOUND (not exported)
+ *       hObject=hClient  -> 0x56 NV_ERR_NOT_SUPPORTED (bound to the
+ *                           _92bfc3 hal: the client RM does not manage ctx
+ *                           buffers on a PF, only a VF's VGPU_STATIC_INFO)
+ *   - 0x2080122d NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES
+ *       (subdevice)      -> 0x57 / 0x56, never returns sizes
+ *   - The producing control (0x20800a32) is internal-only.
+ * The real values are probed from the ctxsw ucode by the physical RM
+ * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and are internal to
+ * GSP-RM, so the emulation currently serves synthesized page-aligned sizes
+ * (see tu104_ctx_buffer_info in kernel/gsp_rm_tu104.h).
+ *
+ * Kept as a reference for the RM client bring-up sequence and for a future
+ * attempt via a kernel-side RM client.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <stdint.h>
+
+typedef uint32_t NvU32;
+typedef uint64_t NvU64;
+typedef uint32_t NvHandle;
+typedef uint32_t NvV32;
+typedef uint64_t NvP64;
+
+#define NV_IOCTL_MAGIC      'F'
+#define NV_ESC_RM_ALLOC     0x2B
+#define NV_ESC_RM_CONTROL   0x2A
+
+#define NV01_ROOT           0x0
+#define NV01_DEVICE_0       0x80
+#define NV20_SUBDEVICE_0    0x2080
+
+typedef struct {
+    NvHandle hRoot;
+    NvHandle hObjectParent;
+    NvHandle hObjectNew;
+    NvV32    hClass;
+    NvP64    pAllocParms __attribute__((aligned(8)));
+    NvU32    paramsSize;
+    NvV32    status;
+} NVOS21_PARAMETERS;
+
+typedef struct {
+    NvHandle hRoot;
+    NvHandle hObjectParent;
+    NvHandle hObjectNew;
+    NvV32    hClass;
+    NvP64    pAllocParms __attribute__((aligned(8)));
+    NvP64    pRightsRequested __attribute__((aligned(8)));
+    NvU32    paramsSize;
+    NvU32    flags;
+    NvV32    status;
+} NVOS64_PARAMETERS;
+
+typedef struct {
+    NvHandle hClient;
+    NvHandle hObject;
+    NvV32    cmd;
+    NvU32    flags;
+    NvP64    params __attribute__((aligned(8)));
+    NvU32    paramsSize;
+    NvV32    status;
+} NVOS54_PARAMETERS;
+
+#define NV_IOCTL_RM_ALLOC   _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS21_PARAMETERS)
+#define NV_IOCTL_RM_CONTROL _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS)
+
+#define NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES 0x801707U
+#define NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES   0x2080122dU
+#define ENGINE_ID_COUNT 0x1a
+
+typedef struct {
+    NvU32 engineId;
+    NvU32 alignment;
+    NvU32 size;
+} FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS;
+
+typedef struct {
+    NvU32 flags;
+    NvU32 pad;
+    NvU64 route;
+} GR_ROUTE_INFO;
+
+typedef struct {
+    GR_ROUTE_INFO grRouteInfo;
+    NvU32 engineId;
+    NvU32 alignment;
+    NvU32 size;
+    NvU32 bInfoPopulated;
+} GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS;
+
+static int fd;
+
+static NvU32 rm_alloc(NvHandle parent, NvHandle client, NvU32 hClass, NvHandle *pOut)
+{
+    NVOS21_PARAMETERS p = {0};
+    p.hRoot = client;
+    p.hObjectParent = parent;
+    p.hClass = hClass;
+    if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc ioctl"); return 1; }
+    if (p.status) { fprintf(stderr, "alloc class 0x%x failed: 0x%x\n", hClass, p.status); return p.status; }
+    *pOut = p.hObjectNew;
+    return 0;
+}
+
+static NvU32 rm_alloc64(NvHandle parent, NvHandle client, NvU32 hClass,
+                        NvHandle newHandle, void *parms, NvU32 parmsSize, NvHandle *pOut)
+{
+    NVOS64_PARAMETERS p = {0};
+    p.hRoot = client;
+    p.hObjectParent = parent;
+    p.hObjectNew = newHandle; /* 0 = let RM generate (device); subdevices take an explicit handle */
+    p.hClass = hClass;
+    p.pAllocParms = (NvP64)parms;
+    p.paramsSize = parmsSize;
+    p.flags = 0;
+    if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc64 ioctl"); return 1; }
+    if (p.status) {
+        fprintf(stderr, "alloc64 class 0x%x failed: 0x%x\n", hClass, p.status);
+        return p.status;
+    }
+    *pOut = p.hObjectNew;
+    return 0;
+}
+
+static NvU32 rm_control(NvHandle hClient, NvHandle hObject, NvU32 cmd, void *params, NvU32 size)
+{
+    NVOS54_PARAMETERS p = {0};
+    p.hClient = hClient; p.hObject = hObject; p.cmd = cmd;
+    p.params = (NvP64)params; p.paramsSize = size;
+    if (ioctl(fd, NV_IOCTL_RM_CONTROL, &p) < 0) { perror("control ioctl"); return 1; }
+    return p.status;
+}
+
+int main(void)
+{
+    NvHandle hClient = 0, hDevice = 0, hSubdevice = 0;
+    NvU32 deviceParams = 0;    /* deviceId 0 */
+    NvU32 subdeviceParams = 0; /* subDeviceId 0 */
+    NvU32 st, i;
+
+    fd = open("/dev/nvidiactl", O_RDWR);
+    if (fd < 0) { perror("open /dev/nvidiactl"); return 1; }
+
+    if ((st = rm_alloc(0, 0, NV01_ROOT, &hClient))) return 1;
+    if ((st = rm_alloc64(hClient, hClient, NV01_DEVICE_0, 0,
+                         &deviceParams, sizeof(deviceParams), &hDevice))) return 1;
+    if ((st = rm_alloc64(hDevice, hClient, NV20_SUBDEVICE_0, 0xcaf00001,
+                         &subdeviceParams, sizeof(subdeviceParams), &hSubdevice))) return 1;
+    printf("hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", hClient, hDevice, hSubdevice);
+
+    printf("=== device-level FIFO_GET_ENGINE_CONTEXT_PROPERTIES (0x801707) ===\n");
+    for (i = 0; i < ENGINE_ID_COUNT; i++) {
+        FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS p;
+        memset(&p, 0, sizeof(p));
+        p.engineId = i;
+        st = rm_control(hClient, hDevice, NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
+                        &p, sizeof(p));
+        printf("engine[%2u] status=0x%x size=0x%08x alignment=0x%08x\n",
+               i, st, p.size, p.alignment);
+    }
+
+    printf("=== subdevice-level GR_GET_ENGINE_CONTEXT_PROPERTIES (0x2080122d) ===\n");
+    for (i = 0; i < ENGINE_ID_COUNT; i++) {
+        GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS p;
+        memset(&p, 0, sizeof(p));
+        p.engineId = i;
+        st = rm_control(hClient, hSubdevice, NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES,
+                        &p, sizeof(p));
+        printf("engine[%2u] status=0x%x size=0x%08x alignment=0x%08x\n",
+               i, st, p.size, p.alignment);
+    }
+
+    return 0;
+}