Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 2501 insertions, 0 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
new file mode 100644
index 0000000..b9f8417
--- /dev/null
+++ b/kernel/gsp_rm_emulation.c
@@ -0,0 +1,2501 @@
+/*
+ * GSP-RM Emulation (physical-path) for Host-Side vGPU
+ *
+ * Emulates the GSP-RM msgq RPC channel in guest system memory.
+ * See gsp_rm_emulation.h for the design overview.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/vfio.h>
+#include <linux/delay.h>
+
+#include "nvidia-vgpu-vfio.h"
+#include "gsp_emulation.h"
+#include "gsp_rm_emulation.h"
+#include "gsp_rm_tu104.h"
+
+/* ============================================================
+ * Guest-memory access ("virtual DMA" on behalf of the device)
+ * ============================================================ */
+
+static int gsp_rm_read(struct gsp_emu_context *ctx, u64 gpa, void *buf,
+		       size_t len)
+{
+	struct nv_vgpu_device *vgpu = ctx->vgpu;
+
+	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+		return -ENODEV;
+
+	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
+}
+
+static int gsp_rm_read_dev(struct nv_vgpu_device *vgpu, u64 gpa, void *buf,
+			   size_t len)
+{
+	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+		return -ENODEV;
+
+	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
+}
+
+static int gsp_rm_write(struct gsp_emu_context *ctx, u64 gpa, void *buf,
+			size_t len)
+{
+	struct nv_vgpu_device *vgpu = ctx->vgpu;
+
+	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+		return -ENODEV;
+
+	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
+}
+
+/*
+ * Debug helper: scan guest RAM for a 64-bit value (8-byte aligned) and/or a
+ * 32-bit value (4-byte aligned), reporting the GPAs where they are found.
+ * Used to locate guest structures (e.g. the UVM pushbuffer) without knowing
+ * the guest's GPU page tables. Triggered via the module parameter
+ * scan_guest_ram=<hex needle>.
+ */
+int gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle,
+			  u64 start, u64 end, u32 max_hits)
+{
+	u8 *buf;
+	u64 gpa;
+	u32 hits = 0;
+
+	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+		return -ENODEV;
+
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
+		u32 off;
+
+		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
+			continue;
+
+		for (off = 0; off + 8 <= PAGE_SIZE; off += 4) {
+			u64 v;
+			u32 v32;
+
+			memcpy(&v, buf + off, 8);
+			memcpy(&v32, buf + off, 4);
+
+			if (v == needle) {
+				pr_info("gsp-resman-vfio: scan: u64 0x%llx at gpa 0x%llx (page off 0x%x)\n",
+					needle, gpa + off, off);
+				if (++hits >= max_hits)
+					break;
+			}
+			if (v32 == (u32)needle && needle <= 0xffffffffULL) {
+				pr_info("gsp-resman-vfio: scan: u32 0x%x at gpa 0x%llx (page off 0x%x)\n",
+					(u32)needle, gpa + off, off);
+				if (++hits >= max_hits)
+					break;
+			}
+		}
+	}
+
+	kfree(buf);
+	pr_info("gsp-resman-vfio: scan: needle=0x%llx range=0x%llx..0x%llx hits=%u\n",
+		needle, start, end, hits);
+	return hits;
+}
+
+/*
+ * Debug helper: scan guest RAM for values matching (v & mask) == value.
+ * Used to find a stored page-base VA when only "base + index*size" is known.
+ */
+int gsp_rm_scan_guest_ram_masked(struct nv_vgpu_device *vgpu, u64 value,
+				 u64 mask, u64 start, u64 end, u32 max_hits)
+{
+	u8 *buf;
+	u64 gpa;
+	u32 hits = 0;
+
+	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+		return -ENODEV;
+
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
+		u32 off;
+
+		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
+			continue;
+
+		for (off = 0; off + 8 <= PAGE_SIZE; off += 4) {
+			u64 v64;
+			u32 v32;
+
+			memcpy(&v64, buf + off, 8);
+			memcpy(&v32, buf + off, 4);
+			if ((v64 & mask) == value) {
+				pr_info("nvidia-vgpu-vfio: masked scan: u64 0x%llx at gpa 0x%llx (off 0x%x)\n",
+					v64, gpa + off, off);
+				if (++hits >= max_hits)
+					break;
+			}
+			if ((v32 & (u32)mask) == (u32)value) {
+				pr_info("nvidia-vgpu-vfio: masked scan: u32 0x%x at gpa 0x%llx (off 0x%x)\n",
+					v32, gpa + off, off);
+				if (++hits >= max_hits)
+					break;
+			}
+		}
+	}
+
+	kfree(buf);
+	pr_info("nvidia-vgpu-vfio: masked scan: value=0x%llx mask=0x%llx hits=%u\n",
+		value, mask, hits);
+	return hits;
+}
+
+/*
+ * Debug helper: scan guest RAM for page-table entries pointing at a given
+ * physical address (matches v & 0xfffff000 == pa for u32 and the 64-bit
+ * equivalent). Used to locate the guest's GPU page tables.
+ */
+int gsp_rm_scan_guest_ram_pt(struct nv_vgpu_device *vgpu, u64 pa,
+			     u64 start, u64 end, u32 max_hits)
+{
+	u8 *buf;
+	u64 gpa;
+	u32 hits = 0;
+	u64 pa_page = pa & ~0xfffULL;
+
+	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+		return -ENODEV;
+
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
+		u32 off;
+
+		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
+			continue;
+
+		for (off = 0; off + 8 <= PAGE_SIZE; off += 4) {
+			u64 v64;
+			u32 v32;
+
+			memcpy(&v64, buf + off, 8);
+			memcpy(&v32, buf + off, 4);
+			if ((v64 & 0xfffffffffffff000ULL) == pa_page) {
+				pr_info("gsp-resman-vfio: pt scan: u64 pte 0x%llx at gpa 0x%llx (off 0x%x)\n",
+					v64, gpa + off, off);
+				if (++hits >= max_hits)
+					break;
+			}
+			if ((v32 & 0xfffff000U) == (u32)pa_page) {
+				pr_info("gsp-resman-vfio: pt scan: u32 pte 0x%x at gpa 0x%llx (off 0x%x)\n",
+					v32, gpa + off, off);
+				if (++hits >= max_hits)
+					break;
+			}
+		}
+	}
+
+	kfree(buf);
+	pr_info("gsp-resman-vfio: pt scan: pa=0x%llx hits=%u\n", pa, hits);
+	return hits;
+}
+
+/*
+ * Debug helper: scan guest RAM for a sequence of up to 4 u32 values.
+ * Used to locate a CE semaphore-release method sequence in a pushbuffer.
+ */
+int gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat,
+			      u32 npat, u64 start, u64 end, u32 max_hits)
+{
+	u8 *buf;
+	u64 gpa;
+	u32 hits = 0;
+
+	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+		return -ENODEV;
+	if (!npat || npat > 4)
+		return -EINVAL;
+
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
+		u32 off;
+
+		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
+			continue;
+
+		for (off = 0; off + npat * 4 <= PAGE_SIZE; off += 4) {
+			u32 i, ok = 1;
+
+			for (i = 0; i < npat; i++) {
+				u32 v;
+
+				memcpy(&v, buf + off + i * 4, 4);
+				if (v != pat[i]) {
+					ok = 0;
+					break;
+				}
+			}
+			if (ok) {
+				pr_info("gsp-resman-vfio: seq scan: hit in guest RAM at gpa 0x%llx (page off 0x%x)\n",
+					gpa + off, off);
+				if (++hits >= max_hits)
+					break;
+			}
+		}
+	}
+
+	kfree(buf);
+	pr_info("gsp-resman-vfio: seq scan: range=0x%llx..0x%llx hits=%u\n",
+		start, end, hits);
+	return hits;
+}static u64 gsp_rm_off_to_gpa(struct gsp_rm_emu *rm, u64 off)
+{
+	u64 idx = off >> 12;
+
+	if (idx >= rm->mq.pageTableEntryCount)
+		return 0;
+	return rm->ptes[idx] + (off & 0xfff);
+}
+
+/*
+ * Page-aware shared-buffer I/O. The queue buffer is NV_MEMORY_NONCONTIGUOUS:
+ * its 4K pages are independent guest allocations, so a range crossing a page
+ * boundary is NOT contiguous in guest physical memory. Translating the start
+ * and blasting a multi-page range would scribble into arbitrary guest RAM
+ * (observed as guest userspace segfaults). Split at page boundaries and
+ * translate each page separately.
+ */
+static int gsp_rm_write_buf(struct gsp_emu_context *ctx, u64 off,
+			    const void *buf, size_t len)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	const u8 *p = buf;
+	int ret;
+
+	while (len) {
+		u64 gpa = gsp_rm_off_to_gpa(rm, off);
+		size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff));
+
+		if (!gpa)
+			return -EINVAL;
+		ret = gsp_rm_write(ctx, gpa, (void *)p, chunk);
+		if (ret)
+			return ret;
+		p += chunk;
+		off += chunk;
+		len -= chunk;
+	}
+	return 0;
+}
+
+static int gsp_rm_read_buf(struct gsp_emu_context *ctx, u64 off,
+			   void *buf, size_t len)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	u8 *p = buf;
+	int ret;
+
+	while (len) {
+		u64 gpa = gsp_rm_off_to_gpa(rm, off);
+		size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff));
+
+		if (!gpa)
+			return -EINVAL;
+		ret = gsp_rm_read(ctx, gpa, p, chunk);
+		if (ret)
+			return ret;
+		p += chunk;
+		off += chunk;
+		len -= chunk;
+	}
+	return 0;
+}
+
+/* ============================================================
+ * Status queue element construction
+ * ============================================================ */
+
+/* XOR-fold checksum over u64s (message_queue_priv.h _checkSum32) */
+static u32 gsp_rm_checksum32(const void *data, u32 len)
+{
+	const u64 *p = data;
+	const u64 *end = (const u64 *)((const u8 *)data + len);
+	u64 sum = 0;
+
+	while (p < end)
+		sum ^= *p++;
+
+	return (u32)(sum >> 32) ^ (u32)sum;
+}
+
+/*
+ * 570.144 GSP_MSG_QUEUE_ELEMENT wire layout (guest driver is 570.144 stock):
+ *   0x00  authTagBuffer[16]  (zero - no CC)
+ *   0x10  aadBuffer[16]      (zero - no CC)
+ *   0x20  checkSum           (XOR-fold over msgLen, makes total 0)
+ *   0x24  seqNum             (per-record sequence)
+ *   0x28  elemCount          (elements this record spans)
+ *   0x30  rpc_message_header_v (32 bytes) + payload
+ * GSP_MSG_QUEUE_ELEMENT_HDR_SIZE = offsetof(rpc) = 0x30.
+ */
+#define GSPRM_ELEM_HDR_SIZE	0x30
+
+/*
+ * Max RPC data that fits in one queue element:
+ * 4096 (element) - 0x30 (element hdr) - 32 (rpc hdr) = 4016 bytes.
+ */
+#define GSPRM_MAX_SINGLE_ELEM_DATA	(GSPRM_QUEUE_ELEMENT_SIZE - \
+					 GSPRM_ELEM_HDR_SIZE - \
+					 sizeof(struct gsprm_rpc_header))
+
+/*
+ * Build a multi-element record into a contiguous buffer.
+ *
+ * The record spans elem_count consecutive queue elements:
+ *   [0]: element header (0x30) + rpc header (32) + data[0..4015]
+ *   [1..N-1]: continuation data only (no headers)
+ *
+ * Returns the total record size in bytes (elem_count * 4096).
+ * Sets *out_elem_count to the number of elements used.
+ */
+static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size,
+				u32 function, u32 sequence, u32 result,
+				const void *data, u32 data_len,
+				u32 *out_elem_count)
+{
+	struct gsprm_rpc_header *hdr =
+		(struct gsprm_rpc_header *)(buf + GSPRM_ELEM_HDR_SIZE);
+	u32 rpc_len = sizeof(*hdr) + data_len;
+	u32 msg_len = GSPRM_ELEM_HDR_SIZE + rpc_len;
+	u32 aligned_len;
+	u32 elem_count;
+	u32 first_data;
+	u32 remaining;
+	u32 i;
+
+	/* How many elements does this record span? */
+	elem_count = (msg_len + GSPRM_QUEUE_ELEMENT_SIZE - 1) /
+		     GSPRM_QUEUE_ELEMENT_SIZE;
+	if (elem_count == 0)
+		elem_count = 1;
+
+	if (elem_count * GSPRM_QUEUE_ELEMENT_SIZE > buf_size) {
+		pr_err("GSPRM: build_record overflow: need %u have %u\n",
+		       elem_count * GSPRM_QUEUE_ELEMENT_SIZE, buf_size);
+		return 0;
+	}
+
+	memset(buf, 0, elem_count * GSPRM_QUEUE_ELEMENT_SIZE);
+
+	/* First element: header + rpc header + start of data */
+	memcpy(buf + 0x24, &rm->stat_seq, 4);		/* seqNum */
+	memcpy(buf + 0x28, &elem_count, 4);		/* elemCount */
+
+	hdr->header_version = GSPRM_RPC_HEADER_VERSION;
+	hdr->signature = GSPRM_RPC_SIGNATURE;
+	hdr->length = rpc_len;
+	hdr->function = function;
+	hdr->rpc_result = result;
+	hdr->rpc_result_private = result;
+	hdr->sequence = sequence;
+	hdr->spare = 0;
+
+	/* Copy data across element boundaries */
+	first_data = min(data_len, (u32)GSPRM_MAX_SINGLE_ELEM_DATA);
+	if (data && first_data)
+		memcpy((u8 *)(hdr + 1), data, first_data);
+
+	remaining = data_len - first_data;
+	if (remaining > 0 && data) {
+		const u8 *src = (const u8 *)data + first_data;
+		u8 *dst = buf + GSPRM_QUEUE_ELEMENT_SIZE;
+
+		for (i = 1; i < elem_count && remaining > 0; i++) {
+			u32 chunk = min(remaining,
+					(u32)GSPRM_QUEUE_ELEMENT_SIZE);
+
+			memcpy(dst, src, chunk);
+			src += chunk;
+			dst += GSPRM_QUEUE_ELEMENT_SIZE;
+			remaining -= chunk;
+		}
+	}
+
+	/* checksum covers msg_len zero-padded to 8 (buffer already zeroed) */
+	aligned_len = (msg_len + 7) & ~7U;
+	*(u32 *)(buf + 0x20) = gsp_rm_checksum32(buf, aligned_len);
+
+	*out_elem_count = elem_count;
+	return elem_count * GSPRM_QUEUE_ELEMENT_SIZE;
+}
+
+/*
+ * Write a status queue record (1+ elements) and advance the writePtr.
+ */
+static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
+		       u32 sequence, u32 result, const void *data, u32 data_len)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	u8 *buf;
+	u32 slot;
+	u32 new_wp;
+	u32 elem_count;
+	u32 record_size;
+	u32 buf_size;
+	int ret;
+
+	/* Allocate buffer for worst-case multi-element record */
+	buf_size = GSPRM_QUEUE_ELEMENT_SIZE *
+		   ((data_len + GSPRM_MAX_SINGLE_ELEM_DATA +
+		     GSPRM_QUEUE_ELEMENT_SIZE - 1) /
+		    GSPRM_QUEUE_ELEMENT_SIZE + 1);
+	if (buf_size < GSPRM_QUEUE_ELEMENT_SIZE)
+		buf_size = GSPRM_QUEUE_ELEMENT_SIZE;
+
+	buf = kzalloc(buf_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	record_size = gsp_rm_build_record(rm, buf, buf_size, function,
+					   sequence, result, data, data_len,
+					   &elem_count);
+	if (!record_size) {
+		kfree(buf);
+		return -EOVERFLOW;
+	}
+
+	/* Check we don't overflow the status queue */
+	if (elem_count > rm->stat_msgcount) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: record too large: %u elements > queue %u",
+			    elem_count, rm->stat_msgcount);
+		kfree(buf);
+		return -EOVERFLOW;
+	}
+
+	/* Write all elements to consecutive queue slots. Page-aware: the
+	 * queue buffer is non-contiguous, so translate per page. */
+	slot = rm->stat_write % rm->stat_msgcount;
+
+	/* Handle wrap-around: write may span the end of the circular queue */
+	{
+		u32 elems_to_end = rm->stat_msgcount - slot;
+		u32 first_chunk = min(elem_count, elems_to_end);
+		u32 second_chunk = elem_count - first_chunk;
+		u64 rec_off = rm->stat_off + rm->entry_off +
+			      (u64)slot * GSPRM_QUEUE_ELEMENT_SIZE;
+
+		ret = gsp_rm_write_buf(ctx, rec_off, buf,
+				       first_chunk * GSPRM_QUEUE_ELEMENT_SIZE);
+		if (ret) {
+			nv_vgpu_err(ctx->vgpu, "GSPRM: status write failed off=0x%llx ret=%d",
+				    rec_off, ret);
+			kfree(buf);
+			return ret;
+		}
+
+		if (second_chunk > 0) {
+			ret = gsp_rm_write_buf(ctx, rm->stat_off + rm->entry_off,
+					       buf + first_chunk * GSPRM_QUEUE_ELEMENT_SIZE,
+					       second_chunk * GSPRM_QUEUE_ELEMENT_SIZE);
+			if (ret) {
+				nv_vgpu_err(ctx->vgpu, "GSPRM: status wrap write failed ret=%d",
+					    ret);
+				kfree(buf);
+				return ret;
+			}
+		}
+	}
+
+	kfree(buf);
+
+	/* Advance the write pointer with the same wrap semantics as the
+	 * driver's msgq (slot index in [0, msgCount) - the peer's free-space
+	 * and slot math breaks on unbounded values). stat_seq stays
+	 * monotonic per-record. */
+	rm->stat_write = (rm->stat_write + elem_count) % rm->stat_msgcount;
+	rm->stat_seq++;
+
+	/* bump status TX header writePtr (offset 0x10) */
+	new_wp = rm->stat_write;
+	ret = gsp_rm_write_buf(ctx, rm->stat_off + 0x10,
+			       &new_wp, sizeof(new_wp));
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: status writePtr update failed ret=%d", ret);
+		return ret;
+	}
+
+	/* Notify the guest (MSI-X vector 0). The driver's bottom-half drains
+	 * events under the GPU lock; the rpcRecvPoll poller holds the same
+	 * lock from send to poll, so a reply can't be consumed by the event
+	 * drain while a poller waits for it (same as real GSP-RM). */
+	gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq);
+
+	return 0;
+}
+
+/* ============================================================
+ * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED)
+ * ============================================================ */
+
+static int gsp_rm_parse_queue_chain(struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	struct gsprm_libos_region *regions;
+	int ret, i;
+	u64 gsp_args_gpa = 0;
+	u32 total_size;
+
+	regions = kmalloc_array(GSPRM_MAX_LIBOS_REGIONS, sizeof(*regions),
+				GFP_KERNEL);
+	if (!regions)
+		return -ENOMEM;
+
+	/* 1. Read the libos init args array */
+	ret = gsp_rm_read(ctx, rm->libos_args_gpa, regions,
+			  GSPRM_MAX_LIBOS_REGIONS * sizeof(*regions));
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: libos args read failed gpa=0x%llx ret=%d",
+			    rm->libos_args_gpa, ret);
+		goto out;
+	}
+
+	/* 2. Find the RMARGS region -> GSP_ARGUMENTS_CACHED */
+	for (i = 0; i < GSPRM_MAX_LIBOS_REGIONS; i++) {
+		if (regions[i].id8 == 0)
+			break;
+		nv_vgpu_info(ctx->vgpu, "GSPRM: libos region[%d] id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u",
+			     i, regions[i].id8, regions[i].pa, regions[i].size,
+			     regions[i].kind, regions[i].loc);
+		if (regions[i].id8 == GSPRM_LIBOS_ID_RMARGS)
+			gsp_args_gpa = regions[i].pa;
+	}
+	if (!gsp_args_gpa) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: RMARGS region not found");
+		ret = -ENOENT;
+		goto out;
+	}
+	rm->gsp_args_gpa = gsp_args_gpa;
+
+	/* 3. Read GSP_ARGUMENTS_CACHED (MESSAGE_QUEUE_INIT_ARGUMENTS first) */
+	ret = gsp_rm_read(ctx, gsp_args_gpa, &rm->mq, sizeof(rm->mq));
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_ARGUMENTS_CACHED read failed ret=%d", ret);
+		goto out;
+	}
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: queue args sharedPA=0x%llx ptes=%u cmdOff=0x%llx statOff=0x%llx hdrSize=0x%llx elemMin=0x%llx elemMax=0x%llx hdrAlign=%u elemAlign=%u",
+		     rm->mq.sharedMemPhysAddr, rm->mq.pageTableEntryCount,
+		     rm->mq.cmdQueueOffset, rm->mq.statQueueOffset,
+		     rm->mq.queueElementHdrSize, rm->mq.queueElementSizeMin,
+		     rm->mq.queueElementSizeMax, rm->mq.queueHeaderAlign,
+		     rm->mq.queueElementAlign);
+	print_hex_dump(KERN_INFO, "GSPRM args raw: ", DUMP_PREFIX_OFFSET,
+		       16, 8, &rm->mq, sizeof(rm->mq), false);
+
+	if (!rm->mq.sharedMemPhysAddr || !rm->mq.pageTableEntryCount ||
+	    rm->mq.pageTableEntryCount > 1024) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: invalid queue args");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* 4. Read the page table (RmPhysAddr per 4K page of the shared buffer) */
+	rm->ptes = kmalloc_array(rm->mq.pageTableEntryCount, sizeof(u64),
+				 GFP_KERNEL);
+	if (!rm->ptes) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	ret = gsp_rm_read(ctx, rm->mq.sharedMemPhysAddr, rm->ptes,
+			  rm->mq.pageTableEntryCount * sizeof(u64));
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: page table read failed ret=%d", ret);
+		goto out;
+	}
+
+	/* 5. Compute queue geometry. NOTE: the shared buffer is
+	 * NV_MEMORY_NONCONTIGUOUS - only queue-offsets are linear, GPAs are
+	 * not. Every access must translate through rm->ptes (gsp_rm_off_to_gpa). */
+	rm->cmd_off = rm->mq.cmdQueueOffset;
+	rm->stat_off = rm->mq.statQueueOffset;
+	rm->cmd_gpa = gsp_rm_off_to_gpa(rm, rm->cmd_off);
+	rm->stat_gpa = gsp_rm_off_to_gpa(rm, rm->stat_off);
+	rm->cmd_size = (u32)(rm->mq.statQueueOffset - rm->mq.cmdQueueOffset);
+	total_size = rm->mq.pageTableEntryCount << 12;
+	rm->stat_size = total_size - (u32)rm->mq.statQueueOffset;
+	rm->entry_off = 0x1000; /* ALIGN_UP(0x20 + 4, 1 << elemAlign(12)) */
+	rm->cmd_msgcount = (rm->cmd_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE;
+	rm->stat_msgcount = (rm->stat_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE;
+
+	if (!rm->cmd_gpa || !rm->stat_gpa || !rm->cmd_msgcount || !rm->stat_msgcount) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: bad queue geometry cmd_gpa=0x%llx stat_gpa=0x%llx cmd_mc=%u stat_mc=%u",
+			    rm->cmd_gpa, rm->stat_gpa, rm->cmd_msgcount,
+			    rm->stat_msgcount);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: cmd queue gpa=0x%llx size=0x%x mc=%u; status queue gpa=0x%llx size=0x%x mc=%u",
+		     rm->cmd_gpa, rm->cmd_size, rm->cmd_msgcount,
+		     rm->stat_gpa, rm->stat_size, rm->stat_msgcount);
+
+	ret = 0;
+out:
+	kfree(regions);
+	return ret;
+}
+
+/*
+ * Write the status queue TX header (what GSP-RM's msgqTxCreate produces).
+ */
+static int gsp_rm_link_status_queue(struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	struct gsprm_msgq_tx_header tx;
+	u32 zero = 0;
+	int ret;
+
+	memset(&tx, 0, sizeof(tx));
+	tx.version = GSPRM_MSGQ_VERSION;
+	tx.size = rm->stat_size;
+	tx.msgSize = GSPRM_QUEUE_ELEMENT_SIZE;
+	tx.msgCount = rm->stat_msgcount;
+	tx.writePtr = 0;
+	tx.flags = GSPRM_MSGQ_FLAGS_SWAP_RX;
+	tx.rxHdrOff = 0x20; /* ALIGN_UP(sizeof(msgqTxHeader)=32, 1<<hdrAlign(4)=16) */
+	tx.entryOff = rm->entry_off;
+
+	ret = gsp_rm_write_buf(ctx, rm->stat_off, &tx, sizeof(tx));
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: status TX header write failed ret=%d", ret);
+		return ret;
+	}
+
+	/* status queue RX header (our command-queue read position, swap mode) */
+	ret = gsp_rm_write_buf(ctx, rm->stat_off + tx.rxHdrOff,
+			       &zero, sizeof(zero));
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: status RX header write failed ret=%d", ret);
+		return ret;
+	}
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: status queue linked (size=0x%x msgCount=%u)",
+		     tx.size, tx.msgCount);
+	return 0;
+}
+
+/* ============================================================
+ * Command processing
+ * ============================================================ */
+
+static const char *gsp_rm_rpc_name(u32 function)
+{
+	switch (function) {
+	case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO:	return "SET_GUEST_SYSTEM_INFO";
+	case GSPRM_RPC_FUNC_ALLOC_ROOT:			return "ALLOC_ROOT";
+	case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO:	return "GET_GSP_STATIC_INFO";
+	case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO:	return "GSP_SET_SYSTEM_INFO";
+	case GSPRM_RPC_FUNC_SET_REGISTRY:		return "SET_REGISTRY";
+	case GSPRM_RPC_FUNC_CONTINUATION_RECORD:	return "CONTINUATION_RECORD";
+	case GSPRM_RPC_FUNC_GSP_RM_CONTROL:		return "GSP_RM_CONTROL";
+	default:					return "UNKNOWN";
+	}
+}
+
+/*
+ * SET_GUEST_SYSTEM_INFO: the driver negotiates the RPC interface version
+ * from the reply's vgxVersionMajorNum/MinorNum fields. Without them the
+ * version-gated HALs stay stubs and every subsequent RPC fails with
+ * NV_VGPU_MSG_RESULT_RPC_UNKNOWN_FUNCTION (0xff100001). Echo the driver's
+ * own version back (no host-guest version skew).
+ */
+static void gsp_rm_handle_set_guest_system_info(struct gsp_emu_context *ctx,
+						const u8 *elem,
+						u32 sequence)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE +
+			     sizeof(struct gsprm_rpc_header);
+	u8 *reply;
+	u32 major, minor;
+
+	memcpy(&major, req_data + 0, 4);
+	memcpy(&minor, req_data + 4, 4);
+	if (!major)
+		major = GSPRM_VGX_VERSION_MAJOR;
+	if (!minor && major != GSPRM_VGX_VERSION_MAJOR)
+		minor = GSPRM_VGX_VERSION_MINOR;
+
+	reply = kzalloc(GSPRM_SET_GUEST_INFO_SIZE, GFP_KERNEL);
+	if (!reply) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply alloc failed");
+		return;
+	}
+	memcpy(reply + 0, &major, 4);
+	memcpy(reply + 4, &minor, 4);
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO seq=%u -> VGX %u.%u",
+		     sequence, major, minor);
+
+	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO, sequence, 0,
+			reply, GSPRM_SET_GUEST_INFO_SIZE))
+		nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply send failed");
+	else
+		rm->rpcs_replied++;
+
+	kfree(reply);
+}
+
+/* ============================================================
+ * GET_GSP_STATIC_INFO (function 65)
+ *
+ * Builds the 1656-byte GspStaticConfigInfo for TU104.
+ * ============================================================ */
+
+static void gsp_rm_build_static_info(u8 *buf)
+{
+	u64 val64;
+	u32 val32;
+	u8 *p;
+
+	memset(buf, 0, TU104_GSCI_SIZE);
+
+	/* grCapsBits (offset 0, 23 bytes) - basic GR caps */
+	buf[0] = 0x01;	/* GR present */
+
+	/* gidInfo (offset 24, 268 bytes): index(4) + flags(4) + length(4) + data(256) */
+	p = buf + TU104_GSCI_OFF_gidInfo;
+	val32 = 0;		memcpy(p + 0, &val32, 4);	/* index */
+	val32 = 0x80000000;	memcpy(p + 4, &val32, 4);	/* flags: valid */
+	val32 = 16;		memcpy(p + 8, &val32, 4);	/* length = 16 (UUID) */
+	/* GPU UUID - deterministic fake for TU104 */
+	memcpy(p + 12, "GPU-9da5e221-18", 15);
+
+	/* SKUInfo (offset 292, 48 bytes) */
+	p = buf + TU104_GSCI_OFF_SKUInfo;
+	memcpy(p + 4, "TU104", 5);	/* chipSKU */
+	memcpy(p + 13, "400", 3);	/* chipSKUMod */
+	memcpy(p + 20, "N18E", 4);	/* project */
+	memcpy(p + 24, "G3", 2);	/* projectSKU */
+
+	/* fbRegionInfoParams (offset 344, 776 bytes) */
+	p = buf + TU104_GSCI_OFF_fbRegionInfoParams;
+	val32 = 1;		memcpy(p, &val32, 4);		/* numFBRegions = 1 */
+	/* Region 0: base=0, limit=256MB-1, reserved=0 */
+	{
+		struct tu104_fb_region *region = (void *)(p + 8); /* 8-aligned */
+		val64 = 0;		memcpy(&region->base, &val64, 8);
+		val64 = 0x0FFFFFFF;	memcpy(&region->limit, &val64, 8);
+		val64 = 0;		memcpy(&region->reserved, &val64, 8);
+		val32 = 0;		memcpy(&region->performance, &val32, 4);
+		region->supportCompressed = 0;
+		/* memmgrCalcReservedFbSpace asserts that SOME region supports
+		 * ISO "even if we have no display" - without it the reserved
+		 * space split fails and kdisp can't allocate its channel
+		 * table. A single ISO-capable region satisfies fast/slow/iso
+		 * assignment. */
+		region->supportISO = 1;
+		region->bProtected = 0;
+		memset(region->blackList, 0, sizeof(region->blackList));
+	}
+
+	/* sriovCaps (offset 1120, 80 bytes) - all zeros (not SR-IOV) */
+
+	/* sriovMaxGfid (offset 1200, 4 bytes) */
+	val32 = 0;
+	memcpy(buf + TU104_GSCI_OFF_sriovMaxGfid, &val32, 4);
+
+	/* engineCaps (offset 1204, 12 bytes) */
+	val32 = 0x60003E02;	/* GR0|CE0-4|NVDEC0|NVDEC1 */
+	memcpy(buf + TU104_GSCI_OFF_engineCaps, &val32, 4);
+	val32 = 0x00362020;	/* NVENC0|SW|SEC2|HOST|PMU|FBFLCN */
+	memcpy(buf + TU104_GSCI_OFF_engineCaps + 4, &val32, 4);
+	val32 = 0;
+	memcpy(buf + TU104_GSCI_OFF_engineCaps + 8, &val32, 4);
+
+	/* poisonFuseEnabled (offset 1216, 1 byte) */
+	buf[TU104_GSCI_OFF_poisonFuseEnabled] = 0;
+
+	/* fb_length (offset 1224, 8 bytes) - 256MB (512MB broke the FWSEC
+	 * WPR2-location check below). */
+	val64 = 0x10000000ULL;
+	memcpy(buf + TU104_GSCI_OFF_fb_length, &val64, 8);
+
+	/* fbio_mask (offset 1232, 8 bytes) - 8 FBIOs for 256-bit bus */
+	val64 = 0xFF;
+	memcpy(buf + TU104_GSCI_OFF_fbio_mask, &val64, 8);
+
+	/* fb_bus_width (offset 1240, 4 bytes) - 256-bit */
+	val32 = 256;
+	memcpy(buf + TU104_GSCI_OFF_fb_bus_width, &val32, 4);
+
+	/* fb_ram_type (offset 1244, 4 bytes) - GDDR6 = 6 */
+	val32 = 6;
+	memcpy(buf + TU104_GSCI_OFF_fb_ram_type, &val32, 4);
+
+	/* fbp_mask (offset 1248, 8 bytes) - 6 FBPs for 6 GPCs */
+	val64 = 0x3F;
+	memcpy(buf + TU104_GSCI_OFF_fbp_mask, &val64, 8);
+
+	/* l2_cache_size (offset 1256, 4 bytes) - 4MB */
+	val32 = 0x400000;
+	memcpy(buf + TU104_GSCI_OFF_l2_cache_size, &val32, 4);
+
+	/* gpuNameString (offset 1260, 64 bytes) */
+	strncpy((char *)(buf + TU104_GSCI_OFF_gpuNameString),
+		"NVIDIA GeForce RTX 2080 SUPER", 64);
+
+	/* gpuShortNameString (offset 1324, 64 bytes) */
+	strncpy((char *)(buf + TU104_GSCI_OFF_gpuShortNameString),
+		"TU104", 64);
+
+	/* gpuNameString_Unicode (offset 1388, 128 bytes) - skip for now */
+
+	/* Branding booleans */
+	buf[TU104_GSCI_OFF_bIsMobile] = 1;	/* Mobile/Max-Q */
+
+	/* bar1PdeBase (offset 1536, 8 bytes) */
+	val64 = 0;
+	memcpy(buf + TU104_GSCI_OFF_bar1PdeBase, &val64, 8);
+
+	/* bar2PdeBase (offset 1544, 8 bytes) */
+	val64 = 0;
+	memcpy(buf + TU104_GSCI_OFF_bar2PdeBase, &val64, 8);
+
+	/* bVbiosValid (offset 1552, 1 byte) */
+	buf[TU104_GSCI_OFF_bVbiosValid] = 1;
+
+	/* vbiosSubVendor (offset 1556, 4 bytes) - ASUS */
+	val32 = 0x1043;
+	memcpy(buf + TU104_GSCI_OFF_vbiosSubVendor, &val32, 4);
+
+	/* vbiosSubDevice (offset 1560, 4 bytes) */
+	val32 = 0x1F01;
+	memcpy(buf + TU104_GSCI_OFF_vbiosSubDevice, &val32, 4);
+
+	/* hInternalClient/Device/Subdevice */
+	val32 = TU104_H_INTERNAL_CLIENT;
+	memcpy(buf + TU104_GSCI_OFF_hInternalClient, &val32, 4);
+	val32 = TU104_H_INTERNAL_DEVICE;
+	memcpy(buf + TU104_GSCI_OFF_hInternalDevice, &val32, 4);
+	val32 = TU104_H_INTERNAL_SUBDEVICE;
+	memcpy(buf + TU104_GSCI_OFF_hInternalSubdevice, &val32, 4);
+
+	/* ecidInfo (offset 1616, 24 bytes) - leave zeroed */
+
+	/* fwWprLayoutOffset (offset 1640, 16 bytes): nonWprHeapOffset, frtsOffset */
+	val64 = 0;
+	memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset, &val64, 8);
+	val64 = 0xFE00000;	/* frtsOffset = fbSize - 2MB (256MB) */
+	memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8);
+}
+
+static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx,
+					   u32 sequence)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	u8 *reply;
+
+	reply = kzalloc(TU104_GSCI_SIZE, GFP_KERNEL);
+	if (!reply) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply alloc failed");
+		return;
+	}
+
+	gsp_rm_build_static_info(reply);
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)",
+		     sequence, TU104_GSCI_SIZE,
+		     0x10000000ULL >> 20, 0x60003E02, 0x362020);
+
+	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO, sequence, 0,
+			reply, TU104_GSCI_SIZE))
+		nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply send failed");
+	else
+		rm->rpcs_replied++;
+
+	kfree(reply);
+}
+
+/* ============================================================
+ * GSP_RM_CONTROL (function 76) - sub-command dispatch
+ *
+ * rpc_gsp_rm_control_v03_00: hClient(4) + hObject(4) + cmd(4) + status(4)
+ *                            + paramsSize(4) + flags(4) + params[]
+ * ============================================================ */
+
+struct gsp_rm_control_req {
+	u32 hClient;
+	u32 hObject;
+	u32 cmd;
+	u32 status;
+	u32 paramsSize;
+	u32 flags;
+	/* params[] follows */
+};
+
+#define GSP_RM_CTRL_HDR_SIZE	24
+static void gsp_rm_handle_fifo_device_table(struct gsp_emu_context *ctx,
+					     u32 sequence,
+					     struct gsp_rm_control_req *req,
+					     const u8 *req_params,
+					     u32 *out_status,
+					     u8 *out_params, u32 *out_params_size)
+{
+	struct tu104_fifo_get_device_info_params *fifo_params;
+	u32 base, count, i, copy_count;
+
+	fifo_params = (struct tu104_fifo_get_device_info_params *)out_params;
+	base = 0;
+	/* baseIndex is the first field of the request params, which live in
+	 * the queue element buffer (NOT after the local header struct!). Only
+	 * readable when the whole request fits in the first element. */
+	if (req->paramsSize >= 4 &&
+	    req->paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA)
+		memcpy(&base, req_params, 4);
+	if (base >= TU104_FIFO_TABLE_MAX_DEVICES)
+		base = 0;
+
+	/* How many entries to return in this page */
+	count = TU104_ENGINE_TABLE_SIZE - base;
+	if (count > TU104_FIFO_TABLE_MAX_ENTRIES)
+		count = TU104_FIFO_TABLE_MAX_ENTRIES;
+
+	copy_count = count;
+	if (base >= TU104_ENGINE_TABLE_SIZE)
+		copy_count = 0;
+
+	fifo_params->baseIndex = base;
+	fifo_params->numEntries = copy_count;
+	fifo_params->bMore = (base + copy_count) < TU104_ENGINE_TABLE_SIZE;
+
+	for (i = 0; i < copy_count; i++)
+		memcpy(&fifo_params->entries[i], &tu104_engine_table[base + i],
+		       sizeof(struct tu104_fifo_device_entry));
+
+	*out_params_size = offsetof(struct tu104_fifo_get_device_info_params,
+				     entries) +
+			   copy_count * sizeof(struct tu104_fifo_device_entry);
+	*out_status = 0; /* NV_OK */
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: FIFO_GET_DEVICE_INFO_TABLE base=%u count=%u more=%u",
+		     base, copy_count, fifo_params->bMore);
+}
+
+static void gsp_rm_handle_intr_kernel_table(struct gsp_emu_context *ctx,
+					     u32 sequence,
+					     u32 *out_status,
+					     u8 *out_params, u32 *out_params_size)
+{
+	struct tu104_intr_get_kernel_table_params *intr_params;
+	u32 copy_size;
+
+	intr_params = (struct tu104_intr_get_kernel_table_params *)out_params;
+
+	intr_params->tableLen = TU104_INTR_KERNEL_TABLE_SIZE;
+	copy_size = TU104_INTR_KERNEL_TABLE_SIZE *
+		    sizeof(struct tu104_intr_table_entry);
+	memcpy(intr_params->table, tu104_intr_kernel_table, copy_size);
+
+	/* Subtree map - must match intrInitSubtreeMap_TU102 (the values
+	 * physical-path guests receive from real GSP-RM):
+	 *   ESCHED_DRIVEN_ENGINE            = {3,3} (STALL_SUBTREE)
+	 *   ESCHED_DRIVEN_ENGINE_NOTIF      = {0,0}
+	 *   UVM_OWNED                       = {1,1} (fault 64, access cntr 72)
+	 *   UVM_SHARED                      = {2,2} (doorbells/GSP/DISP)
+	 *   DEFAULT/RUNLIST/RUNLIST_NOTIF   = INVALID (unset on Turing)
+	 * intrCacheIntrFields requires accessCntrSubtree(=1) within
+	 * UVM_OWNED and UVM_SHARED to be a single subtree. */
+	{
+		u32 c;
+
+		for (c = 0; c < TU104_INTR_CATEGORY_ENUM_COUNT; c++) {
+			intr_params->subtreeMap[c].subtreeStart = TU104_INTR_INVALID_SUBTREE;
+			intr_params->subtreeMap[c].subtreeEnd   = TU104_INTR_INVALID_SUBTREE;
+		}
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeStart = 3;
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeEnd   = 3;
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeStart = 0;
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeEnd   = 0;
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeStart = 1;
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeEnd   = 1;
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeStart = 2;
+		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeEnd   = 2;
+	}
+
+	/* The driver reads subtreeMap at offset 2052 (after the full
+	 * 128-entry array), so the reply must span the whole struct -
+	 * not just the used entries. */
+	*out_params_size = offsetof(struct tu104_intr_get_kernel_table_params,
+				     subtreeMap) +
+			   sizeof(intr_params->subtreeMap);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: INTR_GET_KERNEL_TABLE tableLen=%u",
+		     TU104_INTR_KERNEL_TABLE_SIZE);
+}
+
+static void gsp_rm_handle_constructed_falcon_info(struct gsp_emu_context *ctx,
+						   u32 sequence,
+						   u32 *out_status,
+						   u8 *out_params,
+						   u32 *out_params_size)
+{
+	struct tu104_get_constructed_falcon_info_params *falcon_params;
+
+	falcon_params = (struct tu104_get_constructed_falcon_info_params *)out_params;
+
+	falcon_params->numConstructedFalcons = TU104_FALCON_TABLE_SIZE;
+	memcpy(falcon_params->constructedFalconsTable, tu104_falcon_table,
+	       TU104_FALCON_TABLE_SIZE *
+	       sizeof(struct tu104_constructed_falcon_info));
+
+	*out_params_size = offsetof(struct tu104_get_constructed_falcon_info_params,
+				     constructedFalconsTable) +
+			   TU104_FALCON_TABLE_SIZE *
+			   sizeof(struct tu104_constructed_falcon_info);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: GET_CONSTRUCTED_FALCON_INFO numFalcons=%u",
+		     TU104_FALCON_TABLE_SIZE);
+}
+
+static void gsp_rm_handle_internal_device_table(struct gsp_emu_context *ctx,
+						 u32 sequence,
+						 u32 *out_status,
+						 u8 *out_params,
+						 u32 *out_params_size)
+{
+	struct tu104_internal_get_device_info_params *dev_params;
+	u32 copy_size;
+
+	dev_params = (struct tu104_internal_get_device_info_params *)out_params;
+
+	dev_params->numEntries = TU104_INTERNAL_DEVICE_TABLE_SIZE;
+	copy_size = TU104_INTERNAL_DEVICE_TABLE_SIZE *
+		    sizeof(struct tu104_internal_device_info);
+	memcpy(dev_params->deviceInfoTable, tu104_internal_device_table,
+	       copy_size);
+
+	*out_params_size = offsetof(struct tu104_internal_get_device_info_params,
+				     deviceInfoTable) + copy_size;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: INTERNAL_GET_DEVICE_INFO_TABLE numEntries=%u",
+		     TU104_INTERNAL_DEVICE_TABLE_SIZE);
+}
+
+/*
+ * INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP (0x20800a41)
+ *
+ * Returns a compressed register access map telling the guest which BAR0
+ * registers are userspace-accessible. For our emulated GPU, return an
+ * empty map (no compressed data, no profiling ranges).
+ */
+static void gsp_rm_handle_user_register_access_map(struct gsp_emu_context *ctx,
+						    u32 sequence,
+						    u32 *out_status,
+						    u8 *out_params,
+						    u32 *out_params_size)
+{
+	/* NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS:
+	 *   userRegisterAccessMapSize(4) + compressedSize(4) +
+	 *   compressedData[4096] + profilingRangesSize(4) + profilingRanges[4096]
+	 * Total: 8204 bytes
+	 */
+	u32 val32 = 0;
+
+	memset(out_params, 0, 8204);
+	memcpy(out_params + 0, &val32, 4);		/* userRegisterAccessMapSize */
+	memcpy(out_params + 4, &val32, 4);		/* compressedSize */
+	/* compressedData[4096] = zeros */
+	memcpy(out_params + 4104, &val32, 4);		/* profilingRangesSize */
+	/* profilingRanges[4096] = zeros */
+
+	*out_params_size = 8204;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: USER_REGISTER_ACCESS_MAP -> empty map");
+}
+
+/*
+ * INTERNAL_FIFO_GET_NUM_CHANNELS (0x20800a61)
+ *
+ * The guest asks GSP-RM how many channels each runlist supports
+ * (kfifoRunlistQueryNumChannels). Returning 0 trips the driver's
+ * NV_ASSERT(numChannels > 0) and kdisp can't allocate its
+ * clientChannelTable. Report the consumer-Turing standard 64.
+ */
+static void gsp_rm_handle_fifo_get_num_channels(struct gsp_emu_context *ctx,
+						u32 sequence,
+						u32 *out_status,
+						u8 *out_params,
+						u32 *out_params_size)
+{
+	u32 val32;
+
+	/* params: runlistId(4, in) + numChannels(4, out) */
+	val32 = TU104_FIFO_NUM_CHANNELS_PER_RUNLIST;
+	memcpy(out_params + 4, &val32, 4);
+
+	*out_params_size = 8;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: FIFO_GET_NUM_CHANNELS -> %u", val32);
+}
+
+/*
+ * BUS_GET_PCI_BAR_INFO (0x20801803)
+ *
+ * The guest caches pciBarSizes[] from this call (kern_bus.c). Reporting
+ * zeros leaves BAR1/BAR2 aperture lengths at 0. Report our emulated BAR
+ * layout; index 2 is the instance-memory window convention
+ * (BAR0+0x700000, 1MB - gpuPhysInstAddr fallback when PCI BAR2 is absent).
+ */
+static void gsp_rm_handle_bus_get_pci_bar_info(struct gsp_emu_context *ctx,
+					       u32 sequence,
+					       u32 *out_status,
+					       u8 *out_params,
+					       u32 *out_params_size)
+{
+	struct nv_vgpu_device *vgpu = ctx->vgpu;
+	u32 count = 4;
+	u32 i;
+	u64 sizes[4];
+	u64 offsets[4];
+
+	sizes[0] = vgpu->bars[0].size;   offsets[0] = vgpu->bars[0].gpa;
+	sizes[1] = vgpu->bars[1].size;   offsets[1] = vgpu->bars[1].gpa;
+	sizes[2] = 0x100000;             offsets[2] = vgpu->bars[0].gpa + 0x700000;
+	sizes[3] = vgpu->bars[3].size;   offsets[3] = vgpu->bars[3].gpa;
+
+	memset(out_params, 0, 8 + 8 * 24);
+	memcpy(out_params + 0, &count, 4);
+
+	for (i = 0; i < count; i++) {
+		u8 *e = out_params + 8 + i * 24;
+		u32 flags = 0;
+		u32 barSizeMB = (u32)(sizes[i] >> 20);
+
+		memcpy(e + 0, &flags, 4);
+		memcpy(e + 4, &barSizeMB, 4);
+		memcpy(e + 8, &sizes[i], 8);
+		memcpy(e + 16, &offsets[i], 8);
+	}
+
+	*out_params_size = 8 + count * 24;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: BUS_GET_PCI_BAR_INFO -> %u BARs (BAR1=%uMB)",
+		     count, (u32)(sizes[1] >> 20));
+}
+
+/*
+ * INTERNAL_DISPLAY_GET_STATIC_INFO (0x20800a01)
+ *
+ * kdispStateInitLocked caches numHeads/numDispChannels from this call and
+ * allocates clientChannelTable[numDispChannels] - a zero reply makes the
+ * allocation fail and kills RmInitNvDevice. The driver's wire struct is
+ * 36 bytes (observed paramsSize): feHwSysCap(0) windowPresentMask(4)
+ * bFbRemapperEnabled(8) numHeads(12) i2cPort(16) internalDispActiveMask(20)
+ * embeddedDisplayPortMask(24) muxSupported(28) numDispChannels(32).
+ * Report a minimal headless display: 1 head, 1 (core) channel.
+ */
+static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx,
+						  u32 sequence,
+						  u32 *out_status,
+						  u8 *out_params,
+						  u32 *out_params_size)
+{
+	u32 val32;
+
+	memset(out_params, 0, 36);
+
+	val32 = 1;	/* numHeads */
+	memcpy(out_params + 12, &val32, 4);
+	val32 = 1;	/* numDispChannels (core channel) */
+	memcpy(out_params + 32, &val32, 4);
+
+	*out_params_size = 36;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=1");
+}
+
+/*
+ * INTERNAL_DISPLAY_GET_IP_VERSION (0x20800a4b)
+ *
+ * kdispStatePreInitLocked passes an UNINITIALIZED ctrlParams and feeds the
+ * reply straight into gpuInitDispIpHal - it must be a real IP version.
+ * Turing display is DISP v04.00 (DispIpHal halspec idx 10).
+ */
+static void gsp_rm_handle_display_get_ip_version(struct gsp_emu_context *ctx,
+						 u32 sequence,
+						 u32 *out_status,
+						 u8 *out_params,
+						 u32 *out_params_size)
+{
+	u32 val32 = TU104_DISPLAY_IP_VERSION;
+
+	memcpy(out_params, &val32, 4);
+	*out_params_size = 4;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: DISPLAY_GET_IP_VERSION -> 0x%x", val32);
+}
+
+/*
+ * CE_GET_FAULT_METHOD_BUFFER_SIZE (0x20802a08)
+ *
+ * kchangrpInit allocates one fault method buffer per runqueue of this
+ * size; a zero reply makes the allocation fail (NV_ERR_NO_MEMORY) and
+ * kills channel-group creation. GV100 formula inherited by Turing:
+ * 135 * (13*totalCes + 2) page-aligned; TU104 = 5 PCEs -> 12288.
+ */
+static void gsp_rm_handle_ce_get_fault_method_buffer_size(
+						struct gsp_emu_context *ctx,
+						u32 sequence,
+						u32 *out_status,
+						u8 *out_params,
+						u32 *out_params_size)
+{
+	u32 val32 = TU104_CE_FAULT_METHOD_BUFFER_SIZE;
+
+	memcpy(out_params, &val32, 4);
+	*out_params_size = 4;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_FAULT_METHOD_BUFFER_SIZE -> %u", val32);
+}
+
+/*
+ * CE_GET_PHYSICAL_CAPS (0x20802a07) and friends
+ *
+ * kceGetDeviceCaps (kernel_ce.c) fills the CE caps the client hands to UVM
+ * from NV2080_CTRL_CMD_CE_GET_PHYSICAL_CAPS, which reaches GSP-RM. UVM's
+ * channel manager rejects any CE without the SYSMEM cap for its
+ * CPU_TO_GPU/GPU_TO_CPU channels ("Failed to initialize the channel manager:
+ * NV_ERR_NOT_SUPPORTED" in uvm_gpu.c) -- with the previous zeroed reply every
+ * CE was unusable and UVM_REGISTER_GPU failed with 0x56.
+ *
+ * params: { ceEngineType(4), capsTbl[2] } (NV2080_CTRL_CE_GET_CAPS_V2_PARAMS
+ * layout). Turing LCEs are sysmem-capable, P2P-capable and not GRCE.
+ */
+static void gsp_rm_handle_ce_physical_caps(struct gsp_emu_context *ctx,
+					   const u8 *req_params, u32 req_size,
+					   u32 *out_status, u8 *out_params,
+					   u32 *out_params_size)
+{
+	u32 ce_type = 0;
+
+	if (req_size >= 4)
+		memcpy(&ce_type, req_params, 4);
+
+	memset(out_params, 0, 8);
+	memcpy(out_params + 0, &ce_type, 4);
+
+	if (ce_type >= TU104_ENGINE_TYPE_COPY0 &&
+	    ce_type < TU104_ENGINE_TYPE_COPY0 + TU104_NUM_CES) {
+		out_params[4] = TU104_CE_CAPS0;
+		out_params[5] = TU104_CE_CAPS1;
+	} else {
+		*out_status = 0x56;	/* NV_ERR_NOT_SUPPORTED */
+		*out_params_size = req_size;
+		nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_PHYSICAL_CAPS type=0x%x -> NOT_SUPPORTED",
+			     ce_type);
+		return;
+	}
+
+	*out_params_size = req_size ? req_size : 8;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_PHYSICAL_CAPS type=0x%x -> caps=%02x %02x",
+		     ce_type, out_params[4], out_params[5]);
+}
+
+/*
+ * CE_GET_ALL_PHYSICAL_CAPS (0x20802a0b) / CE_GET_ALL_CAPS (0x20802a0a)
+ *
+ * params: NvU8 capsTbl[64][2] + NvU64 present (136 bytes). Report the five
+ * Turing CEs as present with the same caps as the per-CE query.
+ */
+static void gsp_rm_handle_ce_all_caps(struct gsp_emu_context *ctx,
+				      u32 *out_status, u8 *out_params,
+				      u32 *out_params_size)
+{
+	u64 present = 0;
+	u32 i;
+
+	memset(out_params, 0, 136);
+	for (i = 0; i < TU104_NUM_CES; i++) {
+		out_params[i * 2 + 0] = TU104_CE_CAPS0;
+		out_params[i * 2 + 1] = TU104_CE_CAPS1;
+		present |= 1ULL << i;
+	}
+	memcpy(out_params + 128, &present, 8);
+
+	*out_params_size = 136;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_ALL(_PHYSICAL)_CAPS -> %u CEs present=0x%llx",
+		     TU104_NUM_CES, present);
+}
+
+/*
+ * CE_GET_CE_PCE_MASK (0x20802a02)
+ *
+ * params: { ceEngineType(4), pceMask(4) }. Each Turing LCE has its own PCE,
+ * so every CE maps to exactly one PCE bit. UVM's queryCopyEngines stores this
+ * in UvmGpuCopyEngineCaps.cePceMask.
+ */
+static void gsp_rm_handle_ce_pce_mask(struct gsp_emu_context *ctx,
+				      const u8 *req_params, u32 req_size,
+				      u32 *out_status, u8 *out_params,
+				      u32 *out_params_size)
+{
+	u32 ce_type = 0, mask = 0;
+
+	if (req_size >= 4)
+		memcpy(&ce_type, req_params, 4);
+
+	if (ce_type >= TU104_ENGINE_TYPE_COPY0 &&
+	    ce_type < TU104_ENGINE_TYPE_COPY0 + TU104_NUM_CES)
+		mask = 1u << (ce_type - TU104_ENGINE_TYPE_COPY0);
+
+	memset(out_params, 0, 8);
+	memcpy(out_params + 0, &ce_type, 4);
+	memcpy(out_params + 4, &mask, 4);
+
+	*out_params_size = req_size ? req_size : 8;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_CE_PCE_MASK type=0x%x -> 0x%x",
+		     ce_type, mask);
+}
+
+/*
+ * INTERNAL_GMMU_GET_STATIC_INFO (0x20800a59)
+ *
+ * The guest asks GSP-RM (us) for the GMMU static info; kgmmu. pStaticInfo->
+ * replayableFaultBufferSize feeds kgmmuFaultBufferReplayableAllocate (called
+ * from faultbufConstruct when UVM opens the device). Zero -> 0-page alloc ->
+ * NV_ERR_NO_MEMORY -> faultbufConstruct fails and cuInit aborts.
+ * params: replayableFaultBufferSize(4) + replayableShadowFaultBufferMetadata
+ * Size(4) + nonReplayableFaultBufferSize(4) + nonReplayableShadowFaultBuffer
+ * MetadataSize(4). Shadow metadata is only used with confidential computing,
+ * so it stays 0.
+ */
+static void gsp_rm_handle_gmmu_get_static_info(struct gsp_emu_context *ctx,
+					       u32 sequence,
+					       u32 *out_status,
+					       u8 *out_params,
+					       u32 *out_params_size)
+{
+	u32 replay = TU104_GMMU_REPLAYABLE_FAULT_BUFFER_SIZE;
+	u32 nonreplay = TU104_GMMU_NONREPLAYABLE_FAULT_BUFFER_SIZE;
+	u32 zero = 0;
+
+	memcpy(out_params + 0, &replay, 4);
+	memcpy(out_params + 4, &zero, 4);
+	memcpy(out_params + 8, &nonreplay, 4);
+	memcpy(out_params + 12, &zero, 4);
+
+	*out_params_size = 16;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: GMMU_GET_STATIC_INFO -> replayFaultBuf=0x%x nonReplayFaultBuf=0x%x",
+		     replay, nonreplay);
+}
+
+/*
+ * STATIC_KGR_GET_GLOBAL_SM_ORDER (0x20800a22)
+ *
+ * kgrStaticInit copies globalSmOrder[grIdx] into the GR engine's static
+ * info - init-critical (NV_CHECK_OK_OR_GOTO, cannot fail). Zeros leave
+ * numSm=0 and the driver/nvidia-smi spin on empty topology. Report the
+ * TU104 layout: 6 GPCs x 4 TPCs x 2 SMs = 48 SMs on GR0, engines 1-7 empty.
+ * params: globalSmOrder[8] x (globalSmId[240]x12B + numSm(2) + numTpc(2)).
+ */
+static void gsp_rm_handle_kgr_get_global_sm_order(struct gsp_emu_context *ctx,
+						  u32 sequence,
+						  u32 *out_status,
+						  u8 *out_params,
+						  u32 *out_params_size)
+{
+	struct tu104_gr_global_sm_order *order =
+		(struct tu104_gr_global_sm_order *)out_params; /* engine 0 */
+	u32 idx = 0;
+	u32 gpc, tpc, sm;
+
+	memset(out_params, 0, TU104_GR_MAX_ENGINES *
+	       sizeof(struct tu104_gr_global_sm_order));
+
+	for (gpc = 0; gpc < TU104_NUM_GPC; gpc++) {
+		for (tpc = 0; tpc < TU104_TPC_PER_GPC; tpc++) {
+			for (sm = 0; sm < TU104_SM_PER_TPC; sm++) {
+				struct tu104_gr_sm_id *e = &order->globalSmId[idx++];
+
+				e->gpcId = gpc;
+				e->localTpcId = tpc;
+				e->localSmId = sm;
+				e->globalTpcId = gpc * TU104_TPC_PER_GPC + tpc;
+				e->virtualGpcId = gpc;
+				e->migratableTpcId = e->globalTpcId;
+			}
+		}
+	}
+	order->numSm = TU104_NUM_SM;
+	order->numTpc = TU104_NUM_TPC;
+
+	*out_params_size = TU104_GR_MAX_ENGINES *
+			   sizeof(struct tu104_gr_global_sm_order);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_GLOBAL_SM_ORDER -> %u SMs, %u TPCs on GR0",
+		     TU104_NUM_SM, TU104_NUM_TPC);
+}
+
+/*
+ * STATIC_KGR_GET_INFO (0x20800a2a)
+ *
+ * Returns the GR infoList the driver caches as pGrInfo and reads by index -
+ * critically MAX_SUBCONTEXT_COUNT (0x2c): zero makes the channel group's
+ * numMax==0 and channel allocation asserts (numMax == numFree && numMax != 0
+ * -> NV_ERR_INVALID_STATE -> "No devices"). Fill TU104 litter topology +
+ * subcontext counts (MAX_VEID_COUNT_INIT = 64). engineInfo[8] x infoList[58]
+ * of {index,data} = 3712 bytes; engine 0 = GR0.
+ */
+static void gsp_rm_handle_kgr_get_info(struct gsp_emu_context *ctx,
+				       u32 sequence,
+				       u32 *out_status,
+				       u8 *out_params,
+				       u32 *out_params_size)
+{
+	struct tu104_gr_static_info *info =
+		(struct tu104_gr_static_info *)out_params; /* engine 0 */
+	u32 i;
+
+	memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES *
+	       sizeof(struct tu104_gr_static_info));
+
+	/* positional array: index field = position for every entry */
+	for (i = 0; i < TU104_GR_INFO_MAX_SIZE; i++)
+		info->infoList[i].index = i;
+
+	/* TU104 litter topology: 6 GPC x 4 TPC x 2 SM = 48 SMs */
+	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_GPCS].data = TU104_NUM_GPC;
+	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_TPC_PER_GPC].data = TU104_TPC_PER_GPC;
+	info->infoList[TU104_GR_INFO_IDX_GPU_CORE_COUNT].data = TU104_NUM_SM;
+	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_SM_PER_TPC].data = TU104_SM_PER_TPC;
+	info->infoList[TU104_GR_INFO_IDX_FAMILY_MAX_TPC_PER_GPC].data = TU104_TPC_PER_GPC;
+
+	/* subcontext (VEID) counts: 64 per hw */
+	info->infoList[TU104_GR_INFO_IDX_MAX_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;
+	info->infoList[TU104_GR_INFO_IDX_MAX_LEGACY_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;
+	info->infoList[TU104_GR_INFO_IDX_MAX_PER_ENGINE_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;
+
+	*out_params_size = TU104_GR_INFO_MAX_ENGINES *
+			   sizeof(struct tu104_gr_static_info);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_INFO -> GR0 litter %uGPC/%uTPC/%uSM, maxSubctx=%u",
+		     TU104_NUM_GPC, TU104_TPC_PER_GPC, TU104_NUM_SM,
+		     TU104_MAX_SUBCONTEXT_COUNT);
+}
+
+/*
+ * STATIC_KGR_GET_FLOORSPEEPING_MASKS (0x20800a26)
+ *
+ * Driver computes numGpcs from floorsweepingMasks.gpcMask; zero kills
+ * krcWatchdogInit ("numGpcs > 0" assert -> RmInitAdapter bails). Fill
+ * TU104's present-GPC/TPC masks: 6 GPCs (bits 0-5), 4 TPCs per GPC.
+ */
+static void gsp_rm_handle_kgr_get_floorsweeping_masks(
+						struct gsp_emu_context *ctx,
+						u32 sequence,
+						u32 *out_status,
+						u8 *out_params,
+						u32 *out_params_size)
+{
+	struct tu104_gr_floorsweeping_masks *m =
+		(struct tu104_gr_floorsweeping_masks *)out_params; /* engine 0 */
+	u32 g;
+
+	memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES *
+	       sizeof(struct tu104_gr_floorsweeping_masks));
+
+	m->gpcMask = (1U << TU104_NUM_GPC) - 1;		/* bits 0-5 = 6 GPCs */
+	m->physGpcMask = m->gpcMask;
+	m->physGfxGpcMask = m->gpcMask;
+	m->numGfxTpc = TU104_NUM_TPC;
+	for (g = 0; g < TU104_NUM_GPC; g++) {
+		m->tpcMask[g] = (1U << TU104_TPC_PER_GPC) - 1;	/* 4 TPCs */
+		m->tpcCount[g] = TU104_TPC_PER_GPC;
+		m->mmuPerGpc[g] = 1;
+		m->numPesPerGpc[g] = 1;
+		m->zcullMask[g] = (1U << TU104_TPC_PER_GPC) - 1;
+	}
+
+	*out_params_size = TU104_GR_INFO_MAX_ENGINES *
+			   sizeof(struct tu104_gr_floorsweeping_masks);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_FLOORSWEEPING_MASKS -> gpcMask=0x%x tpc/gpc=%u",
+		     m->gpcMask, TU104_TPC_PER_GPC);
+}
+
+/*
+ * STATIC_KGR_GET_CONTEXT_BUFFERS_INFO (0x20800a32)
+ *
+ * kgraphicsInitializeDeferredStaticData() asks the physical RM (GSP-RM) for
+ * the per-engine context buffer sizes; kgrctxAllocMainCtxBuffer() then sizes
+ * the golden image ctx buffer from engine[GRAPHICS].size. The zeroed generic
+ * reply left size == 0 -> a 0-page allocation -> NV_ERR_NO_MEMORY at
+ * mem_desc.c:1353 (kgrctxAllocMainCtxBuffer -> kgraphicsCreateGoldenImageChannel).
+ *
+ * The real physical RM probes these from the ctxsw ucode
+ * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and reports NV_U32_MAX
+ * for unsupported engine IDs. The client skips NV_U32_MAX entries in
+ * kgraphicsCreateGoldenImageChannel's VA reservation, so the emulated layout
+ * below mirrors that: page-aligned scratch sizes for the buffers the client
+ * allocates, NV_U32_MAX for the engine IDs Turing does not support.
+ */
+static void gsp_rm_handle_kgr_get_context_buffers_info(
+					struct gsp_emu_context *ctx,
+					u32 sequence,
+					u32 *out_status,
+					u8 *out_params,
+					u32 *out_params_size)
+{
+	static const struct tu104_ctx_buffer_info sizes[TU104_CTX_ENGINE_ID_COUNT] = {
+		[TU104_CTX_ENG_GRAPHICS] = { TU104_CTX_SIZE_MAIN, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_ZCULL] = { TU104_CTX_SIZE_ZCULL, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_PM] = { TU104_CTX_SIZE_PM, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_PREEMPT] = { TU104_CTX_SIZE_PREEMPT, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_SPILL] = { TU104_CTX_SIZE_SPILL, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_PAGEPOOL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_BETACB] = { TU104_CTX_SIZE_BETACB, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_RTV] = { TU104_CTX_SIZE_RTV, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_PATCH] = { TU104_CTX_SIZE_PATCH, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_BUNDLE_CB] = { TU104_CTX_SIZE_BUNDLE_CB, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_PAGEPOOL_GLOBAL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_ATTRIBUTE_CB] = { TU104_CTX_SIZE_ATTRIBUTE_CB, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_RTV_CB_GLOBAL] = { TU104_CTX_SIZE_RTV_CB_GLOBAL, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_FECS_EVENT] = { TU104_CTX_SIZE_FECS_EVENT, 0x1000 },
+		[TU104_CTX_ENG_GRAPHICS_PRIV_ACCESS_MAP] = { TU104_CTX_SIZE_PRIV_ACCESS_MAP, 0x1000 },
+	};
+	struct tu104_gr_get_context_buffers_info_params *info =
+		(struct tu104_gr_get_context_buffers_info_params *)out_params;
+	u32 e, i;
+
+	memset(out_params, 0, sizeof(*info));
+
+	for (e = 0; e < TU104_CTX_GPU_MAX_ENGINES; e++) {
+		for (i = 0; i < TU104_CTX_ENGINE_ID_COUNT; i++) {
+			/* engine IDs the chip does not support report
+			 * NV_U32_MAX (the client skips those entries) */
+			if (sizes[i].size == 0) {
+				info->engineContextBuffersInfo[e].engine[i].size = 0xFFFFFFFF;
+				info->engineContextBuffersInfo[e].engine[i].alignment = 0xFFFFFFFF;
+				continue;
+			}
+			info->engineContextBuffersInfo[e].engine[i].size = sizes[i].size;
+			info->engineContextBuffersInfo[e].engine[i].alignment = sizes[i].alignment;
+		}
+	}
+
+	*out_params_size = sizeof(*info);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_CONTEXT_BUFFERS_INFO -> main ctx 0x%x, patch 0x%x, pm 0x%x (engines=%u x %u)",
+		     TU104_CTX_SIZE_MAIN, TU104_CTX_SIZE_PATCH, TU104_CTX_SIZE_PM,
+		     TU104_CTX_GPU_MAX_ENGINES, TU104_CTX_ENGINE_ID_COUNT);
+}
+
+/*
+ * FB_GET_INFO_V2 (0x20801303)
+ *
+ * The client (kern_mem_sys_ctrl.c _fbGetFbInfos) sends a list of FB info
+ * indices to the physical RM and expects the same indices back with data
+ * populated. A zeroed reply trips "pFbInfos[i].index == ..." asserts and
+ * leaves all FB info zero - CUDA/UVM read total/usable memory, BAR1 size,
+ * L2/LTC topology from here. Echo the requested indices and fill in the
+ * emulated TU104 values.
+ */
+static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
+{
+	struct nv_vgpu_device *vgpu = ctx->vgpu;
+	u64 fb = vgpu->fb_size ? vgpu->fb_size : NV_VGPU_BAR1_SIZE;
+
+	switch (index) {
+	case TU104_FB_INFO_IDX_TILE_REGION_COUNT:
+		return 0;
+	case TU104_FB_INFO_IDX_COMPRESSION_SIZE:
+		return 0;
+	case TU104_FB_INFO_IDX_PARTITION_COUNT:
+		return 1;
+	case TU104_FB_INFO_IDX_BAR1_SIZE:
+		return (u32)fb;
+	case TU104_FB_INFO_IDX_RAM_SIZE:
+	case TU104_FB_INFO_IDX_TOTAL_RAM_SIZE:
+	case TU104_FB_INFO_IDX_USABLE_RAM_SIZE:
+		return (u32)fb;
+	case TU104_FB_INFO_IDX_HEAP_SIZE:
+	case TU104_FB_INFO_IDX_MAPPABLE_HEAP_SIZE:
+		return (u32)fb;
+	case TU104_FB_INFO_IDX_BUS_WIDTH:
+		return 256;
+	case TU104_FB_INFO_IDX_RAM_TYPE:
+		return TU104_RAM_TYPE_GDDR6;
+	case TU104_FB_INFO_IDX_BANK_COUNT:
+		return 8;
+	case TU104_FB_INFO_IDX_FB_TAX_SIZE_KB:
+	case TU104_FB_INFO_IDX_LARGEST_FREE_REGION_SIZE_KB:
+		return (u32)(fb >> 10);
+	case TU104_FB_INFO_IDX_PARTITION_MASK:
+		return 1;
+	case TU104_FB_INFO_IDX_HEAP_FREE:
+		return (u32)fb;
+	case TU104_FB_INFO_IDX_RAM_LOCATION:
+		return TU104_RAM_LOCATION_GPU_DEDICATED;
+	case TU104_FB_INFO_IDX_FBP_COUNT:
+		return 1;
+	case TU104_FB_INFO_IDX_FBP_MASK:
+		return 1;
+	case TU104_FB_INFO_IDX_L2CACHE_SIZE:
+		return TU104_L2CACHE_SIZE;
+	case TU104_FB_INFO_IDX_MEMORYINFO_VENDOR_ID:
+		return TU104_MEMORYINFO_VENDOR_ID_SAMSUNG;
+	case TU104_FB_INFO_IDX_BAR1_AVAIL_SIZE:
+	case TU104_FB_INFO_IDX_BAR1_MAX_CONTIG_AVAIL_SIZE:
+		return (u32)fb;
+	case TU104_FB_INFO_IDX_LTC_COUNT:
+		return TU104_LTC_COUNT;
+	case TU104_FB_INFO_IDX_LTS_COUNT:
+		return TU104_LTS_COUNT;
+	case TU104_FB_INFO_IDX_PSEUDO_CHANNEL_MODE:
+		return TU104_PSEUDO_CHANNEL_MODE_DISABLED;
+	case TU104_FB_INFO_IDX_ALLOW_PAGE_RETIREMENT:
+		return 0;
+	case TU104_FB_INFO_IDX_LTC_MASK:
+		return (1U << TU104_LTC_COUNT) - 1;
+	default:
+		return 0;
+	}
+}
+
+static void gsp_rm_handle_fb_get_info_v2(struct gsp_emu_context *ctx,
+					 const u8 *req_params,
+					 u32 req_params_size,
+					 u32 *out_status,
+					 u8 *out_params,
+					 u32 *out_params_size)
+{
+	struct tu104_fb_get_info_v2_params *reply =
+		(struct tu104_fb_get_info_v2_params *)out_params;
+	u32 count = 0;
+	u32 i;
+
+	if (req_params_size >= sizeof(count))
+		memcpy(&count, req_params, sizeof(count));
+	if (count > TU104_FB_INFO_MAX_LIST_SIZE)
+		count = TU104_FB_INFO_MAX_LIST_SIZE;
+
+	memset(out_params, 0, sizeof(*reply));
+	reply->fbInfoListSize = count;
+
+	for (i = 0; i < count; i++) {
+		const u8 *in = req_params + sizeof(count) +
+			       i * sizeof(struct tu104_fb_info_entry);
+		u32 index = 0;
+
+		if (sizeof(count) + (i + 1) * sizeof(struct tu104_fb_info_entry) <=
+		    req_params_size)
+			memcpy(&index, in, sizeof(index));
+
+		reply->fbInfoList[i].index = index;
+		reply->fbInfoList[i].data = tu104_fb_info_data(ctx, index);
+	}
+
+	*out_params_size = sizeof(*reply);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: FB_GET_INFO_V2 -> %u entries populated (first idx=0x%x)",
+		     count, count ? reply->fbInfoList[0].index : 0);
+}
+
+/*
+ * BUS_GET_INFO_V2 (0x20801823)
+ *
+ * Same flat index/data list pattern as FB_GET_INFO_V2. UVM's
+ * gpuGetSysmemLinkInfo() asks for SYSMEM_CONNECTION_TYPE and then
+ * PCIE_GPU_LINK_CAPS to compute the sysmem link bandwidth; a zeroed reply
+ * makes getPCIELinkRateMBps() return NV_ERR_INVALID_STATE ("Unsupported
+ * sysmem connection type"). Report a PCIe Gen3 x16 link for the emulated
+ * vGPU and echo every requested index.
+ */
+static u32 tu104_bus_info_data(u32 index)
+{
+	switch (index) {
+	case TU104_BUS_INFO_IDX_TYPE:
+		return TU104_BUS_INFO_TYPE_PCI_EXPRESS;
+	case TU104_BUS_INFO_IDX_INTLINE:
+		return 0;
+	case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CAPS:
+	case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CAPS:
+	case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CAPS:
+	case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CAPS:
+	case TU104_BUS_INFO_IDX_PCIE_GEN_INFO:
+		return TU104_PCIE_LINK_CAP_GEN3_X16;
+	case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CTRL_STATUS:
+	case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CTRL_STATUS:
+	case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CTRL:
+	case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CTRL:
+		return TU104_PCIE_LINK_CTRL_STATUS_GEN3_X16;
+	case TU104_BUS_INFO_IDX_SYSMEM_CONNECTION_TYPE:
+		return TU104_BUS_INFO_SYSMEM_CONN_PCIE;
+	default:
+		return 0;
+	}
+}
+
+static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx,
+					  const u8 *req_params,
+					  u32 req_params_size,
+					  u32 *out_status,
+					  u8 *out_params,
+					  u32 *out_params_size)
+{
+	struct tu104_bus_get_info_v2_params *reply =
+		(struct tu104_bus_get_info_v2_params *)out_params;
+	u32 count = 0;
+	u32 i;
+
+	if (req_params_size >= sizeof(count))
+		memcpy(&count, req_params, sizeof(count));
+	if (count > TU104_BUS_INFO_MAX_LIST_SIZE)
+		count = TU104_BUS_INFO_MAX_LIST_SIZE;
+
+	memset(out_params, 0, sizeof(*reply));
+	reply->busInfoListSize = count;
+
+	for (i = 0; i < count; i++) {
+		const u8 *in = req_params + sizeof(count) +
+			       i * sizeof(struct tu104_bus_info_entry);
+		u32 index = 0;
+
+		if (sizeof(count) + (i + 1) * sizeof(struct tu104_bus_info_entry) <=
+		    req_params_size)
+			memcpy(&index, in, sizeof(index));
+
+		reply->busInfoList[i].index = index;
+		reply->busInfoList[i].data = tu104_bus_info_data(index);
+	}
+
+	*out_params_size = sizeof(*reply);
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: BUS_GET_INFO_V2 -> %u entries populated (first idx=0x%x)",
+		     count, count ? reply->busInfoList[0].index : 0);
+	for (i = 0; i < count; i++)
+		nv_vgpu_info(ctx->vgpu, "GSPRM:   bus[%u] idx=0x%x data=0x%x",
+			     i, reply->busInfoList[i].index,
+			     reply->busInfoList[i].data);
+}
+
+/*
+ * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106)
+ *
+ * The client tells the server (us) the physical addresses of its GPU page
+ * table levels together with the VA range they cover:
+ *
+ *   struct {
+ *       NvHandle hSubDevice;        // +0
+ *       NvU32    subDeviceId;       // +4
+ *       NvU64    pageSize;          // +8   (coverage of the reserved level)
+ *       NvU64    virtAddrLo;        // +16
+ *       NvU64    virtAddrHi;        // +24
+ *       NvU32    numLevelsToCopy;   // +32
+ *       NvU32    pad;               // +36
+ *       struct {                    // +40, 24 bytes each, 6 max
+ *           NvU64 physAddress;
+ *           NvU64 size;
+ *           NvU32 aperture;         // 1=sysmem, 2=vidmem
+ *           NvU8  pageShift;
+ *       } levels[6];
+ *   } // 184 bytes
+ *
+ * This is the client's own page table geometry: remember it so client GPU VAs
+ * can be translated to guest physical addresses (e.g. to complete emulated
+ * GPU semaphore releases).
+ */
+#define GSPRM_COPY_PDES_LEVELS_OFF	40
+#define GSPRM_COPY_PDES_LEVEL_SIZE	24
+#define GSPRM_COPY_PDES_MAX_LEVELS	6
+
+static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
+					   const u8 *params, u32 params_size)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	u64 va_lo = 0, va_hi = 0, page_size = 0;
+	u32 num_levels = 0, i;
+
+	if (params_size < GSPRM_COPY_PDES_LEVELS_OFF)
+		return;
+
+	memcpy(&page_size, params + 8, 8);
+	memcpy(&va_lo, params + 16, 8);
+	memcpy(&va_hi, params + 24, 8);
+	memcpy(&num_levels, params + 32, 4);
+	if (num_levels > GSPRM_COPY_PDES_MAX_LEVELS)
+		num_levels = GSPRM_COPY_PDES_MAX_LEVELS;
+
+	rm->pt.va_lo = va_lo;
+	rm->pt.va_hi = va_hi;
+	rm->pt.page_size = page_size;
+	rm->pt.num_levels = num_levels;
+
+	nv_vgpu_info(ctx->vgpu,
+		     "GSPRM: COPY_SERVER_PDES va=[0x%llx..0x%llx] pageSize=0x%llx levels=%u",
+		     va_lo, va_hi, page_size, num_levels);
+
+	for (i = 0; i < num_levels; i++) {
+		const u8 *lv = params + GSPRM_COPY_PDES_LEVELS_OFF +
+			       i * GSPRM_COPY_PDES_LEVEL_SIZE;
+		u64 phys = 0, size = 0;
+		u32 aperture = 0;
+		u8 page_shift = 0;
+
+		if (GSPRM_COPY_PDES_LEVELS_OFF +
+		    (i + 1) * GSPRM_COPY_PDES_LEVEL_SIZE > params_size)
+			break;
+
+		memcpy(&phys, lv + 0, 8);
+		memcpy(&size, lv + 8, 8);
+		memcpy(&aperture, lv + 16, 4);
+		memcpy(&page_shift, lv + 20, 1);
+
+		rm->pt.levels[i].phys = phys;
+		rm->pt.levels[i].size = size;
+		rm->pt.levels[i].aperture = aperture;
+		rm->pt.levels[i].page_shift = page_shift;
+
+		nv_vgpu_info(ctx->vgpu,
+			     "GSPRM:   pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u",
+			     i, phys, size, aperture, page_shift);
+	}
+}
+
+static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
+				     const u8 *elem, u32 sequence)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE +
+			     sizeof(struct gsprm_rpc_header);
+	struct gsp_rm_control_req req;
+	u32 reply_data_len;
+	u8 *reply;
+	u32 out_status = 0;
+	u32 out_params_size = 0;
+
+	memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE);
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_RM_CONTROL cmd=0x%x hClient=0x%x hObject=0x%x paramsSize=%u flags=0x%x",
+		     req.cmd, req.hClient, req.hObject, req.paramsSize,
+		     req.flags);
+
+	/* Allocate reply: control header + max possible params */
+	reply_data_len = GSP_RM_CTRL_HDR_SIZE + max(req.paramsSize, (u32)4096);
+	reply = kzalloc(reply_data_len, GFP_KERNEL);
+	if (!reply) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply alloc failed");
+		return;
+	}
+
+	switch (req.cmd) {
+	case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE:
+		gsp_rm_handle_fifo_device_table(ctx, sequence, &req,
+			req_data + GSP_RM_CTRL_HDR_SIZE,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */
+		gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE,
+					       req.paramsSize);
+		out_status = 0;
+		out_params_size = 0;
+		break;
+
+	case TU104_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE:
+		gsp_rm_handle_intr_kernel_table(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO:
+		gsp_rm_handle_constructed_falcon_info(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE:
+		gsp_rm_handle_internal_device_table(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case 0x20800a41: /* INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP */
+		gsp_rm_handle_user_register_access_map(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS:
+		gsp_rm_handle_fifo_get_num_channels(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_BUS_GET_PCI_BAR_INFO:
+		gsp_rm_handle_bus_get_pci_bar_info(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO:
+		gsp_rm_handle_display_get_static_info(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION:
+		gsp_rm_handle_display_get_ip_version(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE:
+		gsp_rm_handle_ce_get_fault_method_buffer_size(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_CE_GET_PHYSICAL_CAPS:
+		gsp_rm_handle_ce_physical_caps(ctx,
+			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_CE_GET_CE_PCE_MASK:
+		gsp_rm_handle_ce_pce_mask(ctx,
+			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_CE_GET_ALL_CAPS:
+	case TU104_CTRL_CE_GET_ALL_PHYSICAL_CAPS:
+		gsp_rm_handle_ce_all_caps(ctx,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO:
+		gsp_rm_handle_gmmu_get_static_info(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_STATIC_KGR_GET_GLOBAL_SM_ORDER:
+		gsp_rm_handle_kgr_get_global_sm_order(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_STATIC_KGR_GET_INFO:
+		gsp_rm_handle_kgr_get_info(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO:
+		gsp_rm_handle_kgr_get_context_buffers_info(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_FB_GET_INFO_V2:
+		gsp_rm_handle_fb_get_info_v2(ctx,
+			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_BUS_GET_INFO_V2:
+		gsp_rm_handle_bus_get_info_v2(ctx,
+			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	case TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS:
+		gsp_rm_handle_kgr_get_floorsweeping_masks(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
+	default:
+		/* Unknown control command - return success with zeroed params.
+		 * Do NOT echo the request: pure queries pass uninitialized
+		 * stack structs as input (e.g. DISPLAY_GET_IP_VERSION), so an
+		 * echo hands the driver its own stack garbage as the answer. */
+		nv_vgpu_info(ctx->vgpu, "GSPRM: RM_CONTROL unknown cmd=0x%x -> generic SUCCESS",
+			     req.cmd);
+		/* Dump small request params so mapping/alloc controls that
+		 * carry PAs can be identified (rate-limited). */
+		if (req.paramsSize && req.paramsSize <= 96 &&
+		    rm->ctrl_param_dumps < 60) {
+			rm->ctrl_param_dumps++;
+			nv_vgpu_info(ctx->vgpu,
+				     "GSPRM:   ctrl 0x%x params[0..%u]=%*phN",
+				     req.cmd, req.paramsSize, req.paramsSize,
+				     req_data + GSP_RM_CTRL_HDR_SIZE);
+		}
+		out_status = 0;
+		out_params_size = req.paramsSize;
+		break;
+
+	case TU104_CTRL_I2C_TRANSACTION:
+		/* I2C transactions have variable-length FINN-serialized
+		 * params: an all-zero reply fails the driver's DeserializeUp
+		 * and the calling ioctl never completes (nvidia-smi hang).
+		 * Echo the request's serialized params (valid stream) and
+		 * report NOT_SUPPORTED - headless vGPU has no DDC/I2C bus. */
+		nv_vgpu_info(ctx->vgpu, "GSPRM: I2C_TRANSACTION -> NOT_SUPPORTED (echo)");
+		out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */
+		out_params_size = req.paramsSize;
+		if (req.paramsSize &&
+		    req.paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA)
+			memcpy(reply + GSP_RM_CTRL_HDR_SIZE,
+			       req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize);
+		break;
+	}
+
+	/* Build the reply header */
+	memcpy(reply + 0, &req.hClient, 4);
+	memcpy(reply + 4, &req.hObject, 4);
+	memcpy(reply + 8, &req.cmd, 4);
+	memcpy(reply + 12, &out_status, 4);
+	memcpy(reply + 16, &out_params_size, 4);
+	memcpy(reply + 20, &req.flags, 4);
+
+	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0,
+			reply, GSP_RM_CTRL_HDR_SIZE + out_params_size))
+		nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply send failed cmd=0x%x",
+			    req.cmd);
+	else
+		rm->rpcs_replied++;
+
+	kfree(reply);
+}
+
+/*
+ * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK.
+ *
+ * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the
+ * request's params back with status=0 so the driver's free rpc_message is a
+ * complete well-formed record. (The poll match is function-only; the echo
+ * keeps the post-poll result struct self-consistent.)
+ */
+static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
+			       u32 sequence)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	const struct gsprm_rpc_header *hdr =
+		(const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
+	const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr);
+	u8 params[16]; /* NVOS00_PARAMETERS: hRoot, hObjectParent, hObjectOld, status */
+	u32 hObjOld = 0;
+
+	if (hdr->length >= sizeof(*hdr) + sizeof(params))
+		memcpy(params, req, sizeof(params));
+	else
+		memset(params, 0, sizeof(params));
+
+	memcpy(&hObjOld, params + 8, 4);
+	memset(params + 12, 0, 4); /* status = NV_OK */
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u",
+		     hObjOld, sequence, rm->cmd_read, rm->stat_write);
+
+	/* Small delay before FREE/UNLOADING replies: mitigates an intermittent
+	 * driver-side reply-matching race in the cleanup path. */
+	udelay(2000);
+
+	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_FREE, sequence, 0,
+			params, sizeof(params)))
+		nv_vgpu_err(ctx->vgpu, "GSPRM: FREE reply send failed");
+	else
+		rm->rpcs_replied++;
+}
+
+/*
+ * Handle one command-queue record (first element). For async init RPCs we
+ * just consume. Synchronous RPCs get a reply; per-function handlers are
+ * implemented iteratively (unknown ones get a generic success reply).
+ */
+static bool gsp_rm_alloc_interesting(u32 class_id)
+{
+	/* Boring classes already decoded: root/device/subdevice */
+	switch (class_id) {
+	case 0x00:	/* NV01_ROOT */
+	case 0x80:	/* NV01_DEVICE_0 */
+	case 0x2080:	/* NV20_SUBDEVICE_0 */
+	case 0x2081:	/* NV2081_BINAPI */
+		return false;
+	default:
+		return true;
+	}
+}
+
+static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	const struct gsprm_rpc_header *hdr =
+		(const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
+	u32 function = hdr->function;
+	u32 sequence = hdr->sequence;
+
+	rm->rpcs_seen++;
+
+	switch (function) {
+	case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO:
+	case GSPRM_RPC_FUNC_SET_REGISTRY:
+	case GSPRM_RPC_FUNC_CONTINUATION_RECORD:
+		/* Async init RPCs - fire and forget, just consume. */
+		nv_vgpu_info(ctx->vgpu, "GSPRM: async RPC %s(%u) seq=%u len=%u consumed",
+			     gsp_rm_rpc_name(function), function, sequence,
+			     hdr->length);
+		return;
+
+	case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO:
+		gsp_rm_handle_set_guest_system_info(ctx, elem, sequence);
+		return;
+
+	case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO:
+		gsp_rm_handle_get_static_info(ctx, sequence);
+		return;
+
+	case GSPRM_RPC_FUNC_GSP_RM_CONTROL:
+		gsp_rm_handle_rm_control(ctx, elem, sequence);
+		return;
+
+	case GSPRM_RPC_FUNC_FREE:
+		gsp_rm_handle_free(ctx, elem, sequence);
+		return;
+
+	default:
+		/* Synchronous RPC: echo function+sequence with success so the
+		 * driver's rpcRecvPoll completes. */
+		rm->unknown_rpcs++;
+		if (function == 47) {
+			/* UNLOADING_GUEST_DRIVER: log for unload-cycle debugging */
+			nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u cmd_read=%u stat_write=%u stat_seq=%u",
+				     gsp_rm_rpc_name(function), function, sequence,
+				     rm->cmd_read, rm->stat_write, rm->stat_seq);
+		} else if (function == GSPRM_RPC_FUNC_GSP_RM_ALLOC) {
+			/* GSP_RM_ALLOC (103): rpc_gsp_rm_alloc_v03_00 at
+			 * elem+GSPRM_ELEM_HDR_SIZE. NOTE: the RPC header
+			 * 'length' only counts the fixed struct size (64), so
+			 * it must NOT be used to bound the params - read the
+			 * alloc header directly. See handoff PART 6. */
+			const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
+				      sizeof(struct gsprm_rpc_header);
+			const u8 *params = p + 32;
+			u32 hc = 0, hp = 0, ho = 0, class_id = 0;
+			u32 params_size = 0, flags = 0;
+
+			memcpy(&hc, p + 0, 4);
+			memcpy(&hp, p + 4, 4);
+			memcpy(&ho, p + 8, 4);
+			memcpy(&class_id, p + 12, 4);
+			memcpy(&params_size, p + 20, 4);
+			memcpy(&flags, p + 24, 4);
+
+			nv_vgpu_info(ctx->vgpu,
+				     "GSPRM: GSP_RM_ALLOC class=0x%x paramsSize=%u flags=0x%x hClient=0x%x hParent=0x%x hObject=0x%x",
+				     class_id, params_size, flags, hc, hp, ho);
+
+			/* Full param dump for memory/channel classes and any
+			 * class we have not classified yet (rate-limited).
+			 * %phN is capped at 64 bytes per print, so chunk it. */
+			if (gsp_rm_alloc_interesting(class_id) &&
+			    rm->alloc_param_dumps < 60) {
+				u32 shown = min(params_size, (u32)384);
+				u32 off;
+
+				rm->alloc_param_dumps++;
+				for (off = 0; off < shown; off += 64) {
+					u32 n = min(shown - off, (u32)64);
+
+					nv_vgpu_info(ctx->vgpu,
+						     "GSPRM:   alloc class=0x%x params[%u..%u]=%*phN",
+						     class_id, off, off + n, (int)n,
+						     params + off);
+				}
+			}
+		} else {
+			nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)",
+				     gsp_rm_rpc_name(function), function, sequence,
+				     hdr->length, rm->unknown_rpcs);
+		}
+		/* Small delay before UNLOADING replies: mitigates an intermittent
+		 * driver-side reply-matching race in the cleanup path. */
+		if (function == 47)
+			udelay(2000);
+		if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0))
+			nv_vgpu_err(ctx->vgpu, "GSPRM: reply send failed fn=%u", function);
+		else
+			rm->rpcs_replied++;
+		return;
+	}
+}
+
+/*
+ * Poll the command queue TX writePtr; consume and handle new records.
+ * A record spans elemCount consecutive queue elements.
+ *
+ * msgq pointers are SLOT INDICES that wrap at msgCount (msgq.c:
+ * "writePtr += n; if (>= msgCount) -= msgCount"). Empty when
+ * writePtr == readPtr. A 62->0 transition is a legitimate wrap, NOT
+ * queue death.
+ */
+static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	struct gsprm_msgq_tx_header tx;
+	int ret;
+
+	ret = gsp_rm_read_buf(ctx, rm->cmd_off, &tx, sizeof(tx));
+	if (ret) {
+		/* VM may be gone or device closed; stay quiet, retry later */
+		return;
+	}
+
+	/* A healthy driver ALWAYS has writePtr < msgCount. Anything else
+	 * means the queue header is garbage (buffer freed on unload/crash
+	 * without an observable GSP shutdown) - declare the emulated GSP
+	 * dead so the next probe's WPR2-up check passes and it re-boots. */
+	if (tx.writePtr >= rm->cmd_msgcount) {
+		nv_vgpu_warn(ctx->vgpu, "GSPRM: insane cmd writePtr %u (msgCount=%u), GSP boot declared dead",
+			     tx.writePtr, rm->cmd_msgcount);
+		gsp_emu_clear_boot_state(ctx, "stale GSP-RM queue header");
+		gsp_rm_emu_reset(ctx);
+		return;
+	}
+
+	/* Consume records until we catch up with the (wrapped) writePtr */
+	while (rm->cmd_read != tx.writePtr) {
+		u8 *elem;
+		u64 elem_off = rm->cmd_off + rm->entry_off +
+			       (u64)(rm->cmd_read % rm->cmd_msgcount) * GSPRM_QUEUE_ELEMENT_SIZE;
+		u32 signature, elem_count;
+		const struct gsprm_rpc_header *hdr;
+
+		elem = kmalloc(GSPRM_QUEUE_ELEMENT_SIZE, GFP_KERNEL);
+		if (!elem)
+			return;
+		ret = gsp_rm_read_buf(ctx, elem_off, elem, GSPRM_QUEUE_ELEMENT_SIZE);
+		if (ret) {
+			kfree(elem);
+			return;
+		}
+
+		hdr = (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
+		signature = hdr->signature;
+		memcpy(&elem_count, elem + 0x28, 4);
+
+		if (signature != GSPRM_RPC_SIGNATURE || elem_count == 0 ||
+		    elem_count > rm->cmd_msgcount) {
+			/* Garbage element: could be teardown (driver stopped
+			 * writing valid elements after UNLOADING_GUEST_DRIVER/
+			 * FREE sequence) or genuine queue death. Don't declare
+			 * death here - stop polling quietly and let the
+			 * Booter-unload kick (SEC2 CPUCTL) trigger the reset
+			 * via gsp_emu_clear_boot_state. This allows the
+			 * driver's cleanup frees to complete. */
+			nv_vgpu_info(ctx->vgpu, "GSPRM: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, stopping poll (teardown?)",
+				     rm->cmd_read, signature, elem_count,
+				     hdr->function);
+			kfree(elem);
+			return;
+		}
+
+		gsp_rm_handle_element(ctx, elem);
+		kfree(elem);
+
+		/* consume the whole record (all its elements), wrapping
+		 * at msgCount like the driver's own writePtr */
+		rm->cmd_read = (rm->cmd_read + elem_count) % rm->cmd_msgcount;
+	}
+
+	/* Publish our command-queue read position into the status queue RX
+	 * header (MSGQ_FLAGS_SWAP_RX: read pointers live in the peer's queue). */
+	gsp_rm_write_buf(ctx, rm->stat_off + 0x20,
+			 &rm->cmd_read, sizeof(rm->cmd_read));
+}
+
+/* ============================================================
+ * Public interface
+ * ============================================================ */
+
+int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm;
+
+	rm = kzalloc(sizeof(*rm), GFP_KERNEL);
+	if (!rm)
+		return -ENOMEM;
+
+	rm->vgpu = vgpu;
+	ctx->rm_emu = rm;
+	return 0;
+}
+
+void gsp_rm_emu_fini(struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+
+	if (!rm)
+		return;
+	kfree(rm->ptes);
+	kfree(rm);
+	ctx->rm_emu = NULL;
+}
+
+/*
+ * Reset the GSP-RM emulation state on GSP ENGINE reset (guest driver reboots
+ * the GSP on every probe/unload). The queue buffer GPAs from the previous
+ * boot are stale (freed by the guest) - drop all parsed state so the next
+ * Booter completion re-parses the chain.
+ */
+void gsp_rm_emu_reset(struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+
+	if (!rm || !rm->parsed)
+		return;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: reset (rpcs=%llu replied=%llu unknown=%llu)",
+		     rm->rpcs_seen, rm->rpcs_replied, rm->unknown_rpcs);
+
+	kfree(rm->ptes);
+	rm->ptes = NULL;
+	rm->parsed = false;
+	rm->active = false;
+	rm->init_done_sent = false;
+	rm->libos_args_gpa = 0;
+	rm->gsp_args_gpa = 0;
+	rm->cmd_read = 0;
+	rm->stat_write = 0;
+	rm->stat_seq = 0;
+	memset(&rm->mq, 0, sizeof(rm->mq));
+	rm->cmd_gpa = 0;
+	rm->stat_gpa = 0;
+	rm->cmd_size = 0;
+	rm->stat_size = 0;
+	rm->cmd_msgcount = 0;
+	rm->stat_msgcount = 0;
+}
+
+/*
+ * Called when the guest's Booter Load completes. The GSP MAILBOX0/1 now hold
+ * the libos init args GPA; parse the queue chain, link the status queue and
+ * emit GSP_INIT_DONE so the driver's kgspWaitForRmInitDone completes.
+ */
+void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	u32 mb_lo, mb_hi;
+	u32 b_is_d3hot = 0;
+	int ret;
+
+	if (!rm || rm->parsed)
+		return;
+
+	mb_lo = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0);
+	mb_hi = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0 + 4);
+	rm->libos_args_gpa = ((u64)mb_hi << 32) | mb_lo;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: libos args at gpa=0x%llx (mailbox 0x%x:0x%x)",
+		     rm->libos_args_gpa, mb_hi, mb_lo);
+
+	if (!rm->libos_args_gpa) {
+		pr_warn_ratelimited("GSPRM: libos args GPA is 0, will retry\n");
+		return;
+	}
+
+	ret = gsp_rm_parse_queue_chain(ctx);
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: queue chain parse failed: %d", ret);
+		return;
+	}
+
+	ret = gsp_rm_link_status_queue(ctx);
+	if (ret)
+		return;
+
+	rm->parsed = true;
+	rm->active = true;
+
+	/* Emit GSP_INIT_DONE (function 0x1001, sequence 0, init_done_v17_00) */
+	ret = gsp_rm_send(ctx, GSPRM_RPC_EVENT_GSP_INIT_DONE, 0, 0,
+			  &b_is_d3hot, sizeof(b_is_d3hot));
+	if (ret) {
+		nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_INIT_DONE send failed: %d", ret);
+		return;
+	}
+	rm->init_done_sent = true;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_INIT_DONE emitted, RPC loop active");
+}
+
+void gsp_rm_emu_poll(struct gsp_emu_context *ctx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+
+	if (!rm)
+		return;
+
+	/* Retry chain parsing if the first attempt at BOOTER_DONE failed
+	 * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE.
+	 * Back off hard: at 100us poll cadence an unthrottled retry spams
+	 * dmesg into unusability when the driver stalls mid-bootstrap. */
+	if (!rm->parsed) {
+		if (ctx->boot_stage == 2 &&
+		    time_after(jiffies, rm->last_parse_retry + HZ / 4)) {
+			rm->last_parse_retry = jiffies;
+			gsp_rm_emu_on_booter_done(ctx);
+		}
+		return;
+	}
+
+	if (!rm->active)
+		return;
+
+	gsp_rm_poll_commands(ctx);
+}