path: root/kernel/gsp_rm_emulation.c

/*
 * GSP-RM Emulation (physical-path) for Host-Side vGPU
 *
 * Emulates the GSP-RM msgq RPC channel in guest system memory.
 * See gsp_rm_emulation.h for the design overview.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/vfio.h>
#include <linux/delay.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_emulation.h"
#include "gsp_rm_emulation.h"
#include "gsp_rm_tu104.h"

/* ============================================================
 * Guest-memory access ("virtual DMA" on behalf of the device)
 * ============================================================ */

static int gsp_rm_read(struct gsp_emu_context *ctx, u64 gpa, void *buf,
		       size_t len)
{
	struct nv_vgpu_device *vgpu = ctx->vgpu;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int gsp_rm_write(struct gsp_emu_context *ctx, u64 gpa, void *buf,
			size_t len)
{
	struct nv_vgpu_device *vgpu = ctx->vgpu;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/* Translate a shared-buffer offset to a GPA via the buffer's page table */
static u64 gsp_rm_off_to_gpa(struct gsp_rm_emu *rm, u64 off)
{
	u64 idx = off >> 12;

	if (idx >= rm->mq.pageTableEntryCount)
		return 0;
	return rm->ptes[idx] + (off & 0xfff);
}

/*
 * Page-aware shared-buffer I/O. The queue buffer is NV_MEMORY_NONCONTIGUOUS:
 * its 4K pages are independent guest allocations, so a range crossing a page
 * boundary is NOT contiguous in guest physical memory. Translating the start
 * and blasting a multi-page range would scribble into arbitrary guest RAM
 * (observed as guest userspace segfaults). Split at page boundaries and
 * translate each page separately.
 */
static int gsp_rm_write_buf(struct gsp_emu_context *ctx, u64 off,
			    const void *buf, size_t len)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const u8 *p = buf;
	int ret;

	while (len) {
		u64 gpa = gsp_rm_off_to_gpa(rm, off);
		size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff));

		if (!gpa)
			return -EINVAL;
		ret = gsp_rm_write(ctx, gpa, (void *)p, chunk);
		if (ret)
			return ret;
		p += chunk;
		off += chunk;
		len -= chunk;
	}
	return 0;
}

static int gsp_rm_read_buf(struct gsp_emu_context *ctx, u64 off,
			   void *buf, size_t len)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u8 *p = buf;
	int ret;

	while (len) {
		u64 gpa = gsp_rm_off_to_gpa(rm, off);
		size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff));

		if (!gpa)
			return -EINVAL;
		ret = gsp_rm_read(ctx, gpa, p, chunk);
		if (ret)
			return ret;
		p += chunk;
		off += chunk;
		len -= chunk;
	}
	return 0;
}

/* ============================================================
 * Status queue element construction
 * ============================================================ */

/* XOR-fold checksum over u64s (message_queue_priv.h _checkSum32) */
static u32 gsp_rm_checksum32(const void *data, u32 len)
{
	const u64 *p = data;
	const u64 *end = (const u64 *)((const u8 *)data + len);
	u64 sum = 0;

	while (p < end)
		sum ^= *p++;

	return (u32)(sum >> 32) ^ (u32)sum;
}

/*
 * 570.144 GSP_MSG_QUEUE_ELEMENT wire layout (guest driver is 570.144 stock):
 *   0x00  authTagBuffer[16]  (zero - no CC)
 *   0x10  aadBuffer[16]      (zero - no CC)
 *   0x20  checkSum           (XOR-fold over msgLen, makes total 0)
 *   0x24  seqNum             (per-record sequence)
 *   0x28  elemCount          (elements this record spans)
 *   0x30  rpc_message_header_v (32 bytes) + payload
 * GSP_MSG_QUEUE_ELEMENT_HDR_SIZE = offsetof(rpc) = 0x30.
 */
#define GSPRM_ELEM_HDR_SIZE	0x30

/*
 * Max RPC data that fits in one queue element:
 * 4096 (element) - 0x30 (element hdr) - 32 (rpc hdr) = 4016 bytes.
 */
#define GSPRM_MAX_SINGLE_ELEM_DATA	(GSPRM_QUEUE_ELEMENT_SIZE - \
					 GSPRM_ELEM_HDR_SIZE - \
					 sizeof(struct gsprm_rpc_header))

/*
 * Build a multi-element record into a contiguous buffer.
 *
 * The record spans elem_count consecutive queue elements:
 *   [0]: element header (0x30) + rpc header (32) + data[0..4015]
 *   [1..N-1]: continuation data only (no headers)
 *
 * Returns the total record size in bytes (elem_count * 4096).
 * Sets *out_elem_count to the number of elements used.
 */
static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size,
				u32 function, u32 sequence, u32 result,
				const void *data, u32 data_len,
				u32 *out_elem_count)
{
	struct gsprm_rpc_header *hdr =
		(struct gsprm_rpc_header *)(buf + GSPRM_ELEM_HDR_SIZE);
	u32 rpc_len = sizeof(*hdr) + data_len;
	u32 msg_len = GSPRM_ELEM_HDR_SIZE + rpc_len;
	u32 aligned_len;
	u32 elem_count;
	u32 first_data;
	u32 remaining;
	u32 i;

	/* How many elements does this record span? */
	elem_count = (msg_len + GSPRM_QUEUE_ELEMENT_SIZE - 1) /
		     GSPRM_QUEUE_ELEMENT_SIZE;
	if (elem_count == 0)
		elem_count = 1;

	if (elem_count * GSPRM_QUEUE_ELEMENT_SIZE > buf_size) {
		pr_err("GSPRM: build_record overflow: need %u have %u\n",
		       elem_count * GSPRM_QUEUE_ELEMENT_SIZE, buf_size);
		return 0;
	}

	memset(buf, 0, elem_count * GSPRM_QUEUE_ELEMENT_SIZE);

	/* First element: header + rpc header + start of data */
	memcpy(buf + 0x24, &rm->stat_seq, 4);		/* seqNum */
	memcpy(buf + 0x28, &elem_count, 4);		/* elemCount */

	hdr->header_version = GSPRM_RPC_HEADER_VERSION;
	hdr->signature = GSPRM_RPC_SIGNATURE;
	hdr->length = rpc_len;
	hdr->function = function;
	hdr->rpc_result = result;
	hdr->rpc_result_private = result;
	hdr->sequence = sequence;
	hdr->spare = 0;

	/* Copy data across element boundaries */
	first_data = min(data_len, (u32)GSPRM_MAX_SINGLE_ELEM_DATA);
	if (data && first_data)
		memcpy((u8 *)(hdr + 1), data, first_data);

	remaining = data_len - first_data;
	if (remaining > 0 && data) {
		const u8 *src = (const u8 *)data + first_data;
		u8 *dst = buf + GSPRM_QUEUE_ELEMENT_SIZE;

		for (i = 1; i < elem_count && remaining > 0; i++) {
			u32 chunk = min(remaining,
					(u32)GSPRM_QUEUE_ELEMENT_SIZE);

			memcpy(dst, src, chunk);
			src += chunk;
			dst += GSPRM_QUEUE_ELEMENT_SIZE;
			remaining -= chunk;
		}
	}

	/* checksum covers msg_len zero-padded to 8 (buffer already zeroed) */
	aligned_len = (msg_len + 7) & ~7U;
	*(u32 *)(buf + 0x20) = gsp_rm_checksum32(buf, aligned_len);

	*out_elem_count = elem_count;
	return elem_count * GSPRM_QUEUE_ELEMENT_SIZE;
}

/*
 * Write a status queue record (1+ elements) and advance the writePtr.
 */
static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
		       u32 sequence, u32 result, const void *data, u32 data_len)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u8 *buf;
	u32 slot;
	u32 new_wp;
	u32 elem_count;
	u32 record_size;
	u32 buf_size;
	int ret;

	/* Allocate buffer for worst-case multi-element record */
	buf_size = GSPRM_QUEUE_ELEMENT_SIZE *
		   ((data_len + GSPRM_MAX_SINGLE_ELEM_DATA +
		     GSPRM_QUEUE_ELEMENT_SIZE - 1) /
		    GSPRM_QUEUE_ELEMENT_SIZE + 1);
	if (buf_size < GSPRM_QUEUE_ELEMENT_SIZE)
		buf_size = GSPRM_QUEUE_ELEMENT_SIZE;

	buf = kzalloc(buf_size, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	record_size = gsp_rm_build_record(rm, buf, buf_size, function,
					   sequence, result, data, data_len,
					   &elem_count);
	if (!record_size) {
		kfree(buf);
		return -EOVERFLOW;
	}

	/* Check we don't overflow the status queue */
	if (elem_count > rm->stat_msgcount) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: record too large: %u elements > queue %u",
			    elem_count, rm->stat_msgcount);
		kfree(buf);
		return -EOVERFLOW;
	}

	/* Write all elements to consecutive queue slots. Page-aware: the
	 * queue buffer is non-contiguous, so translate per page. */
	slot = rm->stat_write % rm->stat_msgcount;

	/* Handle wrap-around: write may span the end of the circular queue */
	{
		u32 elems_to_end = rm->stat_msgcount - slot;
		u32 first_chunk = min(elem_count, elems_to_end);
		u32 second_chunk = elem_count - first_chunk;
		u64 rec_off = rm->stat_off + rm->entry_off +
			      (u64)slot * GSPRM_QUEUE_ELEMENT_SIZE;

		ret = gsp_rm_write_buf(ctx, rec_off, buf,
				       first_chunk * GSPRM_QUEUE_ELEMENT_SIZE);
		if (ret) {
			nv_vgpu_err(ctx->vgpu, "GSPRM: status write failed off=0x%llx ret=%d",
				    rec_off, ret);
			kfree(buf);
			return ret;
		}

		if (second_chunk > 0) {
			ret = gsp_rm_write_buf(ctx, rm->stat_off + rm->entry_off,
					       buf + first_chunk * GSPRM_QUEUE_ELEMENT_SIZE,
					       second_chunk * GSPRM_QUEUE_ELEMENT_SIZE);
			if (ret) {
				nv_vgpu_err(ctx->vgpu, "GSPRM: status wrap write failed ret=%d",
					    ret);
				kfree(buf);
				return ret;
			}
		}
	}

	kfree(buf);

	/* Advance the write pointer with the same wrap semantics as the
	 * driver's msgq (slot index in [0, msgCount) - the peer's free-space
	 * and slot math breaks on unbounded values). stat_seq stays
	 * monotonic per-record. */
	rm->stat_write = (rm->stat_write + elem_count) % rm->stat_msgcount;
	rm->stat_seq++;

	/* bump status TX header writePtr (offset 0x10) */
	new_wp = rm->stat_write;
	ret = gsp_rm_write_buf(ctx, rm->stat_off + 0x10,
			       &new_wp, sizeof(new_wp));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: status writePtr update failed ret=%d", ret);
		return ret;
	}

	/* Notify the guest (MSI-X vector 0). The driver's bottom-half drains
	 * events under the GPU lock; the rpcRecvPoll poller holds the same
	 * lock from send to poll, so a reply can't be consumed by the event
	 * drain while a poller waits for it (same as real GSP-RM). */
	gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq);

	return 0;
}

/* ============================================================
 * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED)
 * ============================================================ */

static int gsp_rm_parse_queue_chain(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsprm_libos_region *regions;
	int ret, i;
	u64 gsp_args_gpa = 0;
	u32 total_size;

	regions = kmalloc_array(GSPRM_MAX_LIBOS_REGIONS, sizeof(*regions),
				GFP_KERNEL);
	if (!regions)
		return -ENOMEM;

	/* 1. Read the libos init args array */
	ret = gsp_rm_read(ctx, rm->libos_args_gpa, regions,
			  GSPRM_MAX_LIBOS_REGIONS * sizeof(*regions));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: libos args read failed gpa=0x%llx ret=%d",
			    rm->libos_args_gpa, ret);
		goto out;
	}

	/* 2. Find the RMARGS region -> GSP_ARGUMENTS_CACHED */
	for (i = 0; i < GSPRM_MAX_LIBOS_REGIONS; i++) {
		if (regions[i].id8 == 0)
			break;
		nv_vgpu_info(ctx->vgpu, "GSPRM: libos region[%d] id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u",
			     i, regions[i].id8, regions[i].pa, regions[i].size,
			     regions[i].kind, regions[i].loc);
		if (regions[i].id8 == GSPRM_LIBOS_ID_RMARGS)
			gsp_args_gpa = regions[i].pa;
	}
	if (!gsp_args_gpa) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: RMARGS region not found");
		ret = -ENOENT;
		goto out;
	}
	rm->gsp_args_gpa = gsp_args_gpa;

	/* 3. Read GSP_ARGUMENTS_CACHED (MESSAGE_QUEUE_INIT_ARGUMENTS first) */
	ret = gsp_rm_read(ctx, gsp_args_gpa, &rm->mq, sizeof(rm->mq));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_ARGUMENTS_CACHED read failed ret=%d", ret);
		goto out;
	}

	nv_vgpu_info(ctx->vgpu, "GSPRM: queue args sharedPA=0x%llx ptes=%u cmdOff=0x%llx statOff=0x%llx hdrSize=0x%llx elemMin=0x%llx elemMax=0x%llx hdrAlign=%u elemAlign=%u",
		     rm->mq.sharedMemPhysAddr, rm->mq.pageTableEntryCount,
		     rm->mq.cmdQueueOffset, rm->mq.statQueueOffset,
		     rm->mq.queueElementHdrSize, rm->mq.queueElementSizeMin,
		     rm->mq.queueElementSizeMax, rm->mq.queueHeaderAlign,
		     rm->mq.queueElementAlign);
	print_hex_dump(KERN_INFO, "GSPRM args raw: ", DUMP_PREFIX_OFFSET,
		       16, 8, &rm->mq, sizeof(rm->mq), false);

	if (!rm->mq.sharedMemPhysAddr || !rm->mq.pageTableEntryCount ||
	    rm->mq.pageTableEntryCount > 1024) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: invalid queue args");
		ret = -EINVAL;
		goto out;
	}

	/* 4. Read the page table (RmPhysAddr per 4K page of the shared buffer) */
	rm->ptes = kmalloc_array(rm->mq.pageTableEntryCount, sizeof(u64),
				 GFP_KERNEL);
	if (!rm->ptes) {
		ret = -ENOMEM;
		goto out;
	}
	ret = gsp_rm_read(ctx, rm->mq.sharedMemPhysAddr, rm->ptes,
			  rm->mq.pageTableEntryCount * sizeof(u64));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: page table read failed ret=%d", ret);
		goto out;
	}

	/* 5. Compute queue geometry. NOTE: the shared buffer is
	 * NV_MEMORY_NONCONTIGUOUS - only queue-offsets are linear, GPAs are
	 * not. Every access must translate through rm->ptes (gsp_rm_off_to_gpa). */
	rm->cmd_off = rm->mq.cmdQueueOffset;
	rm->stat_off = rm->mq.statQueueOffset;
	rm->cmd_gpa = gsp_rm_off_to_gpa(rm, rm->cmd_off);
	rm->stat_gpa = gsp_rm_off_to_gpa(rm, rm->stat_off);
	rm->cmd_size = (u32)(rm->mq.statQueueOffset - rm->mq.cmdQueueOffset);
	total_size = rm->mq.pageTableEntryCount << 12;
	rm->stat_size = total_size - (u32)rm->mq.statQueueOffset;
	rm->entry_off = 0x1000; /* ALIGN_UP(0x20 + 4, 1 << elemAlign(12)) */
	rm->cmd_msgcount = (rm->cmd_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE;
	rm->stat_msgcount = (rm->stat_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE;

	if (!rm->cmd_gpa || !rm->stat_gpa || !rm->cmd_msgcount || !rm->stat_msgcount) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: bad queue geometry cmd_gpa=0x%llx stat_gpa=0x%llx cmd_mc=%u stat_mc=%u",
			    rm->cmd_gpa, rm->stat_gpa, rm->cmd_msgcount,
			    rm->stat_msgcount);
		ret = -EINVAL;
		goto out;
	}

	nv_vgpu_info(ctx->vgpu, "GSPRM: cmd queue gpa=0x%llx size=0x%x mc=%u; status queue gpa=0x%llx size=0x%x mc=%u",
		     rm->cmd_gpa, rm->cmd_size, rm->cmd_msgcount,
		     rm->stat_gpa, rm->stat_size, rm->stat_msgcount);

	ret = 0;
out:
	kfree(regions);
	return ret;
}

/*
 * Write the status queue TX header (what GSP-RM's msgqTxCreate produces).
 */
static int gsp_rm_link_status_queue(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsprm_msgq_tx_header tx;
	u32 zero = 0;
	int ret;

	memset(&tx, 0, sizeof(tx));
	tx.version = GSPRM_MSGQ_VERSION;
	tx.size = rm->stat_size;
	tx.msgSize = GSPRM_QUEUE_ELEMENT_SIZE;
	tx.msgCount = rm->stat_msgcount;
	tx.writePtr = 0;
	tx.flags = GSPRM_MSGQ_FLAGS_SWAP_RX;
	tx.rxHdrOff = 0x20; /* ALIGN_UP(sizeof(msgqTxHeader)=32, 1<<hdrAlign(4)=16) */
	tx.entryOff = rm->entry_off;

	ret = gsp_rm_write_buf(ctx, rm->stat_off, &tx, sizeof(tx));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: status TX header write failed ret=%d", ret);
		return ret;
	}

	/* status queue RX header (our command-queue read position, swap mode) */
	ret = gsp_rm_write_buf(ctx, rm->stat_off + tx.rxHdrOff,
			       &zero, sizeof(zero));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: status RX header write failed ret=%d", ret);
		return ret;
	}

	nv_vgpu_info(ctx->vgpu, "GSPRM: status queue linked (size=0x%x msgCount=%u)",
		     tx.size, tx.msgCount);
	return 0;
}

/* ============================================================
 * Command processing
 * ============================================================ */

static const char *gsp_rm_rpc_name(u32 function)
{
	switch (function) {
	case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO:	return "SET_GUEST_SYSTEM_INFO";
	case GSPRM_RPC_FUNC_ALLOC_ROOT:			return "ALLOC_ROOT";
	case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO:	return "GET_GSP_STATIC_INFO";
	case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO:	return "GSP_SET_SYSTEM_INFO";
	case GSPRM_RPC_FUNC_SET_REGISTRY:		return "SET_REGISTRY";
	case GSPRM_RPC_FUNC_CONTINUATION_RECORD:	return "CONTINUATION_RECORD";
	case GSPRM_RPC_FUNC_GSP_RM_CONTROL:		return "GSP_RM_CONTROL";
	default:					return "UNKNOWN";
	}
}

/*
 * SET_GUEST_SYSTEM_INFO: the driver negotiates the RPC interface version
 * from the reply's vgxVersionMajorNum/MinorNum fields. Without them the
 * version-gated HALs stay stubs and every subsequent RPC fails with
 * NV_VGPU_MSG_RESULT_RPC_UNKNOWN_FUNCTION (0xff100001). Echo the driver's
 * own version back (no host-guest version skew).
 */
static void gsp_rm_handle_set_guest_system_info(struct gsp_emu_context *ctx,
						const u8 *elem,
						u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE +
			     sizeof(struct gsprm_rpc_header);
	u8 *reply;
	u32 major, minor;

	memcpy(&major, req_data + 0, 4);
	memcpy(&minor, req_data + 4, 4);
	if (!major)
		major = GSPRM_VGX_VERSION_MAJOR;
	if (!minor && major != GSPRM_VGX_VERSION_MAJOR)
		minor = GSPRM_VGX_VERSION_MINOR;

	reply = kzalloc(GSPRM_SET_GUEST_INFO_SIZE, GFP_KERNEL);
	if (!reply) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply alloc failed");
		return;
	}
	memcpy(reply + 0, &major, 4);
	memcpy(reply + 4, &minor, 4);

	nv_vgpu_info(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO seq=%u -> VGX %u.%u",
		     sequence, major, minor);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO, sequence, 0,
			reply, GSPRM_SET_GUEST_INFO_SIZE))
		nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply send failed");
	else
		rm->rpcs_replied++;

	kfree(reply);
}

/* ============================================================
 * GET_GSP_STATIC_INFO (function 65)
 *
 * Builds the 1656-byte GspStaticConfigInfo for TU104.
 * ============================================================ */

static void gsp_rm_build_static_info(u8 *buf)
{
	u64 val64;
	u32 val32;
	u8 *p;

	memset(buf, 0, TU104_GSCI_SIZE);

	/* grCapsBits (offset 0, 23 bytes) - basic GR caps */
	buf[0] = 0x01;	/* GR present */

	/* gidInfo (offset 24, 268 bytes): index(4) + flags(4) + length(4) + data(256) */
	p = buf + TU104_GSCI_OFF_gidInfo;
	val32 = 0;		memcpy(p + 0, &val32, 4);	/* index */
	val32 = 0x80000000;	memcpy(p + 4, &val32, 4);	/* flags: valid */
	val32 = 16;		memcpy(p + 8, &val32, 4);	/* length = 16 (UUID) */
	/* GPU UUID - deterministic fake for TU104 */
	memcpy(p + 12, "GPU-9da5e221-18", 15);

	/* SKUInfo (offset 292, 48 bytes) */
	p = buf + TU104_GSCI_OFF_SKUInfo;
	memcpy(p + 4, "TU104", 5);	/* chipSKU */
	memcpy(p + 13, "400", 3);	/* chipSKUMod */
	memcpy(p + 20, "N18E", 4);	/* project */
	memcpy(p + 24, "G3", 2);	/* projectSKU */

	/* fbRegionInfoParams (offset 344, 776 bytes) */
	p = buf + TU104_GSCI_OFF_fbRegionInfoParams;
	val32 = 1;		memcpy(p, &val32, 4);		/* numFBRegions = 1 */
	/* Region 0: base=0, limit=256MB-1, reserved=0 */
	{
		struct tu104_fb_region *region = (void *)(p + 8); /* 8-aligned */
		val64 = 0;		memcpy(&region->base, &val64, 8);
		val64 = 0x0FFFFFFF;	memcpy(&region->limit, &val64, 8);
		val64 = 0;		memcpy(&region->reserved, &val64, 8);
		val32 = 0;		memcpy(&region->performance, &val32, 4);
		region->supportCompressed = 0;
		/* memmgrCalcReservedFbSpace asserts that SOME region supports
		 * ISO "even if we have no display" - without it the reserved
		 * space split fails and kdisp can't allocate its channel
		 * table. A single ISO-capable region satisfies fast/slow/iso
		 * assignment. */
		region->supportISO = 1;
		region->bProtected = 0;
		memset(region->blackList, 0, sizeof(region->blackList));
	}

	/* sriovCaps (offset 1120, 80 bytes) - all zeros (not SR-IOV) */

	/* sriovMaxGfid (offset 1200, 4 bytes) */
	val32 = 0;
	memcpy(buf + TU104_GSCI_OFF_sriovMaxGfid, &val32, 4);

	/* engineCaps (offset 1204, 12 bytes) */
	val32 = 0x60003E02;	/* GR0|CE0-4|NVDEC0|NVDEC1 */
	memcpy(buf + TU104_GSCI_OFF_engineCaps, &val32, 4);
	val32 = 0x00362020;	/* NVENC0|SW|SEC2|HOST|PMU|FBFLCN */
	memcpy(buf + TU104_GSCI_OFF_engineCaps + 4, &val32, 4);
	val32 = 0;
	memcpy(buf + TU104_GSCI_OFF_engineCaps + 8, &val32, 4);

	/* poisonFuseEnabled (offset 1216, 1 byte) */
	buf[TU104_GSCI_OFF_poisonFuseEnabled] = 0;

	/* fb_length (offset 1224, 8 bytes) - 256MB (512MB broke the FWSEC
	 * WPR2-location check below). */
	val64 = 0x10000000ULL;
	memcpy(buf + TU104_GSCI_OFF_fb_length, &val64, 8);

	/* fbio_mask (offset 1232, 8 bytes) - 8 FBIOs for 256-bit bus */
	val64 = 0xFF;
	memcpy(buf + TU104_GSCI_OFF_fbio_mask, &val64, 8);

	/* fb_bus_width (offset 1240, 4 bytes) - 256-bit */
	val32 = 256;
	memcpy(buf + TU104_GSCI_OFF_fb_bus_width, &val32, 4);

	/* fb_ram_type (offset 1244, 4 bytes) - GDDR6 = 6 */
	val32 = 6;
	memcpy(buf + TU104_GSCI_OFF_fb_ram_type, &val32, 4);

	/* fbp_mask (offset 1248, 8 bytes) - 6 FBPs for 6 GPCs */
	val64 = 0x3F;
	memcpy(buf + TU104_GSCI_OFF_fbp_mask, &val64, 8);

	/* l2_cache_size (offset 1256, 4 bytes) - 4MB */
	val32 = 0x400000;
	memcpy(buf + TU104_GSCI_OFF_l2_cache_size, &val32, 4);

	/* gpuNameString (offset 1260, 64 bytes) */
	strncpy((char *)(buf + TU104_GSCI_OFF_gpuNameString),
		"NVIDIA GeForce RTX 2080 SUPER", 64);

	/* gpuShortNameString (offset 1324, 64 bytes) */
	strncpy((char *)(buf + TU104_GSCI_OFF_gpuShortNameString),
		"TU104", 64);

	/* gpuNameString_Unicode (offset 1388, 128 bytes) - skip for now */

	/* Branding booleans */
	buf[TU104_GSCI_OFF_bIsMobile] = 1;	/* Mobile/Max-Q */

	/* bar1PdeBase (offset 1536, 8 bytes) */
	val64 = 0;
	memcpy(buf + TU104_GSCI_OFF_bar1PdeBase, &val64, 8);

	/* bar2PdeBase (offset 1544, 8 bytes) */
	val64 = 0;
	memcpy(buf + TU104_GSCI_OFF_bar2PdeBase, &val64, 8);

	/* bVbiosValid (offset 1552, 1 byte) */
	buf[TU104_GSCI_OFF_bVbiosValid] = 1;

	/* vbiosSubVendor (offset 1556, 4 bytes) - ASUS */
	val32 = 0x1043;
	memcpy(buf + TU104_GSCI_OFF_vbiosSubVendor, &val32, 4);

	/* vbiosSubDevice (offset 1560, 4 bytes) */
	val32 = 0x1F01;
	memcpy(buf + TU104_GSCI_OFF_vbiosSubDevice, &val32, 4);

	/* hInternalClient/Device/Subdevice */
	val32 = TU104_H_INTERNAL_CLIENT;
	memcpy(buf + TU104_GSCI_OFF_hInternalClient, &val32, 4);
	val32 = TU104_H_INTERNAL_DEVICE;
	memcpy(buf + TU104_GSCI_OFF_hInternalDevice, &val32, 4);
	val32 = TU104_H_INTERNAL_SUBDEVICE;
	memcpy(buf + TU104_GSCI_OFF_hInternalSubdevice, &val32, 4);

	/* ecidInfo (offset 1616, 24 bytes) - leave zeroed */

	/* fwWprLayoutOffset (offset 1640, 16 bytes): nonWprHeapOffset, frtsOffset */
	val64 = 0;
	memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset, &val64, 8);
	val64 = 0xFE00000;	/* frtsOffset = fbSize - 2MB (256MB) */
	memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8);
}

static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx,
					   u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u8 *reply;

	reply = kzalloc(TU104_GSCI_SIZE, GFP_KERNEL);
	if (!reply) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply alloc failed");
		return;
	}

	gsp_rm_build_static_info(reply);

	nv_vgpu_info(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)",
		     sequence, TU104_GSCI_SIZE,
		     0x10000000ULL >> 20, 0x60003E02, 0x362020);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO, sequence, 0,
			reply, TU104_GSCI_SIZE))
		nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply send failed");
	else
		rm->rpcs_replied++;

	kfree(reply);
}

/* ============================================================
 * GSP_RM_CONTROL (function 76) - sub-command dispatch
 *
 * rpc_gsp_rm_control_v03_00: hClient(4) + hObject(4) + cmd(4) + status(4)
 *                            + paramsSize(4) + flags(4) + params[]
 * ============================================================ */

struct gsp_rm_control_req {
	u32 hClient;
	u32 hObject;
	u32 cmd;
	u32 status;
	u32 paramsSize;
	u32 flags;
	/* params[] follows */
};

#define GSP_RM_CTRL_HDR_SIZE	24

static void gsp_rm_handle_fifo_device_table(struct gsp_emu_context *ctx,
					     u32 sequence,
					     struct gsp_rm_control_req *req,
					     const u8 *req_params,
					     u32 *out_status,
					     u8 *out_params, u32 *out_params_size)
{
	struct tu104_fifo_get_device_info_params *fifo_params;
	u32 base, count, i, copy_count;

	fifo_params = (struct tu104_fifo_get_device_info_params *)out_params;
	base = 0;
	/* baseIndex is the first field of the request params, which live in
	 * the queue element buffer (NOT after the local header struct!). Only
	 * readable when the whole request fits in the first element. */
	if (req->paramsSize >= 4 &&
	    req->paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA)
		memcpy(&base, req_params, 4);
	if (base >= TU104_FIFO_TABLE_MAX_DEVICES)
		base = 0;

	/* How many entries to return in this page */
	count = TU104_ENGINE_TABLE_SIZE - base;
	if (count > TU104_FIFO_TABLE_MAX_ENTRIES)
		count = TU104_FIFO_TABLE_MAX_ENTRIES;

	copy_count = count;
	if (base >= TU104_ENGINE_TABLE_SIZE)
		copy_count = 0;

	fifo_params->baseIndex = base;
	fifo_params->numEntries = copy_count;
	fifo_params->bMore = (base + copy_count) < TU104_ENGINE_TABLE_SIZE;

	for (i = 0; i < copy_count; i++)
		memcpy(&fifo_params->entries[i], &tu104_engine_table[base + i],
		       sizeof(struct tu104_fifo_device_entry));

	*out_params_size = offsetof(struct tu104_fifo_get_device_info_params,
				     entries) +
			   copy_count * sizeof(struct tu104_fifo_device_entry);
	*out_status = 0; /* NV_OK */

	nv_vgpu_info(ctx->vgpu, "GSPRM: FIFO_GET_DEVICE_INFO_TABLE base=%u count=%u more=%u",
		     base, copy_count, fifo_params->bMore);
}

static void gsp_rm_handle_intr_kernel_table(struct gsp_emu_context *ctx,
					     u32 sequence,
					     u32 *out_status,
					     u8 *out_params, u32 *out_params_size)
{
	struct tu104_intr_get_kernel_table_params *intr_params;
	u32 copy_size;

	intr_params = (struct tu104_intr_get_kernel_table_params *)out_params;

	intr_params->tableLen = TU104_INTR_KERNEL_TABLE_SIZE;
	copy_size = TU104_INTR_KERNEL_TABLE_SIZE *
		    sizeof(struct tu104_intr_table_entry);
	memcpy(intr_params->table, tu104_intr_kernel_table, copy_size);

	/* Subtree map - must match intrInitSubtreeMap_TU102 (the values
	 * physical-path guests receive from real GSP-RM):
	 *   ESCHED_DRIVEN_ENGINE            = {3,3} (STALL_SUBTREE)
	 *   ESCHED_DRIVEN_ENGINE_NOTIF      = {0,0}
	 *   UVM_OWNED                       = {1,1} (fault 64, access cntr 72)
	 *   UVM_SHARED                      = {2,2} (doorbells/GSP/DISP)
	 *   DEFAULT/RUNLIST/RUNLIST_NOTIF   = INVALID (unset on Turing)
	 * intrCacheIntrFields requires accessCntrSubtree(=1) within
	 * UVM_OWNED and UVM_SHARED to be a single subtree. */
	{
		u32 c;

		for (c = 0; c < TU104_INTR_CATEGORY_ENUM_COUNT; c++) {
			intr_params->subtreeMap[c].subtreeStart = TU104_INTR_INVALID_SUBTREE;
			intr_params->subtreeMap[c].subtreeEnd   = TU104_INTR_INVALID_SUBTREE;
		}
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeStart = 3;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeEnd   = 3;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeStart = 0;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeEnd   = 0;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeStart = 1;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeEnd   = 1;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeStart = 2;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeEnd   = 2;
	}

	/* The driver reads subtreeMap at offset 2052 (after the full
	 * 128-entry array), so the reply must span the whole struct -
	 * not just the used entries. */
	*out_params_size = offsetof(struct tu104_intr_get_kernel_table_params,
				     subtreeMap) +
			   sizeof(intr_params->subtreeMap);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: INTR_GET_KERNEL_TABLE tableLen=%u",
		     TU104_INTR_KERNEL_TABLE_SIZE);
}

static void gsp_rm_handle_constructed_falcon_info(struct gsp_emu_context *ctx,
						   u32 sequence,
						   u32 *out_status,
						   u8 *out_params,
						   u32 *out_params_size)
{
	struct tu104_get_constructed_falcon_info_params *falcon_params;

	falcon_params = (struct tu104_get_constructed_falcon_info_params *)out_params;

	falcon_params->numConstructedFalcons = TU104_FALCON_TABLE_SIZE;
	memcpy(falcon_params->constructedFalconsTable, tu104_falcon_table,
	       TU104_FALCON_TABLE_SIZE *
	       sizeof(struct tu104_constructed_falcon_info));

	*out_params_size = offsetof(struct tu104_get_constructed_falcon_info_params,
				     constructedFalconsTable) +
			   TU104_FALCON_TABLE_SIZE *
			   sizeof(struct tu104_constructed_falcon_info);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: GET_CONSTRUCTED_FALCON_INFO numFalcons=%u",
		     TU104_FALCON_TABLE_SIZE);
}

static void gsp_rm_handle_internal_device_table(struct gsp_emu_context *ctx,
						 u32 sequence,
						 u32 *out_status,
						 u8 *out_params,
						 u32 *out_params_size)
{
	struct tu104_internal_get_device_info_params *dev_params;
	u32 copy_size;

	dev_params = (struct tu104_internal_get_device_info_params *)out_params;

	dev_params->numEntries = TU104_INTERNAL_DEVICE_TABLE_SIZE;
	copy_size = TU104_INTERNAL_DEVICE_TABLE_SIZE *
		    sizeof(struct tu104_internal_device_info);
	memcpy(dev_params->deviceInfoTable, tu104_internal_device_table,
	       copy_size);

	*out_params_size = offsetof(struct tu104_internal_get_device_info_params,
				     deviceInfoTable) + copy_size;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: INTERNAL_GET_DEVICE_INFO_TABLE numEntries=%u",
		     TU104_INTERNAL_DEVICE_TABLE_SIZE);
}

/*
 * INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP (0x20800a41)
 *
 * Returns a compressed register access map telling the guest which BAR0
 * registers are userspace-accessible. For our emulated GPU, return an
 * empty map (no compressed data, no profiling ranges).
 */
static void gsp_rm_handle_user_register_access_map(struct gsp_emu_context *ctx,
						    u32 sequence,
						    u32 *out_status,
						    u8 *out_params,
						    u32 *out_params_size)
{
	/* NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS:
	 *   userRegisterAccessMapSize(4) + compressedSize(4) +
	 *   compressedData[4096] + profilingRangesSize(4) + profilingRanges[4096]
	 * Total: 8204 bytes
	 */
	u32 val32 = 0;

	memset(out_params, 0, 8204);
	memcpy(out_params + 0, &val32, 4);		/* userRegisterAccessMapSize */
	memcpy(out_params + 4, &val32, 4);		/* compressedSize */
	/* compressedData[4096] = zeros */
	memcpy(out_params + 4104, &val32, 4);		/* profilingRangesSize */
	/* profilingRanges[4096] = zeros */

	*out_params_size = 8204;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: USER_REGISTER_ACCESS_MAP -> empty map");
}

/*
 * INTERNAL_FIFO_GET_NUM_CHANNELS (0x20800a61)
 *
 * The guest asks GSP-RM how many channels each runlist supports
 * (kfifoRunlistQueryNumChannels). Returning 0 trips the driver's
 * NV_ASSERT(numChannels > 0) and kdisp can't allocate its
 * clientChannelTable. Report the consumer-Turing standard 64.
 */
static void gsp_rm_handle_fifo_get_num_channels(struct gsp_emu_context *ctx,
						u32 sequence,
						u32 *out_status,
						u8 *out_params,
						u32 *out_params_size)
{
	u32 val32;

	/* params: runlistId(4, in) + numChannels(4, out) */
	val32 = TU104_FIFO_NUM_CHANNELS_PER_RUNLIST;
	memcpy(out_params + 4, &val32, 4);

	*out_params_size = 8;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: FIFO_GET_NUM_CHANNELS -> %u", val32);
}

/*
 * BUS_GET_PCI_BAR_INFO (0x20801803)
 *
 * The guest caches pciBarSizes[] from this call (kern_bus.c). Reporting
 * zeros leaves BAR1/BAR2 aperture lengths at 0. Report our emulated BAR
 * layout; index 2 is the instance-memory window convention
 * (BAR0+0x700000, 1MB - gpuPhysInstAddr fallback when PCI BAR2 is absent).
 */
static void gsp_rm_handle_bus_get_pci_bar_info(struct gsp_emu_context *ctx,
					       u32 sequence,
					       u32 *out_status,
					       u8 *out_params,
					       u32 *out_params_size)
{
	struct nv_vgpu_device *vgpu = ctx->vgpu;
	u32 count = 4;
	u32 i;
	u64 sizes[4];
	u64 offsets[4];

	sizes[0] = vgpu->bars[0].size;   offsets[0] = vgpu->bars[0].gpa;
	sizes[1] = vgpu->bars[1].size;   offsets[1] = vgpu->bars[1].gpa;
	sizes[2] = 0x100000;             offsets[2] = vgpu->bars[0].gpa + 0x700000;
	sizes[3] = vgpu->bars[3].size;   offsets[3] = vgpu->bars[3].gpa;

	memset(out_params, 0, 8 + 8 * 24);
	memcpy(out_params + 0, &count, 4);

	for (i = 0; i < count; i++) {
		u8 *e = out_params + 8 + i * 24;
		u32 flags = 0;
		u32 barSizeMB = (u32)(sizes[i] >> 20);

		memcpy(e + 0, &flags, 4);
		memcpy(e + 4, &barSizeMB, 4);
		memcpy(e + 8, &sizes[i], 8);
		memcpy(e + 16, &offsets[i], 8);
	}

	*out_params_size = 8 + count * 24;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: BUS_GET_PCI_BAR_INFO -> %u BARs (BAR1=%uMB)",
		     count, (u32)(sizes[1] >> 20));
}

/*
 * INTERNAL_DISPLAY_GET_STATIC_INFO (0x20800a01)
 *
 * kdispStateInitLocked caches numHeads/numDispChannels from this call and
 * allocates clientChannelTable[numDispChannels] - a zero reply makes the
 * allocation fail and kills RmInitNvDevice. The driver's wire struct is
 * 36 bytes (observed paramsSize): feHwSysCap(0) windowPresentMask(4)
 * bFbRemapperEnabled(8) numHeads(12) i2cPort(16) internalDispActiveMask(20)
 * embeddedDisplayPortMask(24) muxSupported(28) numDispChannels(32).
 * Report a minimal headless display: 1 head, 1 (core) channel.
 */
static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx,
						  u32 sequence,
						  u32 *out_status,
						  u8 *out_params,
						  u32 *out_params_size)
{
	u32 val32;

	memset(out_params, 0, 36);

	val32 = 1;	/* numHeads */
	memcpy(out_params + 12, &val32, 4);
	val32 = 1;	/* numDispChannels (core channel) */
	memcpy(out_params + 32, &val32, 4);

	*out_params_size = 36;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=1");
}

/*
 * INTERNAL_DISPLAY_GET_IP_VERSION (0x20800a4b)
 *
 * kdispStatePreInitLocked passes an UNINITIALIZED ctrlParams and feeds the
 * reply straight into gpuInitDispIpHal - it must be a real IP version.
 * Turing display is DISP v04.00 (DispIpHal halspec idx 10).
 */
static void gsp_rm_handle_display_get_ip_version(struct gsp_emu_context *ctx,
						 u32 sequence,
						 u32 *out_status,
						 u8 *out_params,
						 u32 *out_params_size)
{
	u32 val32 = TU104_DISPLAY_IP_VERSION;

	memcpy(out_params, &val32, 4);
	*out_params_size = 4;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: DISPLAY_GET_IP_VERSION -> 0x%x", val32);
}

/*
 * CE_GET_FAULT_METHOD_BUFFER_SIZE (0x20802a08)
 *
 * kchangrpInit allocates one fault method buffer per runqueue of this
 * size; a zero reply makes the allocation fail (NV_ERR_NO_MEMORY) and
 * kills channel-group creation. GV100 formula inherited by Turing:
 * 135 * (13*totalCes + 2) page-aligned; TU104 = 5 PCEs -> 12288.
 */
static void gsp_rm_handle_ce_get_fault_method_buffer_size(
						struct gsp_emu_context *ctx,
						u32 sequence,
						u32 *out_status,
						u8 *out_params,
						u32 *out_params_size)
{
	u32 val32 = TU104_CE_FAULT_METHOD_BUFFER_SIZE;

	memcpy(out_params, &val32, 4);
	*out_params_size = 4;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_FAULT_METHOD_BUFFER_SIZE -> %u", val32);
}

/*
 * STATIC_KGR_GET_GLOBAL_SM_ORDER (0x20800a22)
 *
 * kgrStaticInit copies globalSmOrder[grIdx] into the GR engine's static
 * info - init-critical (NV_CHECK_OK_OR_GOTO, cannot fail). Zeros leave
 * numSm=0 and the driver/nvidia-smi spin on empty topology. Report the
 * TU104 layout: 6 GPCs x 4 TPCs x 2 SMs = 48 SMs on GR0, engines 1-7 empty.
 * params: globalSmOrder[8] x (globalSmId[240]x12B + numSm(2) + numTpc(2)).
 */
static void gsp_rm_handle_kgr_get_global_sm_order(struct gsp_emu_context *ctx,
						  u32 sequence,
						  u32 *out_status,
						  u8 *out_params,
						  u32 *out_params_size)
{
	struct tu104_gr_global_sm_order *order =
		(struct tu104_gr_global_sm_order *)out_params; /* engine 0 */
	u32 idx = 0;
	u32 gpc, tpc, sm;

	memset(out_params, 0, TU104_GR_MAX_ENGINES *
	       sizeof(struct tu104_gr_global_sm_order));

	for (gpc = 0; gpc < TU104_NUM_GPC; gpc++) {
		for (tpc = 0; tpc < TU104_TPC_PER_GPC; tpc++) {
			for (sm = 0; sm < TU104_SM_PER_TPC; sm++) {
				struct tu104_gr_sm_id *e = &order->globalSmId[idx++];

				e->gpcId = gpc;
				e->localTpcId = tpc;
				e->localSmId = sm;
				e->globalTpcId = gpc * TU104_TPC_PER_GPC + tpc;
				e->virtualGpcId = gpc;
				e->migratableTpcId = e->globalTpcId;
			}
		}
	}
	order->numSm = TU104_NUM_SM;
	order->numTpc = TU104_NUM_TPC;

	*out_params_size = TU104_GR_MAX_ENGINES *
			   sizeof(struct tu104_gr_global_sm_order);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_GLOBAL_SM_ORDER -> %u SMs, %u TPCs on GR0",
		     TU104_NUM_SM, TU104_NUM_TPC);
}

/*
 * STATIC_KGR_GET_INFO (0x20800a2a)
 *
 * Returns the GR infoList the driver caches as pGrInfo and reads by index -
 * critically MAX_SUBCONTEXT_COUNT (0x2c): zero makes the channel group's
 * numMax==0 and channel allocation asserts (numMax == numFree && numMax != 0
 * -> NV_ERR_INVALID_STATE -> "No devices"). Fill TU104 litter topology +
 * subcontext counts (MAX_VEID_COUNT_INIT = 64). engineInfo[8] x infoList[58]
 * of {index,data} = 3712 bytes; engine 0 = GR0.
 */
static void gsp_rm_handle_kgr_get_info(struct gsp_emu_context *ctx,
				       u32 sequence,
				       u32 *out_status,
				       u8 *out_params,
				       u32 *out_params_size)
{
	struct tu104_gr_static_info *info =
		(struct tu104_gr_static_info *)out_params; /* engine 0 */
	u32 i;

	memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES *
	       sizeof(struct tu104_gr_static_info));

	/* positional array: index field = position for every entry */
	for (i = 0; i < TU104_GR_INFO_MAX_SIZE; i++)
		info->infoList[i].index = i;

	/* TU104 litter topology: 6 GPC x 4 TPC x 2 SM = 48 SMs */
	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_GPCS].data = TU104_NUM_GPC;
	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_TPC_PER_GPC].data = TU104_TPC_PER_GPC;
	info->infoList[TU104_GR_INFO_IDX_GPU_CORE_COUNT].data = TU104_NUM_SM;
	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_SM_PER_TPC].data = TU104_SM_PER_TPC;
	info->infoList[TU104_GR_INFO_IDX_FAMILY_MAX_TPC_PER_GPC].data = TU104_TPC_PER_GPC;

	/* subcontext (VEID) counts: 64 per hw */
	info->infoList[TU104_GR_INFO_IDX_MAX_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;
	info->infoList[TU104_GR_INFO_IDX_MAX_LEGACY_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;
	info->infoList[TU104_GR_INFO_IDX_MAX_PER_ENGINE_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;

	*out_params_size = TU104_GR_INFO_MAX_ENGINES *
			   sizeof(struct tu104_gr_static_info);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_INFO -> GR0 litter %uGPC/%uTPC/%uSM, maxSubctx=%u",
		     TU104_NUM_GPC, TU104_TPC_PER_GPC, TU104_NUM_SM,
		     TU104_MAX_SUBCONTEXT_COUNT);
}

/*
 * STATIC_KGR_GET_FLOORSPEEPING_MASKS (0x20800a26)
 *
 * Driver computes numGpcs from floorsweepingMasks.gpcMask; zero kills
 * krcWatchdogInit ("numGpcs > 0" assert -> RmInitAdapter bails). Fill
 * TU104's present-GPC/TPC masks: 6 GPCs (bits 0-5), 4 TPCs per GPC.
 */
static void gsp_rm_handle_kgr_get_floorsweeping_masks(
						struct gsp_emu_context *ctx,
						u32 sequence,
						u32 *out_status,
						u8 *out_params,
						u32 *out_params_size)
{
	struct tu104_gr_floorsweeping_masks *m =
		(struct tu104_gr_floorsweeping_masks *)out_params; /* engine 0 */
	u32 g;

	memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES *
	       sizeof(struct tu104_gr_floorsweeping_masks));

	m->gpcMask = (1U << TU104_NUM_GPC) - 1;		/* bits 0-5 = 6 GPCs */
	m->physGpcMask = m->gpcMask;
	m->physGfxGpcMask = m->gpcMask;
	m->numGfxTpc = TU104_NUM_TPC;
	for (g = 0; g < TU104_NUM_GPC; g++) {
		m->tpcMask[g] = (1U << TU104_TPC_PER_GPC) - 1;	/* 4 TPCs */
		m->tpcCount[g] = TU104_TPC_PER_GPC;
		m->mmuPerGpc[g] = 1;
		m->numPesPerGpc[g] = 1;
		m->zcullMask[g] = (1U << TU104_TPC_PER_GPC) - 1;
	}

	*out_params_size = TU104_GR_INFO_MAX_ENGINES *
			   sizeof(struct tu104_gr_floorsweeping_masks);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_FLOORSWEEPING_MASKS -> gpcMask=0x%x tpc/gpc=%u",
		     m->gpcMask, TU104_TPC_PER_GPC);
}

static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
				     const u8 *elem, u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE +
			     sizeof(struct gsprm_rpc_header);
	struct gsp_rm_control_req req;
	u32 reply_data_len;
	u8 *reply;
	u32 out_status = 0;
	u32 out_params_size = 0;

	memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE);

	nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_RM_CONTROL cmd=0x%x hClient=0x%x hObject=0x%x paramsSize=%u flags=0x%x",
		     req.cmd, req.hClient, req.hObject, req.paramsSize,
		     req.flags);

	/* Allocate reply: control header + max possible params */
	reply_data_len = GSP_RM_CTRL_HDR_SIZE + max(req.paramsSize, (u32)4096);
	reply = kzalloc(reply_data_len, GFP_KERNEL);
	if (!reply) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply alloc failed");
		return;
	}

	switch (req.cmd) {
	case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE:
		gsp_rm_handle_fifo_device_table(ctx, sequence, &req,
			req_data + GSP_RM_CTRL_HDR_SIZE,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE:
		gsp_rm_handle_intr_kernel_table(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO:
		gsp_rm_handle_constructed_falcon_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE:
		gsp_rm_handle_internal_device_table(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case 0x20800a41: /* INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP */
		gsp_rm_handle_user_register_access_map(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS:
		gsp_rm_handle_fifo_get_num_channels(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_BUS_GET_PCI_BAR_INFO:
		gsp_rm_handle_bus_get_pci_bar_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO:
		gsp_rm_handle_display_get_static_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION:
		gsp_rm_handle_display_get_ip_version(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE:
		gsp_rm_handle_ce_get_fault_method_buffer_size(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_STATIC_KGR_GET_GLOBAL_SM_ORDER:
		gsp_rm_handle_kgr_get_global_sm_order(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_STATIC_KGR_GET_INFO:
		gsp_rm_handle_kgr_get_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS:
		gsp_rm_handle_kgr_get_floorsweeping_masks(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	default:
		/* Unknown control command - return success with zeroed params.
		 * Do NOT echo the request: pure queries pass uninitialized
		 * stack structs as input (e.g. DISPLAY_GET_IP_VERSION), so an
		 * echo hands the driver its own stack garbage as the answer. */
		nv_vgpu_info(ctx->vgpu, "GSPRM: RM_CONTROL unknown cmd=0x%x -> generic SUCCESS",
			     req.cmd);
		out_status = 0;
		out_params_size = req.paramsSize;
		break;

	case TU104_CTRL_I2C_TRANSACTION:
		/* I2C transactions have variable-length FINN-serialized
		 * params: an all-zero reply fails the driver's DeserializeUp
		 * and the calling ioctl never completes (nvidia-smi hang).
		 * Echo the request's serialized params (valid stream) and
		 * report NOT_SUPPORTED - headless vGPU has no DDC/I2C bus. */
		nv_vgpu_info(ctx->vgpu, "GSPRM: I2C_TRANSACTION -> NOT_SUPPORTED (echo)");
		out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */
		out_params_size = req.paramsSize;
		if (req.paramsSize &&
		    req.paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA)
			memcpy(reply + GSP_RM_CTRL_HDR_SIZE,
			       req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize);
		break;
	}

	/* Build the reply header */
	memcpy(reply + 0, &req.hClient, 4);
	memcpy(reply + 4, &req.hObject, 4);
	memcpy(reply + 8, &req.cmd, 4);
	memcpy(reply + 12, &out_status, 4);
	memcpy(reply + 16, &out_params_size, 4);
	memcpy(reply + 20, &req.flags, 4);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0,
			reply, GSP_RM_CTRL_HDR_SIZE + out_params_size))
		nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply send failed cmd=0x%x",
			    req.cmd);
	else
		rm->rpcs_replied++;

	kfree(reply);
}

/*
 * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK.
 *
 * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the
 * request's params back with status=0 so the driver's free rpc_message is a
 * complete well-formed record. (The poll match is function-only; the echo
 * keeps the post-poll result struct self-consistent.)
 */
/*
			   sizeof(struct tu104_gr_floorsweeping_masks);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_FLOORSWEEPING_MASKS -> gpcMask=0x%x tpc/gpc=%u",
		     m->gpcMask, TU104_TPC_PER_GPC);
}

/*
 * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK.
 *
 * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the
 * request's params back with status=0 so the driver's free rpc_message is a
 * complete well-formed record. (The poll match is function-only; the echo
 * keeps the post-poll result struct self-consistent.)
 */
static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
			       u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const struct gsprm_rpc_header *hdr =
		(const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
	const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr);
	u8 params[16]; /* NVOS00_PARAMETERS: hRoot, hObjectParent, hObjectOld, status */
	u32 hObjOld = 0;

	if (hdr->length >= sizeof(*hdr) + sizeof(params))
		memcpy(params, req, sizeof(params));
	else
		memset(params, 0, sizeof(params));

	memcpy(&hObjOld, params + 8, 4);
	memset(params + 12, 0, 4); /* status = NV_OK */

	nv_vgpu_info(ctx->vgpu, "GSPRM: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u",
		     hObjOld, sequence, rm->cmd_read, rm->stat_write);

	/* Small delay before FREE/UNLOADING replies: mitigates an intermittent
	 * driver-side reply-matching race in the cleanup path. */
	udelay(2000);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_FREE, sequence, 0,
			params, sizeof(params)))
		nv_vgpu_err(ctx->vgpu, "GSPRM: FREE reply send failed");
	else
		rm->rpcs_replied++;
}

/*
 * Handle one command-queue record (first element). For async init RPCs we
 * just consume. Synchronous RPCs get a reply; per-function handlers are
 * implemented iteratively (unknown ones get a generic success reply).
 */
static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const struct gsprm_rpc_header *hdr =
		(const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
	u32 function = hdr->function;
	u32 sequence = hdr->sequence;

	rm->rpcs_seen++;

	switch (function) {
	case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO:
	case GSPRM_RPC_FUNC_SET_REGISTRY:
	case GSPRM_RPC_FUNC_CONTINUATION_RECORD:
		/* Async init RPCs - fire and forget, just consume. */
		nv_vgpu_info(ctx->vgpu, "GSPRM: async RPC %s(%u) seq=%u len=%u consumed",
			     gsp_rm_rpc_name(function), function, sequence,
			     hdr->length);
		return;

	case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO:
		gsp_rm_handle_set_guest_system_info(ctx, elem, sequence);
		return;

	case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO:
		gsp_rm_handle_get_static_info(ctx, sequence);
		return;

	case GSPRM_RPC_FUNC_GSP_RM_CONTROL:
		gsp_rm_handle_rm_control(ctx, elem, sequence);
		return;

	case GSPRM_RPC_FUNC_FREE:
		gsp_rm_handle_free(ctx, elem, sequence);
		return;

	default:
		/* Synchronous RPC: echo function+sequence with success so the
		 * driver's rpcRecvPoll completes. */
		rm->unknown_rpcs++;
		if (function == 47) {
			/* UNLOADING_GUEST_DRIVER: log for unload-cycle debugging */
			nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u cmd_read=%u stat_write=%u stat_seq=%u",
				     gsp_rm_rpc_name(function), function, sequence,
				     rm->cmd_read, rm->stat_write, rm->stat_seq);
		} else {
			nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)",
				     gsp_rm_rpc_name(function), function, sequence,
				     hdr->length, rm->unknown_rpcs);
		}
		/* Small delay before UNLOADING replies: mitigates an intermittent
		 * driver-side reply-matching race in the cleanup path. */
		if (function == 47)
			udelay(2000);
		if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0))
			nv_vgpu_err(ctx->vgpu, "GSPRM: reply send failed fn=%u", function);
		else
			rm->rpcs_replied++;
		return;
	}
}

/*
 * Poll the command queue TX writePtr; consume and handle new records.
 * A record spans elemCount consecutive queue elements.
 *
 * msgq pointers are SLOT INDICES that wrap at msgCount (msgq.c:
 * "writePtr += n; if (>= msgCount) -= msgCount"). Empty when
 * writePtr == readPtr. A 62->0 transition is a legitimate wrap, NOT
 * queue death.
 */
static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsprm_msgq_tx_header tx;
	int ret;

	ret = gsp_rm_read_buf(ctx, rm->cmd_off, &tx, sizeof(tx));
	if (ret) {
		/* VM may be gone or device closed; stay quiet, retry later */
		return;
	}

	/* A healthy driver ALWAYS has writePtr < msgCount. Anything else
	 * means the queue header is garbage (buffer freed on unload/crash
	 * without an observable GSP shutdown) - declare the emulated GSP
	 * dead so the next probe's WPR2-up check passes and it re-boots. */
	if (tx.writePtr >= rm->cmd_msgcount) {
		nv_vgpu_warn(ctx->vgpu, "GSPRM: insane cmd writePtr %u (msgCount=%u), GSP boot declared dead",
			     tx.writePtr, rm->cmd_msgcount);
		gsp_emu_clear_boot_state(ctx, "stale GSP-RM queue header");
		return;
	}

	/* Consume records until we catch up with the (wrapped) writePtr */
	while (rm->cmd_read != tx.writePtr) {
		u8 *elem;
		u64 elem_off = rm->cmd_off + rm->entry_off +
			       (u64)(rm->cmd_read % rm->cmd_msgcount) * GSPRM_QUEUE_ELEMENT_SIZE;
		u32 signature, elem_count;
		const struct gsprm_rpc_header *hdr;

		elem = kmalloc(GSPRM_QUEUE_ELEMENT_SIZE, GFP_KERNEL);
		if (!elem)
			return;
		ret = gsp_rm_read_buf(ctx, elem_off, elem, GSPRM_QUEUE_ELEMENT_SIZE);
		if (ret) {
			kfree(elem);
			return;
		}

		hdr = (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
		signature = hdr->signature;
		memcpy(&elem_count, elem + 0x28, 4);

		if (signature != GSPRM_RPC_SIGNATURE || elem_count == 0 ||
		    elem_count > rm->cmd_msgcount) {
			/* Garbage element: could be teardown (driver stopped
			 * writing valid elements after UNLOADING_GUEST_DRIVER/
			 * FREE sequence) or genuine queue death. Don't declare
			 * death here - stop polling quietly and let the
			 * Booter-unload kick (SEC2 CPUCTL) trigger the reset
			 * via gsp_emu_clear_boot_state. This allows the
			 * driver's cleanup frees to complete. */
			nv_vgpu_info(ctx->vgpu, "GSPRM: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, stopping poll (teardown?)",
				     rm->cmd_read, signature, elem_count,
				     hdr->function);
			kfree(elem);
			return;
		}

		gsp_rm_handle_element(ctx, elem);
		kfree(elem);

		/* consume the whole record (all its elements), wrapping
		 * at msgCount like the driver's own writePtr */
		rm->cmd_read = (rm->cmd_read + elem_count) % rm->cmd_msgcount;
	}

	/* Publish our command-queue read position into the status queue RX
	 * header (MSGQ_FLAGS_SWAP_RX: read pointers live in the peer's queue). */
	gsp_rm_write_buf(ctx, rm->stat_off + 0x20,
			 &rm->cmd_read, sizeof(rm->cmd_read));
}

/* ============================================================
 * Public interface
 * ============================================================ */

int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm;

	rm = kzalloc(sizeof(*rm), GFP_KERNEL);
	if (!rm)
		return -ENOMEM;

	rm->vgpu = vgpu;
	ctx->rm_emu = rm;
	return 0;
}

void gsp_rm_emu_fini(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;

	if (!rm)
		return;
	kfree(rm->ptes);
	kfree(rm);
	ctx->rm_emu = NULL;
}

/*
 * Reset the GSP-RM emulation state on GSP ENGINE reset (guest driver reboots
 * the GSP on every probe/unload). The queue buffer GPAs from the previous
 * boot are stale (freed by the guest) - drop all parsed state so the next
 * Booter completion re-parses the chain.
 */
void gsp_rm_emu_reset(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;

	if (!rm || !rm->parsed)
		return;

	nv_vgpu_info(ctx->vgpu, "GSPRM: reset (rpcs=%llu replied=%llu unknown=%llu)",
		     rm->rpcs_seen, rm->rpcs_replied, rm->unknown_rpcs);

	kfree(rm->ptes);
	rm->ptes = NULL;
	rm->parsed = false;
	rm->active = false;
	rm->init_done_sent = false;
	rm->libos_args_gpa = 0;
	rm->gsp_args_gpa = 0;
	rm->cmd_read = 0;
	rm->stat_write = 0;
	rm->stat_seq = 0;
	memset(&rm->mq, 0, sizeof(rm->mq));
	rm->cmd_gpa = 0;
	rm->stat_gpa = 0;
	rm->cmd_size = 0;
	rm->stat_size = 0;
	rm->cmd_msgcount = 0;
	rm->stat_msgcount = 0;
}

/*
 * Called when the guest's Booter Load completes. The GSP MAILBOX0/1 now hold
 * the libos init args GPA; parse the queue chain, link the status queue and
 * emit GSP_INIT_DONE so the driver's kgspWaitForRmInitDone completes.
 */
void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u32 mb_lo, mb_hi;
	u32 b_is_d3hot = 0;
	int ret;

	if (!rm || rm->parsed)
		return;

	mb_lo = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0);
	mb_hi = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0 + 4);
	rm->libos_args_gpa = ((u64)mb_hi << 32) | mb_lo;

	nv_vgpu_info(ctx->vgpu, "GSPRM: libos args at gpa=0x%llx (mailbox 0x%x:0x%x)",
		     rm->libos_args_gpa, mb_hi, mb_lo);

	if (!rm->libos_args_gpa) {
		pr_warn_ratelimited("GSPRM: libos args GPA is 0, will retry\n");
		return;
	}

	ret = gsp_rm_parse_queue_chain(ctx);
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: queue chain parse failed: %d", ret);
		return;
	}

	ret = gsp_rm_link_status_queue(ctx);
	if (ret)
		return;

	rm->parsed = true;
	rm->active = true;

	/* Emit GSP_INIT_DONE (function 0x1001, sequence 0, init_done_v17_00) */
	ret = gsp_rm_send(ctx, GSPRM_RPC_EVENT_GSP_INIT_DONE, 0, 0,
			  &b_is_d3hot, sizeof(b_is_d3hot));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_INIT_DONE send failed: %d", ret);
		return;
	}
	rm->init_done_sent = true;

	nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_INIT_DONE emitted, RPC loop active");
}

void gsp_rm_emu_poll(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;

	if (!rm)
		return;

	/* Retry chain parsing if the first attempt at BOOTER_DONE failed
	 * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE.
	 * Back off hard: at 100us poll cadence an unthrottled retry spams
	 * dmesg into unusability when the driver stalls mid-bootstrap. */
	if (!rm->parsed) {
		if (ctx->boot_stage == 2 &&
		    time_after(jiffies, rm->last_parse_retry + HZ / 4)) {
			rm->last_parse_retry = jiffies;
			gsp_rm_emu_on_booter_done(ctx);
		}
		return;
	}

	if (!rm->active)
		return;

	gsp_rm_poll_commands(ctx);
}