/*
 * GSP-RM Emulation (physical-path) for Host-Side vGPU
 *
 * Emulates the GSP-RM msgq RPC channel in guest system memory.
 * See gsp_rm_emulation.h for the design overview.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/vfio.h>
#include <linux/delay.h>
#include <linux/string.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_emulation.h"
#include "gsp_rm_emulation.h"
#include "gsp_rm_tu104.h"
#include "gsp_sema_service.h"
#include "gsp_phys_booter.h"
#include "gsp_phys_gpu.h"

/* ============================================================
 * Guest-memory access ("virtual DMA" on behalf of the device)
 * ============================================================ */

static int gsp_rm_read(struct gsp_emu_context *ctx, u64 gpa, void *buf,
		       size_t len)
{
	struct nv_vgpu_device *vgpu = ctx->vgpu;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int gsp_rm_read_dev(struct nv_vgpu_device *vgpu, u64 gpa, void *buf,
			   size_t len)
{
	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int gsp_rm_write(struct gsp_emu_context *ctx, u64 gpa, void *buf,
			size_t len)
{
	struct nv_vgpu_device *vgpu = ctx->vgpu;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/*
 * Debug helper: scan guest RAM for a 64-bit value (8-byte aligned) and/or a
 * 32-bit value (4-byte aligned), reporting the GPAs where they are found.
 * Used to locate guest structures (e.g. the UVM pushbuffer) without knowing
 * the guest's GPU page tables. Triggered via the module parameter
 * scan_guest_ram=<hex needle>.
 */
int gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle,
			  u64 start, u64 end, u32 max_hits)
{
	u8 *buf;
	u64 gpa;
	u32 hits = 0;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
		u32 off;

		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
			continue;

		for (off = 0; off + 8 <= PAGE_SIZE; off += 4) {
			u64 v;
			u32 v32;

			memcpy(&v, buf + off, 8);
			memcpy(&v32, buf + off, 4);

			if (v == needle) {
				pr_info("vgsp-rm: scan: u64 0x%llx at gpa 0x%llx (page off 0x%x)\n",
					needle, gpa + off, off);
				if (++hits >= max_hits)
					break;
			}
			if (v32 == (u32)needle && needle <= 0xffffffffULL) {
				pr_info("vgsp-rm: scan: u32 0x%x at gpa 0x%llx (page off 0x%x)\n",
					(u32)needle, gpa + off, off);
				if (++hits >= max_hits)
					break;
			}
		}
	}

	kfree(buf);
	pr_info("vgsp-rm: scan: needle=0x%llx range=0x%llx..0x%llx hits=%u\n",
		needle, start, end, hits);
	return hits;
}

/*
 * Debug helper: scan guest RAM for values matching (v & mask) == value.
 * Used to find a stored page-base VA when only "base + index*size" is known.
 */
int gsp_rm_scan_guest_ram_masked(struct nv_vgpu_device *vgpu, u64 value,
				 u64 mask, u64 start, u64 end, u32 max_hits)
{
	u8 *buf;
	u64 gpa;
	u32 hits = 0;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
		u32 off;

		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
			continue;

		for (off = 0; off + 8 <= PAGE_SIZE; off += 4) {
			u64 v64;
			u32 v32;

			memcpy(&v64, buf + off, 8);
			memcpy(&v32, buf + off, 4);
			if ((v64 & mask) == value) {
				pr_info("vgsp-rm: masked scan: u64 0x%llx at gpa 0x%llx (off 0x%x)\n",
					v64, gpa + off, off);
				if (++hits >= max_hits)
					break;
			}
			if ((v32 & (u32)mask) == (u32)value) {
				pr_info("vgsp-rm: masked scan: u32 0x%x at gpa 0x%llx (off 0x%x)\n",
					v32, gpa + off, off);
				if (++hits >= max_hits)
					break;
			}
		}
	}

	kfree(buf);
	pr_info("vgsp-rm: masked scan: value=0x%llx mask=0x%llx hits=%u\n",
		value, mask, hits);
	return hits;
}

/*
 * Debug helper: scan guest RAM for page-table entries pointing at a given
 * physical address (matches v & 0xfffff000 == pa for u32 and the 64-bit
 * equivalent). Used to locate the guest's GPU page tables.
 */
int gsp_rm_scan_guest_ram_pt(struct nv_vgpu_device *vgpu, u64 pa,
			     u64 start, u64 end, u32 max_hits)
{
	u8 *buf;
	u64 gpa;
	u32 hits = 0;
	u64 pa_page = pa & ~0xfffULL;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;

	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
		u32 off;

		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
			continue;

		for (off = 0; off + 8 <= PAGE_SIZE; off += 4) {
			u64 v64;
			u32 v32;

			memcpy(&v64, buf + off, 8);
			memcpy(&v32, buf + off, 4);
			if ((v64 & 0xfffffffffffff000ULL) == pa_page) {
				pr_info("vgsp-rm: pt scan: u64 pte 0x%llx at gpa 0x%llx (off 0x%x)\n",
					v64, gpa + off, off);
				if (++hits >= max_hits)
					break;
			}
			if ((v32 & 0xfffff000U) == (u32)pa_page) {
				pr_info("vgsp-rm: pt scan: u32 pte 0x%x at gpa 0x%llx (off 0x%x)\n",
					v32, gpa + off, off);
				if (++hits >= max_hits)
					break;
			}
		}
	}

	kfree(buf);
	pr_info("vgsp-rm: pt scan: pa=0x%llx hits=%u\n", pa, hits);
	return hits;
}

/*
 * Debug helper: scan guest RAM for a sequence of up to 4 u32 values.
 * Used to locate a CE semaphore-release method sequence in a pushbuffer.
 */
int gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat,
			      u32 npat, u64 start, u64 end, u32 max_hits)
{
	u8 *buf;
	u64 gpa;
	u32 hits = 0;

	if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
		return -ENODEV;
	if (!npat || npat > 4)
		return -EINVAL;

	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) {
		u32 off;

		if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE))
			continue;

		for (off = 0; off + npat * 4 <= PAGE_SIZE; off += 4) {
			u32 i, ok = 1;

			for (i = 0; i < npat; i++) {
				u32 v;

				memcpy(&v, buf + off + i * 4, 4);
				if (v != pat[i]) {
					ok = 0;
					break;
				}
			}
			if (ok) {
				pr_info("vgsp-rm: seq scan: hit in guest RAM at gpa 0x%llx (page off 0x%x)\n",
					gpa + off, off);
				if (++hits >= max_hits)
					break;
			}
		}
	}

	kfree(buf);
	pr_info("vgsp-rm: seq scan: range=0x%llx..0x%llx hits=%u\n",
		start, end, hits);
	return hits;
}static u64 gsp_rm_off_to_gpa(struct gsp_rm_emu *rm, u64 off)
{
	u64 idx = off >> 12;

	if (idx >= rm->mq.pageTableEntryCount)
		return 0;
	return rm->ptes[idx] + (off & 0xfff);
}

/*
 * Page-aware shared-buffer I/O. The queue buffer is NV_MEMORY_NONCONTIGUOUS:
 * its 4K pages are independent guest allocations, so a range crossing a page
 * boundary is NOT contiguous in guest physical memory. Translating the start
 * and blasting a multi-page range would scribble into arbitrary guest RAM
 * (observed as guest userspace segfaults). Split at page boundaries and
 * translate each page separately.
 */
static int gsp_rm_write_buf(struct gsp_emu_context *ctx, u64 off,
			    const void *buf, size_t len)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const u8 *p = buf;
	int ret;

	while (len) {
		u64 gpa = gsp_rm_off_to_gpa(rm, off);
		size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff));

		if (!gpa)
			return -EINVAL;
		ret = gsp_rm_write(ctx, gpa, (void *)p, chunk);
		if (ret)
			return ret;
		p += chunk;
		off += chunk;
		len -= chunk;
	}
	return 0;
}

static int gsp_rm_read_buf(struct gsp_emu_context *ctx, u64 off,
			   void *buf, size_t len)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u8 *p = buf;
	int ret;

	while (len) {
		u64 gpa = gsp_rm_off_to_gpa(rm, off);
		size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff));

		if (!gpa)
			return -EINVAL;
		ret = gsp_rm_read(ctx, gpa, p, chunk);
		if (ret)
			return ret;
		p += chunk;
		off += chunk;
		len -= chunk;
	}
	return 0;
}

/* ============================================================
 * Status queue element construction
 * ============================================================ */

/* XOR-fold checksum over u64s (message_queue_priv.h _checkSum32) */
static u32 gsp_rm_checksum32(const void *data, u32 len)
{
	const u64 *p = data;
	const u64 *end = (const u64 *)((const u8 *)data + len);
	u64 sum = 0;

	while (p < end)
		sum ^= *p++;

	return (u32)(sum >> 32) ^ (u32)sum;
}

/*
 * 570.144 GSP_MSG_QUEUE_ELEMENT wire layout (guest driver is 570.144 stock):
 *   0x00  authTagBuffer[16]  (zero - no CC)
 *   0x10  aadBuffer[16]      (zero - no CC)
 *   0x20  checkSum           (XOR-fold over msgLen, makes total 0)
 *   0x24  seqNum             (per-record sequence)
 *   0x28  elemCount          (elements this record spans)
 *   0x30  rpc_message_header_v (32 bytes) + payload
 * GSP_MSG_QUEUE_ELEMENT_HDR_SIZE = offsetof(rpc) = 0x30.
 */
#define GSPRM_ELEM_HDR_SIZE	0x30

/* rpc_gsp_rm_alloc_v03_00 fixed header: hClient, hParent, hObject, hClass,
 * status, paramsSize, flags, reserved[4]. */
#define GSPRM_ALLOC_HDR_SIZE	32

/*
 * Max RPC data that fits in one queue element:
 * 4096 (element) - 0x30 (element hdr) - 32 (rpc hdr) = 4016 bytes.
 */
#define GSPRM_MAX_SINGLE_ELEM_DATA	(GSPRM_QUEUE_ELEMENT_SIZE - \
					 GSPRM_ELEM_HDR_SIZE - \
					 sizeof(struct gsprm_rpc_header))

/*
 * Build a multi-element record into a contiguous buffer.
 *
 * The record spans elem_count consecutive queue elements:
 *   [0]: element header (0x30) + rpc header (32) + data[0..4015]
 *   [1..N-1]: continuation data only (no headers)
 *
 * Returns the total record size in bytes (elem_count * 4096).
 * Sets *out_elem_count to the number of elements used.
 */
static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size,
				u32 function, u32 sequence, u32 result,
				const void *data, u32 data_len,
				u32 *out_elem_count)
{
	struct gsprm_rpc_header *hdr =
		(struct gsprm_rpc_header *)(buf + GSPRM_ELEM_HDR_SIZE);
	u32 rpc_len = sizeof(*hdr) + data_len;
	u32 msg_len = GSPRM_ELEM_HDR_SIZE + rpc_len;
	u32 aligned_len;
	u32 elem_count;
	u32 first_data;
	u32 remaining;
	u32 i;

	/* How many elements does this record span? */
	elem_count = (msg_len + GSPRM_QUEUE_ELEMENT_SIZE - 1) /
		     GSPRM_QUEUE_ELEMENT_SIZE;
	if (elem_count == 0)
		elem_count = 1;

	if (elem_count * GSPRM_QUEUE_ELEMENT_SIZE > buf_size) {
		pr_err("vgsp-rm: build_record overflow: need %u have %u\n",
		       elem_count * GSPRM_QUEUE_ELEMENT_SIZE, buf_size);
		return 0;
	}

	memset(buf, 0, elem_count * GSPRM_QUEUE_ELEMENT_SIZE);

	/* First element: header + rpc header + start of data */
	memcpy(buf + 0x24, &rm->stat_seq, 4);		/* seqNum */
	memcpy(buf + 0x28, &elem_count, 4);		/* elemCount */

	hdr->header_version = GSPRM_RPC_HEADER_VERSION;
	hdr->signature = GSPRM_RPC_SIGNATURE;
	hdr->length = rpc_len;
	hdr->function = function;
	hdr->rpc_result = result;
	hdr->rpc_result_private = result;
	hdr->sequence = sequence;
	hdr->spare = 0;

	/* Copy data across element boundaries */
	first_data = min(data_len, (u32)GSPRM_MAX_SINGLE_ELEM_DATA);
	if (data && first_data)
		memcpy((u8 *)(hdr + 1), data, first_data);

	remaining = data_len - first_data;
	if (remaining > 0 && data) {
		const u8 *src = (const u8 *)data + first_data;
		u8 *dst = buf + GSPRM_QUEUE_ELEMENT_SIZE;

		for (i = 1; i < elem_count && remaining > 0; i++) {
			u32 chunk = min(remaining,
					(u32)GSPRM_QUEUE_ELEMENT_SIZE);

			memcpy(dst, src, chunk);
			src += chunk;
			dst += GSPRM_QUEUE_ELEMENT_SIZE;
			remaining -= chunk;
		}
	}

	/* checksum covers msg_len zero-padded to 8 (buffer already zeroed) */
	aligned_len = (msg_len + 7) & ~7U;
	*(u32 *)(buf + 0x20) = gsp_rm_checksum32(buf, aligned_len);

	*out_elem_count = elem_count;
	return elem_count * GSPRM_QUEUE_ELEMENT_SIZE;
}

/*
 * Write a status queue record (1+ elements) and advance the writePtr.
 */
static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
		       u32 sequence, u32 result, const void *data, u32 data_len)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u8 *buf;
	u32 slot;
	u32 new_wp;
	u32 elem_count;
	u32 record_size;
	u32 buf_size;
	int ret;

	/* Allocate buffer for worst-case multi-element record */
	buf_size = GSPRM_QUEUE_ELEMENT_SIZE *
		   ((data_len + GSPRM_MAX_SINGLE_ELEM_DATA +
		     GSPRM_QUEUE_ELEMENT_SIZE - 1) /
		    GSPRM_QUEUE_ELEMENT_SIZE + 1);
	if (buf_size < GSPRM_QUEUE_ELEMENT_SIZE)
		buf_size = GSPRM_QUEUE_ELEMENT_SIZE;

	buf = kzalloc(buf_size, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	/* The status queue is shared: RPC replies run on the GSP emulation
	 * thread, channel-completion events are posted from the progress
	 * service's thread. */
	mutex_lock(&rm->stat_lock);

	/*
	 * Bound the number of event records written between replies. Events are
	 * only wake-ups (the client re-reads channel state), so dropping some is
	 * harmless, but an unbounded event burst can wrap the status ring while
	 * a client is blocked waiting for its reply - the read then returns a
	 * newer record and the client aborts with "Bad sequence number".
	 */
	if (function == GSPRM_RPC_EVENT_POST_EVENT) {
		if (rm->event_burst >= GSPRM_EVENT_BURST_MAX) {
			rm->events_dropped++;
			if (rm->events_dropped <= 8 ||
			    (rm->events_dropped % 1000) == 0)
				nv_vgpu_info(ctx->vgpu,
					     "vgsp-rm: event burst capped (%u), dropped total=%llu",
					     GSPRM_EVENT_BURST_MAX,
					     rm->events_dropped);
			mutex_unlock(&rm->stat_lock);
			kfree(buf);
			return -EAGAIN;
		}
		rm->event_burst++;
	} else {
		rm->event_burst = 0;
	}

	record_size = gsp_rm_build_record(rm, buf, buf_size, function,
					   sequence, result, data, data_len,
					   &elem_count);
	if (!record_size) {
		kfree(buf);
		mutex_unlock(&rm->stat_lock);
		return -EOVERFLOW;
	}

	/* Check we don't overflow the status queue */
	if (elem_count > rm->stat_msgcount) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: record too large: %u elements > queue %u",
			    elem_count, rm->stat_msgcount);
		kfree(buf);
		mutex_unlock(&rm->stat_lock);
		return -EOVERFLOW;
	}

	/* Write all elements to consecutive queue slots. Page-aware: the
	 * queue buffer is non-contiguous, so translate per page. */
	slot = rm->stat_write % rm->stat_msgcount;

	/* Handle wrap-around: write may span the end of the circular queue */
	{
		u32 elems_to_end = rm->stat_msgcount - slot;
		u32 first_chunk = min(elem_count, elems_to_end);
		u32 second_chunk = elem_count - first_chunk;
		u64 rec_off = rm->stat_off + rm->entry_off +
			      (u64)slot * GSPRM_QUEUE_ELEMENT_SIZE;

		ret = gsp_rm_write_buf(ctx, rec_off, buf,
				       first_chunk * GSPRM_QUEUE_ELEMENT_SIZE);
		if (ret) {
			nv_vgpu_err(ctx->vgpu, "vgsp-rm: status write failed off=0x%llx ret=%d",
				    rec_off, ret);
			kfree(buf);
			mutex_unlock(&rm->stat_lock);
			return ret;
		}

		if (second_chunk > 0) {
			ret = gsp_rm_write_buf(ctx, rm->stat_off + rm->entry_off,
					       buf + first_chunk * GSPRM_QUEUE_ELEMENT_SIZE,
					       second_chunk * GSPRM_QUEUE_ELEMENT_SIZE);
			if (ret) {
				nv_vgpu_err(ctx->vgpu, "vgsp-rm: status wrap write failed ret=%d",
					    ret);
				kfree(buf);
				mutex_unlock(&rm->stat_lock);
				return ret;
			}
		}
	}

	kfree(buf);

	/* Advance the write pointer with the same wrap semantics as the
	 * driver's msgq (slot index in [0, msgCount) - the peer's free-space
	 * and slot math breaks on unbounded values). stat_seq stays
	 * monotonic per-record. */
	rm->stat_write = (rm->stat_write + elem_count) % rm->stat_msgcount;
	rm->stat_seq++;

	/* bump status TX header writePtr (offset 0x10) */
	new_wp = rm->stat_write;
	ret = gsp_rm_write_buf(ctx, rm->stat_off + 0x10,
			       &new_wp, sizeof(new_wp));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: status writePtr update failed ret=%d", ret);
		mutex_unlock(&rm->stat_lock);
		return ret;
	}

	/* Notify the guest (MSI-X vector 0). The driver's bottom-half drains
	 * events under the GPU lock; the rpcRecvPoll poller holds the same
	 * lock from send to poll, so a reply can't be consumed by the event
	 * drain while a poller waits for it (same as real GSP-RM). */
	gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq);

	mutex_unlock(&rm->stat_lock);
	return 0;
}

/*
 * Post the GPU event that tells the guest's CPU-side RM a channel's
 * submitted work is complete. A real GPU raises an interrupt, the host GSP
 * turns it into an NV_VGPU_MSG_EVENT_POST_EVENT record for the client's
 * event object, and the client's RM signals the OS event (eventfd) the
 * submitting thread polls - this is what makes libcuda's work waits finish.
 *
 * Safe to call from the progress service's thread: the status queue write is
 * serialized by rm->stat_lock and the guest is woken with MSI-X vector 0.
 */
int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src)
{
	struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
	struct gsp_rm_emu *rm;
	u8 payload[32];
	u32 i;
	int posted = 0;

	if (!ctx)
		return -ENODEV;
	rm = ctx->rm_emu;
	if (!rm || !h_src)
		return -ENODEV;
	/* Never post across a queue reset: the guest has freed the old queue
	 * and re-parses on the next Booter completion. */
	if (!rm->parsed)
		return -EAGAIN;

	for (i = 0; i < rm->events_n; i++) {
		if (!rm->events[i].valid || rm->events[i].h_src != h_src)
			continue;

		memset(payload, 0, sizeof(payload));
		memcpy(payload + 0, &rm->events[i].h_client, 4);
		memcpy(payload + 4, &rm->events[i].h_event, 4);
		memcpy(payload + 8, &rm->events[i].notify_index, 4);
		memcpy(payload + 12, &rm->events[i].data, 4);
		/* info16@16, status@20, eventDataSize@24, bNotifyList@28 = 0 */
		if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
				payload, sizeof(payload)))
			continue;
		posted++;
	}

	/*
	 * Fallback when the event's source object could not be associated with
	 * the channel: nudge every registered event. The client's dispatcher
	 * re-reads each channel's state on any event, so a spurious wake-up is
	 * harmless (and this only runs when a submission was consumed).
	 */
	if (!posted) {
		for (i = 0; i < rm->events_n; i++) {
			if (!rm->events[i].valid)
				continue;
			memset(payload, 0, sizeof(payload));
			memcpy(payload + 0, &rm->events[i].h_client, 4);
			memcpy(payload + 4, &rm->events[i].h_event, 4);
			memcpy(payload + 8, &rm->events[i].notify_index, 4);
			memcpy(payload + 12, &rm->events[i].data, 4);
			if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
					payload, sizeof(payload)))
				continue;
			posted++;
		}
	}

	if (posted) {
		rm->events_posted += posted;
		if (gsp_sema_dbg &&
		    (rm->events_posted <= 32 || (rm->events_posted % 5000) == 0))
			nv_vgpu_info(vgpu,
				     "vgsp-rm: post_event hSrc=0x%x posted=%d total=%llu",
				     h_src, posted, rm->events_posted);
	}
	return posted ? 0 : -ENOENT;
}

/*
 * Debug: post the event record for one registered event (or all of them with
 * idx 0xffffffff). Used to test the guest's event path on demand without
 * waiting for a channel submission.
 */
static int gsp_rm_post_event_idx(struct gsp_emu_context *ctx, u32 idx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u8 payload[32];

	if (idx >= rm->events_n || !rm->events[idx].valid)
		return -EINVAL;

	memset(payload, 0, sizeof(payload));
	memcpy(payload + 0, &rm->events[idx].h_client, 4);
	memcpy(payload + 4, &rm->events[idx].h_event, 4);
	memcpy(payload + 8, &rm->events[idx].notify_index, 4);
	memcpy(payload + 12, &rm->events[idx].data, 4);

	return gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
			   payload, sizeof(payload));
}

int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx)
{
	struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
	struct gsp_rm_emu *rm;
	u32 i;
	int n = 0, last = -ENOENT;

	if (!ctx)
		return -ENODEV;
	rm = ctx->rm_emu;
	if (!rm)
		return -ENODEV;

	if (idx != 0xffffffffu) {
		int r = gsp_rm_post_event_idx(ctx, idx);

		return r ? r : 1;
	}

	for (i = 0; i < rm->events_n; i++) {
		int r = gsp_rm_post_event_idx(ctx, i);

		if (r)
			last = r;
		else
			n++;
	}

	return n ? 0 : last;
}

/*
 * Post the completion event to every registered OS event (the classes the
 * guest allocates carry no source object, so a per-channel association is
 * not available in the ABI). Used by the service's completion hint so a
 * client that starts waiting after the event fired still gets a wake-up.
 */
int gsp_rm_post_all_events(struct nv_vgpu_device *vgpu)
{
	return gsp_rm_force_post(vgpu, 0xffffffffu);
}

/*
 * Debug: dump the msgq cursors (ours and the guest's) and the event table.
 * The guest's consumption cursor of the status queue lives in its own
 * command-queue buffer at the rxHdrOff it advertised (SWAP_RX queues store
 * the peer read pointer in the local buffer).
 */
int gsp_rm_dump_state(struct nv_vgpu_device *vgpu)
{
	struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
	struct gsp_rm_emu *rm;
	u8 cmd_hdr[32] = {0}, stat_hdr[32] = {0};
	u32 cmd_write = 0, cmd_flags = 0, cmd_rxoff = 0, cmd_entryoff = 0;
	u32 stat_write = 0, stat_flags = 0, stat_rxoff = 0;
	u32 guest_read = 0xdeadbeef, our_cmd_read = 0xdeadbeef;
	u64 cmd_gpa_now = 0, stat_gpa_now = 0;
	int cmd_ret = -1, stat_ret = -1, gr_ret = -1;
	u32 i;

	if (!ctx)
		return -ENODEV;
	rm = ctx->rm_emu;
	if (!rm || !rm->parsed)
		return -ENODEV;

	cmd_gpa_now = gsp_rm_off_to_gpa(rm, rm->cmd_off);
	stat_gpa_now = gsp_rm_off_to_gpa(rm, rm->stat_off);

	cmd_ret = gsp_rm_read_buf(ctx, rm->cmd_off, cmd_hdr, sizeof(cmd_hdr));
	if (cmd_ret == 0) {
		memcpy(&cmd_write, cmd_hdr + 16, 4);
		memcpy(&cmd_flags, cmd_hdr + 20, 4);
		memcpy(&cmd_rxoff, cmd_hdr + 24, 4);
		memcpy(&cmd_entryoff, cmd_hdr + 28, 4);
	}
	stat_ret = gsp_rm_read_buf(ctx, rm->stat_off, stat_hdr, sizeof(stat_hdr));
	if (stat_ret == 0) {
		memcpy(&stat_write, stat_hdr + 16, 4);
		memcpy(&stat_flags, stat_hdr + 20, 4);
		memcpy(&stat_rxoff, stat_hdr + 24, 4);
	}

	if (cmd_rxoff && cmd_rxoff < 0x1000)
		gr_ret = gsp_rm_read_buf(ctx, rm->cmd_off + cmd_rxoff,
					 &guest_read, 4);
	if (stat_rxoff && stat_rxoff < 0x1000)
		gsp_rm_read_buf(ctx, rm->stat_off + stat_rxoff, &our_cmd_read, 4);

	nv_vgpu_info(vgpu,
		     "vgsp-rm: rmstate cmd{gpa=0x%llx->0x%llx off=0x%llx size=0x%x mc=%u ptes=%u read=%d our_read=%u hdr{wr=%u fl=0x%x rxoff=0x%x entryoff=0x%x}} "
		     "stat{gpa=0x%llx->0x%llx off=0x%llx size=0x%x mc=%u read=%d our_write=%u seq=%u hdr{wr=%u fl=0x%x rxoff=0x%x ours_written=%u} guest_read=%u gr_ret=%d}",
		     rm->cmd_gpa, cmd_gpa_now, rm->cmd_off, rm->cmd_size,
		     rm->cmd_msgcount, rm->mq.pageTableEntryCount, cmd_ret,
		     rm->cmd_read, cmd_write, cmd_flags, cmd_rxoff, cmd_entryoff,
		     rm->stat_gpa, stat_gpa_now, rm->stat_off, rm->stat_size,
		     rm->stat_msgcount, stat_ret, rm->stat_write, rm->stat_seq,
		     stat_write, stat_flags, stat_rxoff, our_cmd_read,
		     guest_read, gr_ret);


	nv_vgpu_info(vgpu, "vgsp-rm: rmstate events_n=%u posted=%llu",
		     rm->events_n, rm->events_posted);

	for (i = 0; i < rm->events_n && i < GSPRM_MAX_EVENTS; i++) {
		if (!rm->events[i].valid)
			continue;
		nv_vgpu_info(vgpu,
			     "vgsp-rm:   event[%u] hClient=0x%x hEvent=0x%x hSrc=0x%x notifyIndex=0x%x data=0x%x",
			     i, rm->events[i].h_client, rm->events[i].h_event,
			     rm->events[i].h_src, rm->events[i].notify_index,
			     rm->events[i].data);
	}

	return 0;
}

/* ============================================================
 * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED)
 * ============================================================ */

static int gsp_rm_parse_queue_chain(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsprm_libos_region *regions;
	int ret, i;
	u64 gsp_args_gpa = 0;
	u32 total_size;

	regions = kmalloc_array(GSPRM_MAX_LIBOS_REGIONS, sizeof(*regions),
				GFP_KERNEL);
	if (!regions)
		return -ENOMEM;

	/* 1. Read the libos init args array */
	ret = gsp_rm_read(ctx, rm->libos_args_gpa, regions,
			  GSPRM_MAX_LIBOS_REGIONS * sizeof(*regions));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: libos args read failed gpa=0x%llx ret=%d",
			    rm->libos_args_gpa, ret);
		goto out;
	}

	/* 2. Find the RMARGS region -> GSP_ARGUMENTS_CACHED */
	for (i = 0; i < GSPRM_MAX_LIBOS_REGIONS; i++) {
		if (regions[i].id8 == 0)
			break;
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: libos region[%d] id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u",
			     i, regions[i].id8, regions[i].pa, regions[i].size,
			     regions[i].kind, regions[i].loc);
		if (regions[i].id8 == GSPRM_LIBOS_ID_RMARGS)
			gsp_args_gpa = regions[i].pa;
	}
	if (!gsp_args_gpa) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: RMARGS region not found");
		ret = -ENOENT;
		goto out;
	}
	rm->gsp_args_gpa = gsp_args_gpa;

	/* 3. Read GSP_ARGUMENTS_CACHED (MESSAGE_QUEUE_INIT_ARGUMENTS first) */
	ret = gsp_rm_read(ctx, gsp_args_gpa, &rm->mq, sizeof(rm->mq));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_ARGUMENTS_CACHED read failed ret=%d", ret);
		goto out;
	}

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: queue args sharedPA=0x%llx ptes=%u cmdOff=0x%llx statOff=0x%llx hdrSize=0x%llx elemMin=0x%llx elemMax=0x%llx hdrAlign=%u elemAlign=%u",
		     rm->mq.sharedMemPhysAddr, rm->mq.pageTableEntryCount,
		     rm->mq.cmdQueueOffset, rm->mq.statQueueOffset,
		     rm->mq.queueElementHdrSize, rm->mq.queueElementSizeMin,
		     rm->mq.queueElementSizeMax, rm->mq.queueHeaderAlign,
		     rm->mq.queueElementAlign);
	print_hex_dump(KERN_INFO, "GSPRM args raw: ", DUMP_PREFIX_OFFSET,
		       16, 8, &rm->mq, sizeof(rm->mq), false);

	if (!rm->mq.sharedMemPhysAddr || !rm->mq.pageTableEntryCount ||
	    rm->mq.pageTableEntryCount > 1024) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: invalid queue args");
		ret = -EINVAL;
		goto out;
	}

	/* 4. Read the page table (RmPhysAddr per 4K page of the shared buffer) */
	/* A retry (or a post-reset re-parse) must not leak the previous table. */
	kfree(rm->ptes);
	rm->ptes = kmalloc_array(rm->mq.pageTableEntryCount, sizeof(u64),
				 GFP_KERNEL);
	if (!rm->ptes) {
		ret = -ENOMEM;
		goto out;
	}
	ret = gsp_rm_read(ctx, rm->mq.sharedMemPhysAddr, rm->ptes,
			  rm->mq.pageTableEntryCount * sizeof(u64));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: page table read failed ret=%d", ret);
		goto out;
	}

	/* 5. Compute queue geometry. NOTE: the shared buffer is
	 * NV_MEMORY_NONCONTIGUOUS - only queue-offsets are linear, GPAs are
	 * not. Every access must translate through rm->ptes (gsp_rm_off_to_gpa). */
	rm->cmd_off = rm->mq.cmdQueueOffset;
	rm->stat_off = rm->mq.statQueueOffset;
	rm->cmd_gpa = gsp_rm_off_to_gpa(rm, rm->cmd_off);
	rm->stat_gpa = gsp_rm_off_to_gpa(rm, rm->stat_off);
	rm->cmd_size = (u32)(rm->mq.statQueueOffset - rm->mq.cmdQueueOffset);
	total_size = rm->mq.pageTableEntryCount << 12;
	rm->stat_size = total_size - (u32)rm->mq.statQueueOffset;
	rm->entry_off = 0x1000; /* ALIGN_UP(0x20 + 4, 1 << elemAlign(12)) */
	rm->cmd_msgcount = (rm->cmd_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE;
	rm->stat_msgcount = (rm->stat_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE;

	if (!rm->cmd_gpa || !rm->stat_gpa || !rm->cmd_msgcount || !rm->stat_msgcount) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: bad queue geometry cmd_gpa=0x%llx stat_gpa=0x%llx cmd_mc=%u stat_mc=%u",
			    rm->cmd_gpa, rm->stat_gpa, rm->cmd_msgcount,
			    rm->stat_msgcount);
		ret = -EINVAL;
		goto out;
	}

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: cmd queue gpa=0x%llx size=0x%x mc=%u; status queue gpa=0x%llx size=0x%x mc=%u",
		     rm->cmd_gpa, rm->cmd_size, rm->cmd_msgcount,
		     rm->stat_gpa, rm->stat_size, rm->stat_msgcount);

	ret = 0;
out:
	kfree(regions);
	return ret;
}

/*
 * Write the status queue TX header (what GSP-RM's msgqTxCreate produces).
 */
static int gsp_rm_link_status_queue(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsprm_msgq_tx_header tx;
	u32 zero = 0;
	int ret;

	memset(&tx, 0, sizeof(tx));
	tx.version = GSPRM_MSGQ_VERSION;
	tx.size = rm->stat_size;
	tx.msgSize = GSPRM_QUEUE_ELEMENT_SIZE;
	tx.msgCount = rm->stat_msgcount;
	tx.writePtr = 0;
	tx.flags = GSPRM_MSGQ_FLAGS_SWAP_RX;
	tx.rxHdrOff = 0x20; /* ALIGN_UP(sizeof(msgqTxHeader)=32, 1<<hdrAlign(4)=16) */
	tx.entryOff = rm->entry_off;

	ret = gsp_rm_write_buf(ctx, rm->stat_off, &tx, sizeof(tx));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: status TX header write failed ret=%d", ret);
		return ret;
	}

	/* status queue RX header (our command-queue read position, swap mode) */
	ret = gsp_rm_write_buf(ctx, rm->stat_off + tx.rxHdrOff,
			       &zero, sizeof(zero));
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: status RX header write failed ret=%d", ret);
		return ret;
	}

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: status queue linked (size=0x%x msgCount=%u)",
		     tx.size, tx.msgCount);
	return 0;
}

/* ============================================================
 * Command processing
 * ============================================================ */

static const char *gsp_rm_rpc_name(u32 function)
{
	switch (function) {
	case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO:	return "SET_GUEST_SYSTEM_INFO";
	case GSPRM_RPC_FUNC_ALLOC_ROOT:			return "ALLOC_ROOT";
	case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO:	return "GET_GSP_STATIC_INFO";
	case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO:	return "GSP_SET_SYSTEM_INFO";
	case GSPRM_RPC_FUNC_SET_REGISTRY:		return "SET_REGISTRY";
	case GSPRM_RPC_FUNC_CONTINUATION_RECORD:	return "CONTINUATION_RECORD";
	case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY:		return "SET_PAGE_DIRECTORY";
	case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY:	return "UNSET_PAGE_DIRECTORY";
	case GSPRM_RPC_FUNC_GSP_RM_CONTROL:		return "GSP_RM_CONTROL";
	case GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER:	return "UNLOADING_GUEST_DRIVER";
	case GSPRM_RPC_FUNC_UPDATE_BAR_PDE:		return "UPDATE_BAR_PDE";
	default:					return "UNKNOWN";
	}
}

/*
 * SET_GUEST_SYSTEM_INFO: the driver negotiates the RPC interface version
 * from the reply's vgxVersionMajorNum/MinorNum fields. Without them the
 * version-gated HALs stay stubs and every subsequent RPC fails with
 * NV_VGPU_MSG_RESULT_RPC_UNKNOWN_FUNCTION (0xff100001). Echo the driver's
 * own version back (no host-guest version skew).
 */
static void gsp_rm_handle_set_guest_system_info(struct gsp_emu_context *ctx,
						const u8 *elem,
						u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE +
			     sizeof(struct gsprm_rpc_header);
	u8 *reply;
	u32 major, minor;

	memcpy(&major, req_data + 0, 4);
	memcpy(&minor, req_data + 4, 4);
	if (!major)
		major = GSPRM_VGX_VERSION_MAJOR;
	if (!minor && major != GSPRM_VGX_VERSION_MAJOR)
		minor = GSPRM_VGX_VERSION_MINOR;

	reply = kzalloc(GSPRM_SET_GUEST_INFO_SIZE, GFP_KERNEL);
	if (!reply) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: SET_GUEST_SYSTEM_INFO reply alloc failed");
		return;
	}
	memcpy(reply + 0, &major, 4);
	memcpy(reply + 4, &minor, 4);

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: SET_GUEST_SYSTEM_INFO seq=%u -> VGX %u.%u",
		     sequence, major, minor);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO, sequence, 0,
			reply, GSPRM_SET_GUEST_INFO_SIZE))
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: SET_GUEST_SYSTEM_INFO reply send failed");
	else
		rm->rpcs_replied++;

	kfree(reply);
}

/* ============================================================
 * GET_GSP_STATIC_INFO (function 65)
 *
 * Builds the 1656-byte GspStaticConfigInfo for TU104.
 * ============================================================ */

static void gsp_rm_build_static_info(u8 *buf, u64 fb_size)
{
	u64 val64;
	u32 val32;
	u8 *p;

	memset(buf, 0, TU104_GSCI_SIZE);

	/* grCapsBits (offset 0, 23 bytes) - basic GR caps */
	buf[0] = 0x01;	/* GR present */

	/* gidInfo (offset 24, 268 bytes): index(4) + flags(4) + length(4) + data(256) */
	p = buf + TU104_GSCI_OFF_gidInfo;
	val32 = 0;		memcpy(p + 0, &val32, 4);	/* index */
	val32 = 0x80000000;	memcpy(p + 4, &val32, 4);	/* flags: valid */
	val32 = 16;		memcpy(p + 8, &val32, 4);	/* length = 16 (UUID) */
	/* GPU UUID - deterministic fake for TU104 */
	memcpy(p + 12, "GPU-9da5e221-18", 15);

	/* SKUInfo (offset 292, 48 bytes) */
	p = buf + TU104_GSCI_OFF_SKUInfo;
	memcpy(p + 4, "TU104", 5);	/* chipSKU */
	memcpy(p + 13, "400", 3);	/* chipSKUMod */
	memcpy(p + 20, "N18E", 4);	/* project */
	memcpy(p + 24, "G3", 2);	/* projectSKU */

	/* fbRegionInfoParams (offset 344, 776 bytes) */
	p = buf + TU104_GSCI_OFF_fbRegionInfoParams;
	val32 = 1;		memcpy(p, &val32, 4);		/* numFBRegions = 1 */
	/* Region 0: base=0, limit=512MB-1, reserved=0 */
	{
		struct tu104_fb_region *region = (void *)(p + 8); /* 8-aligned */
		val64 = 0;		memcpy(&region->base, &val64, 8);
		val64 = (fb_size - 1);
		memcpy(&region->limit, &val64, 8);
		val64 = 0;		memcpy(&region->reserved, &val64, 8);
		val32 = 0;		memcpy(&region->performance, &val32, 4);
		region->supportCompressed = 0;
		/* memmgrCalcReservedFbSpace asserts that SOME region supports
		 * ISO "even if we have no display" - without it the reserved
		 * space split fails and kdisp can't allocate its channel
		 * table. A single ISO-capable region satisfies fast/slow/iso
		 * assignment. */
		region->supportISO = 1;
		region->bProtected = 0;
		memset(region->blackList, 0, sizeof(region->blackList));
	}

	/* sriovCaps (offset 1120, 80 bytes) - all zeros (not SR-IOV) */

	/* sriovMaxGfid (offset 1200, 4 bytes) */
	val32 = 0;
	memcpy(buf + TU104_GSCI_OFF_sriovMaxGfid, &val32, 4);

	/* engineCaps (offset 1204, 12 bytes)
	 *
	 * Bit N = RM_ENGINE_TYPE_<N> present (RM_ENGINE_TYPE_* equals the bit
	 * index, see gpu_engine_type.h). The async copy engine is
	 * RM_ENGINE_TYPE_COPY14 (0x17); clients (libcuda) refuse to create a
	 * context without it: the secondary/async engine record stays empty and
	 * the engine-context RM alloc is issued with class 0 (NV01_ROOT), which
	 * the driver must reject. */
	val32 = 0x60803E02;	/* GR0|CE0-4|COPY14|NVDEC0|NVDEC1 */
	memcpy(buf + TU104_GSCI_OFF_engineCaps, &val32, 4);
	val32 = 0x00362020;	/* NVENC0|SW|SEC2|HOST|PMU|FBFLCN */
	memcpy(buf + TU104_GSCI_OFF_engineCaps + 4, &val32, 4);
	val32 = 0;
	memcpy(buf + TU104_GSCI_OFF_engineCaps + 8, &val32, 4);

	/* poisonFuseEnabled (offset 1216, 1 byte) */
	buf[TU104_GSCI_OFF_poisonFuseEnabled] = 0;

	/* fb_length (offset 1224, 8 bytes) - this mdev's emulated FB size.
	 * Kept in lockstep with NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE and
	 * wpr2_addr_lo (all derive from nv_vgpu_fb_emul_size). */
	val64 = fb_size;
	memcpy(buf + TU104_GSCI_OFF_fb_length, &val64, 8);

	/* fbio_mask (offset 1232, 8 bytes) - 8 FBIOs for 256-bit bus */
	val64 = 0xFF;
	memcpy(buf + TU104_GSCI_OFF_fbio_mask, &val64, 8);

	/* fb_bus_width (offset 1240, 4 bytes) - 256-bit */
	val32 = 256;
	memcpy(buf + TU104_GSCI_OFF_fb_bus_width, &val32, 4);

	/* fb_ram_type (offset 1244, 4 bytes) - GDDR6 = 6 */
	val32 = 6;
	memcpy(buf + TU104_GSCI_OFF_fb_ram_type, &val32, 4);

	/* fbp_mask (offset 1248, 8 bytes) - 6 FBPs for 6 GPCs */
	val64 = 0x3F;
	memcpy(buf + TU104_GSCI_OFF_fbp_mask, &val64, 8);

	/* l2_cache_size (offset 1256, 4 bytes) - 4MB */
	val32 = 0x400000;
	memcpy(buf + TU104_GSCI_OFF_l2_cache_size, &val32, 4);

	/* gpuNameString (offset 1260, 64 bytes) */
	strncpy((char *)(buf + TU104_GSCI_OFF_gpuNameString),
		"NVIDIA GeForce RTX 2080 SUPER", 64);

	/* gpuShortNameString (offset 1324, 64 bytes) */
	strncpy((char *)(buf + TU104_GSCI_OFF_gpuShortNameString),
		"TU104", 64);

	/* gpuNameString_Unicode (offset 1388, 128 bytes) - skip for now */

	/* Branding booleans */
	buf[TU104_GSCI_OFF_bIsMobile] = 1;	/* Mobile/Max-Q */

	/* bar1PdeBase (offset 1536, 8 bytes) */
	val64 = 0;
	memcpy(buf + TU104_GSCI_OFF_bar1PdeBase, &val64, 8);

	/* bar2PdeBase (offset 1544, 8 bytes) */
	val64 = 0;
	memcpy(buf + TU104_GSCI_OFF_bar2PdeBase, &val64, 8);

	/* bVbiosValid (offset 1552, 1 byte) */
	buf[TU104_GSCI_OFF_bVbiosValid] = 1;

	/* vbiosSubVendor (offset 1556, 4 bytes) - ASUS */
	val32 = 0x1043;
	memcpy(buf + TU104_GSCI_OFF_vbiosSubVendor, &val32, 4);

	/* vbiosSubDevice (offset 1560, 4 bytes) */
	val32 = 0x1F01;
	memcpy(buf + TU104_GSCI_OFF_vbiosSubDevice, &val32, 4);

	/* hInternalClient/Device/Subdevice */
	val32 = TU104_H_INTERNAL_CLIENT;
	memcpy(buf + TU104_GSCI_OFF_hInternalClient, &val32, 4);
	val32 = TU104_H_INTERNAL_DEVICE;
	memcpy(buf + TU104_GSCI_OFF_hInternalDevice, &val32, 4);
	val32 = TU104_H_INTERNAL_SUBDEVICE;
	memcpy(buf + TU104_GSCI_OFF_hInternalSubdevice, &val32, 4);

	/* ecidInfo (offset 1616, 24 bytes) - leave zeroed */

	/* fwWprLayoutOffset (offset 1640, 16 bytes): nonWprHeapOffset, frtsOffset */
	val64 = 0;
	memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset, &val64, 8);
	val64 = fb_size - 0x200000; /* frtsOffset = fbSize - 2MB */
	memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8);
}

/*
 * P2 static-config identity overlay: replace the emulated identity fields
 * (GPU name, short name, VBIOS valid/subvendor/subdevice) with the live
 * decode from the backend snapshot. FB geometry, masks, engine caps, PDE
 * bases and handles stay emulated in this phase. The wait is bounded and
 * sleeps; any miss leaves the emulated reply intact (fail-open).
 */
#define GSPRM_STATIC_LIVE_WAIT_MS	2000

static void gsp_rm_apply_static_overlay(struct gsp_emu_context *ctx, u8 *buf)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsp_phys_backend_static_config snap;
	u16 val16;
	int ret;

	if (!gsp_phys_static_live_enabled())
		return;

	ret = gsp_phys_backend_static_wait(&snap, GSPRM_STATIC_LIVE_WAIT_MS);
	if (ret) {
		rm->static_fallback++;
		if (ret == -ENODATA)
			rm->static_timeout++;
		return;
	}

	/* Wire slots: name 64 B at 1260, short name 64 B at 1324. An empty
	 * live string never clears the emulated one. */
	if (snap.gpu_name[0])
		strscpy((char *)(buf + TU104_GSCI_OFF_gpuNameString),
			snap.gpu_name,
			TU104_GSCI_OFF_gpuShortNameString -
				TU104_GSCI_OFF_gpuNameString);
	if (snap.gpu_short_name[0])
		strscpy((char *)(buf + TU104_GSCI_OFF_gpuShortNameString),
			snap.gpu_short_name,
			TU104_GSCI_OFF_gpuNameString_Unicode -
				TU104_GSCI_OFF_gpuShortNameString);

	/* Only overlay the VBIOS identity when the live decode carries a valid
	 * VBIOS; never clear an emulated valid flag. */
	if (snap.b_vbios_valid) {
		buf[TU104_GSCI_OFF_bVbiosValid] = 1;
		val16 = (u16)snap.vbios_sub_vendor;
		memcpy(buf + TU104_GSCI_OFF_vbiosSubVendor, &val16, sizeof(val16));
		val16 = (u16)snap.vbios_sub_device;
		memcpy(buf + TU104_GSCI_OFF_vbiosSubDevice, &val16, sizeof(val16));
	}

	rm->static_live++;
	nv_vgpu_info(ctx->vgpu,
		     "vgsp-rm: static-info overlay applied (generation=%u): name='%s' short='%s' vbiosValid=%u subVendor=0x%04x subDevice=0x%04x",
		     snap.generation, snap.gpu_name, snap.gpu_short_name,
		     snap.b_vbios_valid, (u16)snap.vbios_sub_vendor,
		     (u16)snap.vbios_sub_device);
}

static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx,
					   u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u8 *reply;

	reply = kzalloc(TU104_GSCI_SIZE, GFP_KERNEL);
	if (!reply) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GET_GSP_STATIC_INFO reply alloc failed");
		return;
	}

	gsp_rm_build_static_info(reply, nv_vgpu_fb_emul_size(ctx->vgpu));
	gsp_rm_apply_static_overlay(ctx, reply);

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)",
		     sequence, TU104_GSCI_SIZE,
		     (unsigned long long)(nv_vgpu_fb_emul_size(ctx->vgpu) >> 20),
		     0x60003E02, 0x362020);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO, sequence, 0,
			reply, TU104_GSCI_SIZE))
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GET_GSP_STATIC_INFO reply send failed");
	else
		rm->rpcs_replied++;

	kfree(reply);
}

/* ============================================================
 * GSP_RM_CONTROL (function 76) - sub-command dispatch
 *
 * rpc_gsp_rm_control_v03_00: hClient(4) + hObject(4) + cmd(4) + status(4)
 *                            + paramsSize(4) + flags(4) + params[]
 * ============================================================ */

struct gsp_rm_control_req {
	u32 hClient;
	u32 hObject;
	u32 cmd;
	u32 status;
	u32 paramsSize;
	u32 flags;
	/* params[] follows */
};

#define GSP_RM_CTRL_HDR_SIZE	24
static void gsp_rm_handle_fifo_device_table(struct gsp_emu_context *ctx,
					     u32 sequence,
					     struct gsp_rm_control_req *req,
					     const u8 *req_params,
					     u32 *out_status,
					     u8 *out_params, u32 *out_params_size)
{
	struct tu104_fifo_get_device_info_params *fifo_params;
	u32 base, count, i, copy_count;

	fifo_params = (struct tu104_fifo_get_device_info_params *)out_params;
	base = 0;
	/* baseIndex is the first field of the request params, which live in
	 * the queue element buffer (NOT after the local header struct!). Only
	 * readable when the whole request fits in the first element. */
	if (req->paramsSize >= 4 &&
	    req->paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA)
		memcpy(&base, req_params, 4);
	if (base >= TU104_FIFO_TABLE_MAX_DEVICES)
		base = 0;

	/* How many entries to return in this page */
	count = TU104_ENGINE_TABLE_SIZE - base;
	if (count > TU104_FIFO_TABLE_MAX_ENTRIES)
		count = TU104_FIFO_TABLE_MAX_ENTRIES;

	copy_count = count;
	if (base >= TU104_ENGINE_TABLE_SIZE)
		copy_count = 0;

	fifo_params->baseIndex = base;
	fifo_params->numEntries = copy_count;
	fifo_params->bMore = (base + copy_count) < TU104_ENGINE_TABLE_SIZE;

	for (i = 0; i < copy_count; i++)
		memcpy(&fifo_params->entries[i], &tu104_engine_table[base + i],
		       sizeof(struct tu104_fifo_device_entry));

	*out_params_size = offsetof(struct tu104_fifo_get_device_info_params,
				     entries) +
			   copy_count * sizeof(struct tu104_fifo_device_entry);
	*out_status = 0; /* NV_OK */

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: FIFO_GET_DEVICE_INFO_TABLE base=%u count=%u more=%u",
		     base, copy_count, fifo_params->bMore);
}

static void gsp_rm_handle_intr_kernel_table(struct gsp_emu_context *ctx,
					     u32 sequence,
					     u32 *out_status,
					     u8 *out_params, u32 *out_params_size)
{
	struct tu104_intr_get_kernel_table_params *intr_params;
	u32 copy_size;

	intr_params = (struct tu104_intr_get_kernel_table_params *)out_params;

	intr_params->tableLen = TU104_INTR_KERNEL_TABLE_SIZE;
	copy_size = TU104_INTR_KERNEL_TABLE_SIZE *
		    sizeof(struct tu104_intr_table_entry);
	memcpy(intr_params->table, tu104_intr_kernel_table, copy_size);

	/* Subtree map - must match intrInitSubtreeMap_TU102 (the values
	 * physical-path guests receive from real GSP-RM):
	 *   ESCHED_DRIVEN_ENGINE            = {3,3} (STALL_SUBTREE)
	 *   ESCHED_DRIVEN_ENGINE_NOTIF      = {0,0}
	 *   UVM_OWNED                       = {1,1} (fault 64, access cntr 72)
	 *   UVM_SHARED                      = {2,2} (doorbells/GSP/DISP)
	 *   DEFAULT/RUNLIST/RUNLIST_NOTIF   = INVALID (unset on Turing)
	 * intrCacheIntrFields requires accessCntrSubtree(=1) within
	 * UVM_OWNED and UVM_SHARED to be a single subtree. */
	{
		u32 c;

		for (c = 0; c < TU104_INTR_CATEGORY_ENUM_COUNT; c++) {
			intr_params->subtreeMap[c].subtreeStart = TU104_INTR_INVALID_SUBTREE;
			intr_params->subtreeMap[c].subtreeEnd   = TU104_INTR_INVALID_SUBTREE;
		}
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeStart = 3;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeEnd   = 3;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeStart = 0;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeEnd   = 0;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeStart = 1;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeEnd   = 1;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeStart = 2;
		intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeEnd   = 2;
	}

	/* The driver reads subtreeMap at offset 2052 (after the full
	 * 128-entry array), so the reply must span the whole struct -
	 * not just the used entries. */
	*out_params_size = offsetof(struct tu104_intr_get_kernel_table_params,
				     subtreeMap) +
			   sizeof(intr_params->subtreeMap);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: INTR_GET_KERNEL_TABLE tableLen=%u",
		     TU104_INTR_KERNEL_TABLE_SIZE);
}

static void gsp_rm_handle_constructed_falcon_info(struct gsp_emu_context *ctx,
						   u32 sequence,
						   u32 *out_status,
						   u8 *out_params,
						   u32 *out_params_size)
{
	struct tu104_get_constructed_falcon_info_params *falcon_params;

	falcon_params = (struct tu104_get_constructed_falcon_info_params *)out_params;

	falcon_params->numConstructedFalcons = TU104_FALCON_TABLE_SIZE;
	memcpy(falcon_params->constructedFalconsTable, tu104_falcon_table,
	       TU104_FALCON_TABLE_SIZE *
	       sizeof(struct tu104_constructed_falcon_info));

	*out_params_size = offsetof(struct tu104_get_constructed_falcon_info_params,
				     constructedFalconsTable) +
			   TU104_FALCON_TABLE_SIZE *
			   sizeof(struct tu104_constructed_falcon_info);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: GET_CONSTRUCTED_FALCON_INFO numFalcons=%u",
		     TU104_FALCON_TABLE_SIZE);
}

static void gsp_rm_handle_internal_device_table(struct gsp_emu_context *ctx,
						 u32 sequence,
						 u32 *out_status,
						 u8 *out_params,
						 u32 *out_params_size)
{
	struct tu104_internal_get_device_info_params *dev_params;
	u32 copy_size;

	dev_params = (struct tu104_internal_get_device_info_params *)out_params;

	dev_params->numEntries = TU104_INTERNAL_DEVICE_TABLE_SIZE;
	copy_size = TU104_INTERNAL_DEVICE_TABLE_SIZE *
		    sizeof(struct tu104_internal_device_info);
	memcpy(dev_params->deviceInfoTable, tu104_internal_device_table,
	       copy_size);

	*out_params_size = offsetof(struct tu104_internal_get_device_info_params,
				     deviceInfoTable) + copy_size;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: INTERNAL_GET_DEVICE_INFO_TABLE numEntries=%u",
		     TU104_INTERNAL_DEVICE_TABLE_SIZE);
}

/*
 * INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP (0x20800a41)
 *
 * Returns a compressed register access map telling the guest which BAR0
 * registers are userspace-accessible. For our emulated GPU, return an
 * empty map (no compressed data, no profiling ranges).
 */
static void gsp_rm_handle_user_register_access_map(struct gsp_emu_context *ctx,
						    u32 sequence,
						    u32 *out_status,
						    u8 *out_params,
						    u32 *out_params_size)
{
	/* NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS:
	 *   userRegisterAccessMapSize(4) + compressedSize(4) +
	 *   compressedData[4096] + profilingRangesSize(4) + profilingRanges[4096]
	 * Total: 8204 bytes
	 */
	u32 val32 = 0;

	memset(out_params, 0, 8204);
	memcpy(out_params + 0, &val32, 4);		/* userRegisterAccessMapSize */
	memcpy(out_params + 4, &val32, 4);		/* compressedSize */
	/* compressedData[4096] = zeros */
	memcpy(out_params + 4104, &val32, 4);		/* profilingRangesSize */
	/* profilingRanges[4096] = zeros */

	*out_params_size = 8204;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: USER_REGISTER_ACCESS_MAP -> empty map");
}

/*
 * INTERNAL_FIFO_GET_NUM_CHANNELS (0x20800a61)
 *
 * The guest asks GSP-RM how many channels each runlist supports
 * (kfifoRunlistQueryNumChannels). Returning 0 trips the driver's
 * NV_ASSERT(numChannels > 0) and kdisp can't allocate its
 * clientChannelTable. Report the consumer-Turing standard 64.
 */
static void gsp_rm_handle_fifo_get_num_channels(struct gsp_emu_context *ctx,
						u32 sequence,
						u32 *out_status,
						u8 *out_params,
						u32 *out_params_size)
{
	u32 val32;

	/* params: runlistId(4, in) + numChannels(4, out) */
	val32 = TU104_FIFO_NUM_CHANNELS_PER_RUNLIST;
	memcpy(out_params + 4, &val32, 4);

	*out_params_size = 8;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: FIFO_GET_NUM_CHANNELS -> %u", val32);
}

/*
 * BUS_GET_PCI_BAR_INFO (0x20801803)
 *
 * The guest caches pciBarSizes[] from this call (kern_bus.c). Reporting
 * zeros leaves BAR1/BAR2 aperture lengths at 0. Report our emulated BAR
 * layout; index 2 is the instance-memory window convention
 * (BAR0+0x700000, 1MB - gpuPhysInstAddr fallback when PCI BAR2 is absent).
 */
static void gsp_rm_handle_bus_get_pci_bar_info(struct gsp_emu_context *ctx,
					       u32 sequence,
					       u32 *out_status,
					       u8 *out_params,
					       u32 *out_params_size)
{
	struct nv_vgpu_device *vgpu = ctx->vgpu;
	u32 count = 4;
	u32 i;
	u64 sizes[4];
	u64 offsets[4];

	sizes[0] = vgpu->bars[0].size;   offsets[0] = vgpu->bars[0].gpa;
	sizes[1] = vgpu->bars[1].size;   offsets[1] = vgpu->bars[1].gpa;
	sizes[2] = 0x100000;             offsets[2] = vgpu->bars[0].gpa + 0x700000;
	sizes[3] = vgpu->bars[3].size;   offsets[3] = vgpu->bars[3].gpa;

	memset(out_params, 0, 8 + 8 * 24);
	memcpy(out_params + 0, &count, 4);

	for (i = 0; i < count; i++) {
		u8 *e = out_params + 8 + i * 24;
		u32 flags = 0;
		u32 barSizeMB = (u32)(sizes[i] >> 20);

		memcpy(e + 0, &flags, 4);
		memcpy(e + 4, &barSizeMB, 4);
		memcpy(e + 8, &sizes[i], 8);
		memcpy(e + 16, &offsets[i], 8);
	}

	*out_params_size = 8 + 8 * 24;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: BUS_GET_PCI_BAR_INFO -> %u BARs (BAR1=%uMB)",
		     count, (u32)(sizes[1] >> 20));
}

/*
 * INTERNAL_DISPLAY_GET_STATIC_INFO (0x20800a01)
 *
 * kdispStateInitLocked caches numHeads/numDispChannels from this call and
 * allocates clientChannelTable[numDispChannels] - a zero reply makes the
 * allocation fail and kills RmInitNvDevice. The driver's wire struct is
 * 36 bytes (observed paramsSize): feHwSysCap(0) windowPresentMask(4)
 * bFbRemapperEnabled(8) numHeads(12) i2cPort(16) internalDispActiveMask(20)
 * embeddedDisplayPortMask(24) muxSupported(28) numDispChannels(32).
 * Report a minimal headless display: 1 head, 1 (core) channel.
 */
static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx,
						  u32 sequence,
						  u32 *out_status,
						  u8 *out_params,
						  u32 *out_params_size)
{
	u32 val32;

	memset(out_params, 0, 36);

	val32 = 1;	/* numHeads */
	memcpy(out_params + 12, &val32, 4);
	val32 = 1;	/* numDispChannels (core channel) */
	memcpy(out_params + 32, &val32, 4);

	*out_params_size = 36;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=1");
}

/*
 * INTERNAL_DISPLAY_GET_IP_VERSION (0x20800a4b)
 *
 * kdispStatePreInitLocked passes an UNINITIALIZED ctrlParams and feeds the
 * reply straight into gpuInitDispIpHal - it must be a real IP version.
 * Turing display is DISP v04.00 (DispIpHal halspec idx 10).
 */
static void gsp_rm_handle_display_get_ip_version(struct gsp_emu_context *ctx,
						 u32 sequence,
						 u32 *out_status,
						 u8 *out_params,
						 u32 *out_params_size)
{
	u32 val32 = TU104_DISPLAY_IP_VERSION;

	memcpy(out_params, &val32, 4);
	*out_params_size = 4;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: DISPLAY_GET_IP_VERSION -> 0x%x", val32);
}

/*
 * CE_GET_FAULT_METHOD_BUFFER_SIZE (0x20802a08)
 *
 * kchangrpInit allocates one fault method buffer per runqueue of this
 * size; a zero reply makes the allocation fail (NV_ERR_NO_MEMORY) and
 * kills channel-group creation. GV100 formula inherited by Turing:
 * 135 * (13*totalCes + 2) page-aligned; TU104 = 5 PCEs -> 12288.
 */
static void gsp_rm_handle_ce_get_fault_method_buffer_size(
						struct gsp_emu_context *ctx,
						u32 sequence,
						u32 *out_status,
						u8 *out_params,
						u32 *out_params_size)
{
	u32 val32 = TU104_CE_FAULT_METHOD_BUFFER_SIZE;

	memcpy(out_params, &val32, 4);
	*out_params_size = 4;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: CE_GET_FAULT_METHOD_BUFFER_SIZE -> %u", val32);
}

/*
 * CE_GET_PHYSICAL_CAPS (0x20802a07) and friends
 *
 * kceGetDeviceCaps (kernel_ce.c) fills the CE caps the client hands to UVM
 * from NV2080_CTRL_CMD_CE_GET_PHYSICAL_CAPS, which reaches GSP-RM. UVM's
 * channel manager rejects any CE without the SYSMEM cap for its
 * CPU_TO_GPU/GPU_TO_CPU channels ("Failed to initialize the channel manager:
 * NV_ERR_NOT_SUPPORTED" in uvm_gpu.c) -- with the previous zeroed reply every
 * CE was unusable and UVM_REGISTER_GPU failed with 0x56.
 *
 * params: { ceEngineType(4), capsTbl[2] } (NV2080_CTRL_CE_GET_CAPS_V2_PARAMS
 * layout). Turing LCEs are sysmem-capable, P2P-capable and not GRCE.
 */
static void gsp_rm_handle_ce_physical_caps(struct gsp_emu_context *ctx,
					   const u8 *req_params, u32 req_size,
					   u32 *out_status, u8 *out_params,
					   u32 *out_params_size)
{
	u32 ce_type = 0;

	if (req_size >= 4)
		memcpy(&ce_type, req_params, 4);

	memset(out_params, 0, 8);
	memcpy(out_params + 0, &ce_type, 4);

	if (ce_type >= TU104_ENGINE_TYPE_COPY0 &&
	    ce_type < TU104_ENGINE_TYPE_COPY0 + TU104_NUM_CES) {
		u32 idx = ce_type - TU104_ENGINE_TYPE_COPY0;

		out_params[4] = TU104_CE_CAPS0;
		out_params[5] = TU104_CE_CAPS1;

		/* The GR-synchronous CEs (GRCE) are the ones sharing the GR
		 * PBDMA. The client picks the last GRCE as its compute
		 * companion ("ceForCompute", rm_device.c) and every non-GRCE
		 * CE becomes an async CE. Measured on the same silicon: two
		 * GRCEs (host ASYNC_ENGINE_COUNT=3 of 5); with fewer the
		 * client's context-creation walk ends with an empty
		 * secondary-engine record and the engine RM alloc is issued
		 * with class 0. */
		if (TU104_IS_GRCE(idx))
			out_params[4] |= TU104_CE_CAPS0_GRCE;
	} else {
		*out_status = 0x56;	/* NV_ERR_NOT_SUPPORTED */
		*out_params_size = req_size;
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: CE_GET_PHYSICAL_CAPS type=0x%x -> NOT_SUPPORTED",
			     ce_type);
		return;
	}

	*out_params_size = req_size ? req_size : 8;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: CE_GET_PHYSICAL_CAPS type=0x%x -> caps=%02x %02x",
		     ce_type, out_params[4], out_params[5]);
}

/*
 * CE_GET_ALL_PHYSICAL_CAPS (0x20802a0b) / CE_GET_ALL_CAPS (0x20802a0a)
 *
 * params: NvU8 capsTbl[64][2] + NvU64 present (136 bytes). Report the five
 * Turing CEs as present with the same caps as the per-CE query.
 */
static void gsp_rm_handle_ce_all_caps(struct gsp_emu_context *ctx,
				      u32 *out_status, u8 *out_params,
				      u32 *out_params_size)
{
	u64 present = 0;
	u32 i;

	memset(out_params, 0, 136);
	for (i = 0; i < TU104_NUM_CES; i++) {
		out_params[i * 2 + 0] = TU104_CE_CAPS0;
		out_params[i * 2 + 1] = TU104_CE_CAPS1;
		if (TU104_IS_GRCE(i))
			out_params[i * 2 + 0] |= TU104_CE_CAPS0_GRCE;
		present |= 1ULL << i;
	}
	memcpy(out_params + 128, &present, 8);

	*out_params_size = 136;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: CE_GET_ALL(_PHYSICAL)_CAPS -> %u CEs present=0x%llx",
		     TU104_NUM_CES, present);
}

/*
 * CE_GET_CE_PCE_MASK (0x20802a02)
 *
 * params: { ceEngineType(4), pceMask(4) }. Each Turing LCE has its own PCE,
 * so every CE maps to exactly one PCE bit. UVM's queryCopyEngines stores this
 * in UvmGpuCopyEngineCaps.cePceMask.
 */
static void gsp_rm_handle_ce_pce_mask(struct gsp_emu_context *ctx,
				      const u8 *req_params, u32 req_size,
				      u32 *out_status, u8 *out_params,
				      u32 *out_params_size)
{
	u32 ce_type = 0, mask = 0;

	if (req_size >= 4)
		memcpy(&ce_type, req_params, 4);

	if (ce_type >= TU104_ENGINE_TYPE_COPY0 &&
	    ce_type < TU104_ENGINE_TYPE_COPY0 + TU104_NUM_CES)
		mask = 1u << (ce_type - TU104_ENGINE_TYPE_COPY0);

	memset(out_params, 0, 8);
	memcpy(out_params + 0, &ce_type, 4);
	memcpy(out_params + 4, &mask, 4);

	*out_params_size = req_size ? req_size : 8;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: CE_GET_CE_PCE_MASK type=0x%x -> 0x%x",
		     ce_type, mask);
}

/*
 * INTERNAL_GMMU_GET_STATIC_INFO (0x20800a59)
 *
 * The guest asks GSP-RM (us) for the GMMU static info; kgmmu. pStaticInfo->
 * replayableFaultBufferSize feeds kgmmuFaultBufferReplayableAllocate (called
 * from faultbufConstruct when UVM opens the device). Zero -> 0-page alloc ->
 * NV_ERR_NO_MEMORY -> faultbufConstruct fails and cuInit aborts.
 * params: replayableFaultBufferSize(4) + replayableShadowFaultBufferMetadata
 * Size(4) + nonReplayableFaultBufferSize(4) + nonReplayableShadowFaultBuffer
 * MetadataSize(4). Shadow metadata is only used with confidential computing,
 * so it stays 0.
 */
static void gsp_rm_handle_gmmu_get_static_info(struct gsp_emu_context *ctx,
					       u32 sequence,
					       u32 *out_status,
					       u8 *out_params,
					       u32 *out_params_size)
{
	u32 replay = TU104_GMMU_REPLAYABLE_FAULT_BUFFER_SIZE;
	u32 nonreplay = TU104_GMMU_NONREPLAYABLE_FAULT_BUFFER_SIZE;
	u32 zero = 0;

	memcpy(out_params + 0, &replay, 4);
	memcpy(out_params + 4, &zero, 4);
	memcpy(out_params + 8, &nonreplay, 4);
	memcpy(out_params + 12, &zero, 4);

	*out_params_size = 16;
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: GMMU_GET_STATIC_INFO -> replayFaultBuf=0x%x nonReplayFaultBuf=0x%x",
		     replay, nonreplay);
}

/*
 * STATIC_KGR_GET_GLOBAL_SM_ORDER (0x20800a22)
 *
 * kgrStaticInit copies globalSmOrder[grIdx] into the GR engine's static
 * info - init-critical (NV_CHECK_OK_OR_GOTO, cannot fail). Zeros leave
 * numSm=0 and the driver/nvidia-smi spin on empty topology. Report the
 * TU104 layout: 6 GPCs x 4 TPCs x 2 SMs = 48 SMs on GR0, engines 1-7 empty.
 * params: globalSmOrder[8] x (globalSmId[240]x12B + numSm(2) + numTpc(2)).
 */
static void gsp_rm_handle_kgr_get_global_sm_order(struct gsp_emu_context *ctx,
						  u32 sequence,
						  u32 *out_status,
						  u8 *out_params,
						  u32 *out_params_size)
{
	struct tu104_gr_global_sm_order *order =
		(struct tu104_gr_global_sm_order *)out_params; /* engine 0 */
	u32 idx = 0;
	u32 gpc, tpc, sm;

	memset(out_params, 0, TU104_GR_MAX_ENGINES *
	       sizeof(struct tu104_gr_global_sm_order));

	for (gpc = 0; gpc < TU104_NUM_GPC; gpc++) {
		for (tpc = 0; tpc < TU104_TPC_PER_GPC; tpc++) {
			for (sm = 0; sm < TU104_SM_PER_TPC; sm++) {
				struct tu104_gr_sm_id *e = &order->globalSmId[idx++];

				e->gpcId = gpc;
				e->localTpcId = tpc;
				e->localSmId = sm;
				e->globalTpcId = gpc * TU104_TPC_PER_GPC + tpc;
				e->virtualGpcId = gpc;
				e->migratableTpcId = e->globalTpcId;
			}
		}
	}
	order->numSm = TU104_NUM_SM;
	order->numTpc = TU104_NUM_TPC;

	*out_params_size = TU104_GR_MAX_ENGINES *
			   sizeof(struct tu104_gr_global_sm_order);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: KGR_GET_GLOBAL_SM_ORDER -> %u SMs, %u TPCs on GR0",
		     TU104_NUM_SM, TU104_NUM_TPC);
}

/*
 * STATIC_KGR_GET_INFO (0x20800a2a)
 *
 * Returns the GR infoList the driver caches as pGrInfo and reads by index -
 * critically MAX_SUBCONTEXT_COUNT (0x2c): zero makes the channel group's
 * numMax==0 and channel allocation asserts (numMax == numFree && numMax != 0
 * -> NV_ERR_INVALID_STATE -> "No devices"). Fill TU104 litter topology +
 * subcontext counts (MAX_VEID_COUNT_INIT = 64). engineInfo[8] x infoList[58]
 * of {index,data} = 3712 bytes; engine 0 = GR0.
 */
static void gsp_rm_handle_kgr_get_info(struct gsp_emu_context *ctx,
				       u32 sequence,
				       u32 *out_status,
				       u8 *out_params,
				       u32 *out_params_size)
{
	struct tu104_gr_static_info *info =
		(struct tu104_gr_static_info *)out_params; /* engine 0 */
	u32 i;

	memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES *
	       sizeof(struct tu104_gr_static_info));

	/* positional array: index field = position for every entry */
	for (i = 0; i < TU104_GR_INFO_MAX_SIZE; i++)
		info->infoList[i].index = i;

	/* TU104 litter topology: 6 GPC x 4 TPC x 2 SM = 48 SMs */
	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_GPCS].data = TU104_NUM_GPC;
	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_TPC_PER_GPC].data =
		TU104_GR_INFO_LITTER_NUM_TPC_PER_GPC;
	info->infoList[TU104_GR_INFO_IDX_GPU_CORE_COUNT].data = TU104_NUM_SM;
	info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_SM_PER_TPC].data = TU104_SM_PER_TPC;
	info->infoList[TU104_GR_INFO_IDX_FAMILY_MAX_TPC_PER_GPC].data = TU104_TPC_PER_GPC;

	/* Indices libcuda fetches via NV2080_CTRL_CMD_GR_GET_INFO when it
	 * builds the ucode-image clock table (GR_GET_INFO copies
	 * pGrInfo->infoList[index].data). Observed on real TU104: index 9
	 * -> 24 (devstate+0xe94 TPC count; 0 makes FUN_003e35f0 return
	 * NV_ERR_INVALID_ARGUMENT and cuCtxCreate fail 999), index 7 -> 6,
	 * index 0x17 -> 6 (devstate+0xb0/+0xb4 SM/TPC), index 0x22 -> 48. */
	info->infoList[TU104_GR_INFO_IDX_SHADER_PIPE_COUNT].data =
		TU104_GR_INFO_SHADER_PIPE_COUNT;
	info->infoList[TU104_GR_INFO_IDX_SHADER_PIPE_SUB_COUNT].data =
		TU104_GR_INFO_SHADER_PIPE_SUB_COUNT;
	info->infoList[TU104_GR_INFO_IDX_RT_CORE_COUNT].data =
		TU104_GR_INFO_RT_CORE_COUNT;

	/* subcontext (VEID) counts: 64 per hw */
	info->infoList[TU104_GR_INFO_IDX_MAX_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;
	info->infoList[TU104_GR_INFO_IDX_MAX_LEGACY_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;
	info->infoList[TU104_GR_INFO_IDX_MAX_PER_ENGINE_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT;

	*out_params_size = TU104_GR_INFO_MAX_ENGINES *
			   sizeof(struct tu104_gr_static_info);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: KGR_GET_INFO -> GR0 litter %uGPC/%uTPC/%uSM, maxSubctx=%u",
		     TU104_NUM_GPC, TU104_TPC_PER_GPC, TU104_NUM_SM,
		     TU104_MAX_SUBCONTEXT_COUNT);
}

/*
 * STATIC_KGR_GET_FLOORSPEEPING_MASKS (0x20800a26)
 *
 * Driver computes numGpcs from floorsweepingMasks.gpcMask; zero kills
 * krcWatchdogInit ("numGpcs > 0" assert -> RmInitAdapter bails). Fill
 * TU104's present-GPC/TPC masks: 6 GPCs (bits 0-5), 4 TPCs per GPC.
 */
static void gsp_rm_handle_kgr_get_floorsweeping_masks(
						struct gsp_emu_context *ctx,
						u32 sequence,
						u32 *out_status,
						u8 *out_params,
						u32 *out_params_size)
{
	struct tu104_gr_floorsweeping_masks *m =
		(struct tu104_gr_floorsweeping_masks *)out_params; /* engine 0 */
	u32 g;

	memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES *
	       sizeof(struct tu104_gr_floorsweeping_masks));

	m->gpcMask = (1U << TU104_NUM_GPC) - 1;		/* bits 0-5 = 6 GPCs */
	m->physGpcMask = m->gpcMask;
	m->physGfxGpcMask = m->gpcMask;
	m->numGfxTpc = TU104_NUM_TPC;
	for (g = 0; g < TU104_NUM_GPC; g++) {
		m->tpcMask[g] = (1U << TU104_TPC_PER_GPC) - 1;	/* 4 TPCs */
		m->tpcCount[g] = TU104_TPC_PER_GPC;
		m->mmuPerGpc[g] = 1;
		m->numPesPerGpc[g] = 1;
		m->zcullMask[g] = (1U << TU104_TPC_PER_GPC) - 1;
	}

	*out_params_size = TU104_GR_INFO_MAX_ENGINES *
			   sizeof(struct tu104_gr_floorsweeping_masks);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: KGR_GET_FLOORSWEEPING_MASKS -> gpcMask=0x%x tpc/gpc=%u",
		     m->gpcMask, TU104_TPC_PER_GPC);
}

/*
 * STATIC_KGR_GET_CONTEXT_BUFFERS_INFO (0x20800a32)
 *
 * kgraphicsInitializeDeferredStaticData() asks the physical RM (GSP-RM) for
 * the per-engine context buffer sizes; kgrctxAllocMainCtxBuffer() then sizes
 * the golden image ctx buffer from engine[GRAPHICS].size. The zeroed generic
 * reply left size == 0 -> a 0-page allocation -> NV_ERR_NO_MEMORY at
 * mem_desc.c:1353 (kgrctxAllocMainCtxBuffer -> kgraphicsCreateGoldenImageChannel).
 *
 * The real physical RM probes these from the ctxsw ucode
 * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and reports NV_U32_MAX
 * for unsupported engine IDs. The client skips NV_U32_MAX entries in
 * kgraphicsCreateGoldenImageChannel's VA reservation, so the emulated layout
 * below mirrors that: page-aligned scratch sizes for the buffers the client
 * allocates, NV_U32_MAX for the engine IDs Turing does not support.
 */
static void gsp_rm_handle_kgr_get_context_buffers_info(
					struct gsp_emu_context *ctx,
					u32 sequence,
					u32 *out_status,
					u8 *out_params,
					u32 *out_params_size)
{
	static const struct tu104_ctx_buffer_info sizes[TU104_CTX_ENGINE_ID_COUNT] = {
		[TU104_CTX_ENG_GRAPHICS] = { TU104_CTX_SIZE_MAIN, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_ZCULL] = { TU104_CTX_SIZE_ZCULL, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_PM] = { TU104_CTX_SIZE_PM, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_PREEMPT] = { TU104_CTX_SIZE_PREEMPT, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_SPILL] = { TU104_CTX_SIZE_SPILL, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_PAGEPOOL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_BETACB] = { TU104_CTX_SIZE_BETACB, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_RTV] = { TU104_CTX_SIZE_RTV, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_PATCH] = { TU104_CTX_SIZE_PATCH, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_BUNDLE_CB] = { TU104_CTX_SIZE_BUNDLE_CB, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_PAGEPOOL_GLOBAL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_ATTRIBUTE_CB] = { TU104_CTX_SIZE_ATTRIBUTE_CB, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_RTV_CB_GLOBAL] = { TU104_CTX_SIZE_RTV_CB_GLOBAL, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_FECS_EVENT] = { TU104_CTX_SIZE_FECS_EVENT, 0x1000 },
		[TU104_CTX_ENG_GRAPHICS_PRIV_ACCESS_MAP] = { TU104_CTX_SIZE_PRIV_ACCESS_MAP, 0x1000 },
	};
	struct tu104_gr_get_context_buffers_info_params *info =
		(struct tu104_gr_get_context_buffers_info_params *)out_params;
	u32 e, i;

	memset(out_params, 0, sizeof(*info));

	for (e = 0; e < TU104_CTX_GPU_MAX_ENGINES; e++) {
		for (i = 0; i < TU104_CTX_ENGINE_ID_COUNT; i++) {
			/* engine IDs the chip does not support report
			 * NV_U32_MAX (the client skips those entries) */
			if (sizes[i].size == 0) {
				info->engineContextBuffersInfo[e].engine[i].size = 0xFFFFFFFF;
				info->engineContextBuffersInfo[e].engine[i].alignment = 0xFFFFFFFF;
				continue;
			}
			info->engineContextBuffersInfo[e].engine[i].size = sizes[i].size;
			info->engineContextBuffersInfo[e].engine[i].alignment = sizes[i].alignment;
		}
	}

	*out_params_size = sizeof(*info);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: KGR_GET_CONTEXT_BUFFERS_INFO -> main ctx 0x%x, patch 0x%x, pm 0x%x (engines=%u x %u)",
		     TU104_CTX_SIZE_MAIN, TU104_CTX_SIZE_PATCH, TU104_CTX_SIZE_PM,
		     TU104_CTX_GPU_MAX_ENGINES, TU104_CTX_ENGINE_ID_COUNT);
}

/*
 * FB_GET_INFO_V2 (0x20801303)
 *
 * The client (kern_mem_sys_ctrl.c _fbGetFbInfos) sends a list of FB info
 * indices to the physical RM and expects the same indices back with data
 * populated. A zeroed reply trips "pFbInfos[i].index == ..." asserts and
 * leaves all FB info zero - CUDA/UVM read total/usable memory, BAR1 size,
 * L2/LTC topology from here. Echo the requested indices and fill in the
 * emulated TU104 values.
 */
static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
{
	u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu);

	switch (index) {
	case TU104_FB_INFO_IDX_TILE_REGION_COUNT:
		return 0;
	case TU104_FB_INFO_IDX_COMPRESSION_SIZE:
		return 0;
	case TU104_FB_INFO_IDX_PARTITION_COUNT:
		return 1;
	case TU104_FB_INFO_IDX_BAR1_SIZE:
		return (u32)fb;
	case TU104_FB_INFO_IDX_RAM_SIZE:
	case TU104_FB_INFO_IDX_TOTAL_RAM_SIZE:
	case TU104_FB_INFO_IDX_USABLE_RAM_SIZE:
		return (u32)fb;
	case TU104_FB_INFO_IDX_HEAP_SIZE:
	case TU104_FB_INFO_IDX_MAPPABLE_HEAP_SIZE:
		return (u32)fb;
	case TU104_FB_INFO_IDX_BUS_WIDTH:
		return 256;
	case TU104_FB_INFO_IDX_RAM_TYPE:
		return TU104_RAM_TYPE_GDDR6;
	case TU104_FB_INFO_IDX_BANK_COUNT:
		return 8;
	case TU104_FB_INFO_IDX_FB_TAX_SIZE_KB:
	case TU104_FB_INFO_IDX_LARGEST_FREE_REGION_SIZE_KB:
		return (u32)(fb >> 10);
	case TU104_FB_INFO_IDX_PARTITION_MASK:
		return 1;
	case TU104_FB_INFO_IDX_HEAP_FREE:
		return (u32)fb;
	case TU104_FB_INFO_IDX_RAM_LOCATION:
		return TU104_RAM_LOCATION_GPU_DEDICATED;
	case TU104_FB_INFO_IDX_FBP_COUNT:
		return 1;
	case TU104_FB_INFO_IDX_FBP_MASK:
		return 1;
	case TU104_FB_INFO_IDX_L2CACHE_SIZE:
		return TU104_L2CACHE_SIZE;
	case TU104_FB_INFO_IDX_MEMORYINFO_VENDOR_ID:
		return TU104_MEMORYINFO_VENDOR_ID_SAMSUNG;
	case TU104_FB_INFO_IDX_BAR1_AVAIL_SIZE:
	case TU104_FB_INFO_IDX_BAR1_MAX_CONTIG_AVAIL_SIZE:
		return (u32)fb;
	case TU104_FB_INFO_IDX_LTC_COUNT:
		return TU104_LTC_COUNT;
	case TU104_FB_INFO_IDX_LTS_COUNT:
		return TU104_LTS_COUNT;
	case TU104_FB_INFO_IDX_PSEUDO_CHANNEL_MODE:
		return TU104_PSEUDO_CHANNEL_MODE_DISABLED;
	case TU104_FB_INFO_IDX_ALLOW_PAGE_RETIREMENT:
		return 0;
	case TU104_FB_INFO_IDX_LTC_MASK:
		return (1U << TU104_LTC_COUNT) - 1;
	default:
		return 0;
	}
}

static void gsp_rm_handle_fb_get_info_v2(struct gsp_emu_context *ctx,
					 const u8 *req_params,
					 u32 req_params_size,
					 u32 *out_status,
					 u8 *out_params,
					 u32 *out_params_size)
{
	struct tu104_fb_get_info_v2_params *reply =
		(struct tu104_fb_get_info_v2_params *)out_params;
	u32 count = 0;
	u32 i;

	if (req_params_size >= sizeof(count))
		memcpy(&count, req_params, sizeof(count));
	if (count > TU104_FB_INFO_MAX_LIST_SIZE)
		count = TU104_FB_INFO_MAX_LIST_SIZE;

	memset(out_params, 0, sizeof(*reply));
	reply->fbInfoListSize = count;

	for (i = 0; i < count; i++) {
		const u8 *in = req_params + sizeof(count) +
			       i * sizeof(struct tu104_fb_info_entry);
		u32 index = 0;

		if (sizeof(count) + (i + 1) * sizeof(struct tu104_fb_info_entry) <=
		    req_params_size)
			memcpy(&index, in, sizeof(index));

		reply->fbInfoList[i].index = index;
		reply->fbInfoList[i].data = tu104_fb_info_data(ctx, index);
	}

	*out_params_size = sizeof(*reply);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: FB_GET_INFO_V2 -> %u entries populated (first idx=0x%x)",
		     count, count ? reply->fbInfoList[0].index : 0);
}

/*
 * CLK legacy domain queries (0x20809001 / 0x20809009 / 0x20809064)
 *
 * libcuda's clock-domain helper (libcuda+0xe8c410) walks these three queries
 * before it will report clock/ECC/perf data. It requires:
 *   - 0x20809009: bits 32 and 34 set in the 8-byte reply (domain presence);
 *   - 0x20809064: non-zero domain entries at reply offsets 12, 20, 28, 36;
 *   - 0x20809001: a selector mask overlapping those entries.
 * With the generic zero reply the helper returns CUDA_ERROR_NOT_FOUND, which
 * makes the dependent device-attribute queries (ECC_ENABLED = attr 36,
 * TOTAL_CONSTANT_MEMORY = attr 13) abort device initialization with
 * cudaErrorInitializationError. Populate only the fields the checks read;
 * clock rates stay zero (no fabricated frequencies).
 */
static void gsp_rm_handle_clk_domain_info(struct gsp_emu_context *ctx, u32 cmd,
					  u32 req_size,
					  u32 *out_status,
					  u8 *out_params,
					  u32 *out_params_size)
{
	u32 n = req_size ? req_size : 8;
	u32 one = 1;

	if (n > 4096)
		n = 4096;
	memset(out_params, 0, n);

	if (cmd == TU104_CTRL_CLK_DOMAIN_MASK) {
		u64 mask = (1ULL << 32) | (1ULL << 34);

		if (n >= sizeof(mask))
			memcpy(out_params, &mask, sizeof(mask));
	} else if (cmd == TU104_CTRL_CLK_DOMAIN_DATA) {
		if (n >= 16)
			memcpy(out_params + 12, &one, sizeof(one));
		if (n >= 24)
			memcpy(out_params + 20, &one, sizeof(one));
		if (n >= 32)
			memcpy(out_params + 28, &one, sizeof(one));
		if (n >= 40)
			memcpy(out_params + 36, &one, sizeof(one));
	} else { /* TU104_CTRL_CLK_DOMAIN_SEL */
		if (n >= sizeof(one))
			memcpy(out_params, &one, sizeof(one));
	}

	*out_params_size = n;
	*out_status = 0;

	if (gsp_sema_dbg)
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: CLK domain info cmd=0x%x size=%u",
			     cmd, n);
}

/*
 * BUS_GET_INFO_V2 (0x20801823)
 *
 * Same flat index/data list pattern as FB_GET_INFO_V2. UVM's
 * gpuGetSysmemLinkInfo() asks for SYSMEM_CONNECTION_TYPE and then
 * PCIE_GPU_LINK_CAPS to compute the sysmem link bandwidth; a zeroed reply
 * makes getPCIELinkRateMBps() return NV_ERR_INVALID_STATE ("Unsupported
 * sysmem connection type"). Report a PCIe Gen3 x16 link for the emulated
 * vGPU and echo every requested index.
 */
static u32 tu104_bus_info_data(u32 index)
{
	switch (index) {
	case TU104_BUS_INFO_IDX_TYPE:
		return TU104_BUS_INFO_TYPE_PCI_EXPRESS;
	case TU104_BUS_INFO_IDX_INTLINE:
		return 0;
	case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CAPS:
	case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CAPS:
	case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CAPS:
	case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CAPS:
	case TU104_BUS_INFO_IDX_PCIE_GEN_INFO:
		return TU104_PCIE_LINK_CAP_GEN3_X16;
	case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CTRL_STATUS:
	case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CTRL_STATUS:
	case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CTRL:
	case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CTRL:
		return TU104_PCIE_LINK_CTRL_STATUS_GEN3_X16;
	case TU104_BUS_INFO_IDX_SYSMEM_CONNECTION_TYPE:
		return TU104_BUS_INFO_SYSMEM_CONN_PCIE;
	default:
		return 0;
	}
}

static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx,
					  const u8 *req_params,
					  u32 req_params_size,
					  u32 *out_status,
					  u8 *out_params,
					  u32 *out_params_size)
{
	struct tu104_bus_get_info_v2_params *reply =
		(struct tu104_bus_get_info_v2_params *)out_params;
	u32 count = 0;
	u32 i;

	if (req_params_size >= sizeof(count))
		memcpy(&count, req_params, sizeof(count));
	if (count > TU104_BUS_INFO_MAX_LIST_SIZE)
		count = TU104_BUS_INFO_MAX_LIST_SIZE;

	memset(out_params, 0, sizeof(*reply));
	reply->busInfoListSize = count;

	for (i = 0; i < count; i++) {
		const u8 *in = req_params + sizeof(count) +
			       i * sizeof(struct tu104_bus_info_entry);
		u32 index = 0;

		if (sizeof(count) + (i + 1) * sizeof(struct tu104_bus_info_entry) <=
		    req_params_size)
			memcpy(&index, in, sizeof(index));

		reply->busInfoList[i].index = index;
		reply->busInfoList[i].data = tu104_bus_info_data(index);
	}

	*out_params_size = sizeof(*reply);
	*out_status = 0;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: BUS_GET_INFO_V2 -> %u entries populated (first idx=0x%x)",
		     count, count ? reply->busInfoList[0].index : 0);
	for (i = 0; i < count; i++)
		nv_vgpu_info(ctx->vgpu, "vgsp-rm:   bus[%u] idx=0x%x data=0x%x",
			     i, reply->busInfoList[i].index,
			     reply->busInfoList[i].data);
}

/*
 * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106)
 *
 * The client tells the server (us) the physical addresses of its GPU page
 * table levels together with the VA range they cover:
 *
 *   struct {
 *       NvHandle hSubDevice;        // +0
 *       NvU32    subDeviceId;       // +4
 *       NvU64    pageSize;          // +8   (coverage of the reserved level)
 *       NvU64    virtAddrLo;        // +16
 *       NvU64    virtAddrHi;        // +24
 *       NvU32    numLevelsToCopy;   // +32
 *       NvU32    pad;               // +36
 *       struct {                    // +40, 24 bytes each, 6 max
 *           NvU64 physAddress;
 *           NvU64 size;
 *           NvU32 aperture;         // 1=sysmem, 2=vidmem
 *           NvU8  pageShift;
 *       } levels[6];
 *   } // 184 bytes
 *
 * This is the client's own page table geometry: remember it so client GPU VAs
 * can be translated to guest physical addresses (e.g. to complete emulated
 * GPU semaphore releases).
 */
#define GSPRM_COPY_PDES_LEVELS_OFF	40
#define GSPRM_COPY_PDES_LEVEL_SIZE	24
#define GSPRM_COPY_PDES_MAX_LEVELS	6

static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
					   const u8 *params, u32 params_size)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u64 va_lo = 0, va_hi = 0, page_size = 0;
	u32 num_levels = 0, i;

	if (params_size < GSPRM_COPY_PDES_LEVELS_OFF)
		return;

	memcpy(&page_size, params + 8, 8);
	memcpy(&va_lo, params + 16, 8);
	memcpy(&va_hi, params + 24, 8);
	memcpy(&num_levels, params + 32, 4);
	if (num_levels > GSPRM_COPY_PDES_MAX_LEVELS)
		num_levels = GSPRM_COPY_PDES_MAX_LEVELS;

	rm->pt.va_lo = va_lo;
	rm->pt.va_hi = va_hi;
	rm->pt.page_size = page_size;
	rm->pt.num_levels = num_levels;

	nv_vgpu_info(ctx->vgpu,
		     "vgsp-rm: COPY_SERVER_PDES va=[0x%llx..0x%llx] pageSize=0x%llx levels=%u",
		     va_lo, va_hi, page_size, num_levels);

	for (i = 0; i < num_levels; i++) {
		const u8 *lv = params + GSPRM_COPY_PDES_LEVELS_OFF +
			       i * GSPRM_COPY_PDES_LEVEL_SIZE;
		u64 phys = 0, size = 0;
		u32 aperture = 0;
		u8 page_shift = 0;

		if (GSPRM_COPY_PDES_LEVELS_OFF +
		    (i + 1) * GSPRM_COPY_PDES_LEVEL_SIZE > params_size)
			break;

		memcpy(&phys, lv + 0, 8);
		memcpy(&size, lv + 8, 8);
		memcpy(&aperture, lv + 16, 4);
		memcpy(&page_shift, lv + 20, 1);

		rm->pt.levels[i].phys = phys;
		rm->pt.levels[i].size = size;
		rm->pt.levels[i].aperture = aperture;
		rm->pt.levels[i].page_shift = page_shift;

		nv_vgpu_info(ctx->vgpu,
			     "vgsp-rm:   pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u",
			     i, phys, size, aperture, page_shift);
	}
}

static bool gsp_rm_alloc_interesting(u32 class_id);

/*
 * Channel (GPFIFO) classes across generations. Their allocation parameters
 * carry the GPFIFO GPU VA/entry count and the guest-physical USERD/instance
 * blocks, which is what the emulated-GPU progress service needs to run the
 * work the guest submits on the channel.
 */
static bool gsp_rm_is_channel_class(u32 class_id)
{
	switch (class_id) {
	case 0x906f:	/* MAXWELL_CHANNEL_GPFIFO_A */
	case 0xa06f:	/* KEPLER_CHANNEL_GPFIFO_A */
	case 0xb06f:	/* PASCAL_CHANNEL_GPFIFO_A */
	case 0xc06f:	/* VOLTA_CHANNEL_GPFIFO_A */
	case 0xc46f:	/* TURING_CHANNEL_GPFIFO_A */
		return true;
	default:
		return false;
	}
}

/* NV_CHANNEL_ALLOC_PARAMS offsets (alloc/alloc_channel.h) */
#define GSPRM_CHAN_GPFIFO_OFFSET	8
#define GSPRM_CHAN_GPFIFO_ENTRIES	16
#define GSPRM_CHAN_ENGINE_TYPE		128
#define GSPRM_CHAN_CID			132
#define GSPRM_CHAN_INSTANCE_MEM		144
#define GSPRM_CHAN_USERD_MEM		168
#define GSPRM_CHAN_MEM_APERTURE		16	/* offset of addressSpace in the desc */
#define GSPRM_CHAN_NOTIFIER_MEM		248

static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
				    const u8 *params, u32 params_size)
{
	struct sema_channel_desc desc;

	if (params_size < GSPRM_CHAN_NOTIFIER_MEM + 8)
		return;

	memset(&desc, 0, sizeof(desc));
	desc.h_object = h_object;
	memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8);
	memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4);
	memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4);
	memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4);
	memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8);
	memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8);
	memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM +
	       GSPRM_CHAN_MEM_APERTURE, 4);
	memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8);
	memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM +
	       GSPRM_CHAN_MEM_APERTURE, 4);

	gsp_sema_service_channel_add(&desc);
}

/*
 * GSP_RM_ALLOC (103) - rpc_gsp_rm_alloc_v03_00 (32 byte header) followed by
 * the class-specific allocation parameters.
 *
 * The reply MUST echo the request payload back: the client copies the reply
 * params over the caller's params buffer using the class-expected size (not
 * the size the caller reserved), so replying with zeroed params smashes
 * whatever sits above the caller's params buffer. The alloc params are
 * in/out for the classes we do not model in detail, so echoing the input is
 * both the safe and the correct "unchanged" answer. status (offset 16) is
 * the only field we deliberately rewrite (to NV_OK).
 *
 * NOTE: the RPC header 'length' only counts the fixed struct size, so it
 * must NOT be used to bound the params - read the paramsSize field from the
 * alloc header instead (see handoff PART 6).
 */
static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
				   const u8 *elem, u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
		      sizeof(struct gsprm_rpc_header);
	const u8 *params = p + GSPRM_ALLOC_HDR_SIZE;
	u32 hc = 0, hp = 0, ho = 0, class_id = 0;
	u32 params_size = 0, flags = 0;
	u32 echo_len, reply_len;
	u8 *reply;

	memcpy(&hc, p + 0, 4);
	memcpy(&hp, p + 4, 4);
	memcpy(&ho, p + 8, 4);
	memcpy(&class_id, p + 12, 4);
	memcpy(&params_size, p + 20, 4);
	memcpy(&flags, p + 24, 4);

	if (gsp_sema_dbg)
		nv_vgpu_info(ctx->vgpu,
			     "vgsp-rm: GSP_RM_ALLOC class=0x%x paramsSize=%u flags=0x%x hClient=0x%x hParent=0x%x hObject=0x%x",
			     class_id, params_size, flags, hc, hp, ho);

	/* Full param dump for memory/channel classes and any class we have not
	 * classified yet (rate-limited). %phN is capped at 64 bytes per print,
	 * so chunk it. */
	if (gsp_sema_dbg && gsp_rm_alloc_interesting(class_id) &&
	    rm->alloc_param_dumps < 60) {
		u32 shown = min(params_size, (u32)384);
		u32 off;

		rm->alloc_param_dumps++;
		for (off = 0; off < shown; off += 64) {
			u32 n = min(shown - off, (u32)64);

			nv_vgpu_info(ctx->vgpu,
				     "vgsp-rm:   alloc class=0x%x params[%u..%u]=%*phN",
				     class_id, off, off + n, (int)n,
				     params + off);
		}
	}

	/* Channels: hand the progress service everything it needs to run the
	 * work submitted on this channel (GPFIFO ring + USERD). */
	if (gsp_rm_is_channel_class(class_id))
		gsp_rm_register_channel(ctx, ho, params, params_size);

	/*
	 * NV01_EVENT (allocated with class 0x79 in the guest ABI; params are
	 * NV0005_ALLOC_PARAMETERS {hParentClient, hSrcResource, hClass,
	 * notifyIndex, data}). The driver allocates one per channel and waits
	 * for the GPU to post the event before it considers submitted work
	 * complete - remember them so completed work can be signalled back.
	 */
	if (class_id == 0x79 && params_size >= 24 && rm->events_n < GSPRM_MAX_EVENTS) {
		u32 h_src = 0, notify_index = 0, hclass = 0;
		u64 data64 = 0;
		int hc_off = -1, i;

		/*
		 * Locate hClass (0x79) inside the params instead of trusting a
		 * fixed offset: NV0005_ALLOC_PARAMETERS is
		 * {hParentClient, hSrcResource, hClass, notifyIndex, data},
		 * with hSrcResource immediately before hClass.
		 */
		for (i = 0; i + 4 <= (int)params_size; i += 4) {
			u32 v;

			memcpy(&v, params + i, 4);
			if (v == 0x79) {
				hc_off = i;
				break;
			}
		}
		if (hc_off >= 4) {
			memcpy(&h_src, params + hc_off - 4, 4);
			memcpy(&hclass, params + hc_off, 4);
			if (hc_off + 8 <= (int)params_size)
				memcpy(&notify_index, params + hc_off + 4, 4);
			if (hc_off + 16 <= (int)params_size)
				memcpy(&data64, params + hc_off + 8, 8);
		} else {
			memcpy(&h_src, params + 4, 4);
			memcpy(&notify_index, params + 12, 4);
			memcpy(&data64, params + 16, 8);
		}
		rm->events[rm->events_n].h_src = h_src;
		rm->events[rm->events_n].h_event = ho;
		rm->events[rm->events_n].h_client = hc;
		rm->events[rm->events_n].notify_index = notify_index;
		rm->events[rm->events_n].data = (u32)data64;
		rm->events[rm->events_n].valid = true;
		rm->events_n++;
		if (gsp_sema_dbg)
			nv_vgpu_info(ctx->vgpu,
				     "vgsp-rm: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d) params=%*phN",
				     ho, h_src, notify_index, hc_off,
				     (int)min(params_size, (u32)24), params);
	}

	/* Echo the fixed alloc header + as many param bytes as are contiguous
	 * in the first queue element. */
	echo_len = min(params_size,
		       (u32)GSPRM_MAX_SINGLE_ELEM_DATA - GSPRM_ALLOC_HDR_SIZE);
	reply_len = GSPRM_ALLOC_HDR_SIZE + echo_len;

	reply = kmalloc(reply_len, GFP_KERNEL);
	if (!reply) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_RM_ALLOC reply alloc failed");
		return;
	}

	memcpy(reply, p, reply_len);
	memset(reply + 16, 0, 4);	/* status = NV_OK */

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_ALLOC, sequence, 0,
			reply, reply_len))
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_RM_ALLOC reply send failed");

	kfree(reply);
}

/* ============================================================
 * Missing RM_CONTROL handlers for 570.144 guest driver
 * ============================================================ */

static void gsp_rm_handle_memsys_l2_invalidate_evict(struct gsp_emu_context *ctx,
						     const u8 *req_params, u32 params_size,
						     u32 *out_status,
						     u8 *out_params, u32 *out_params_size)
{
	/* NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT (0x20800a6c)
	 * Input:  flags (NvU32)
	 * Output: none (just status)
	 * The guest driver invalidates/evicts L2 cache lines.
	 * For vGPU, we just acknowledge success - no actual L2 to invalidate.
	 */
	const u32 *flags = (const u32 *)req_params;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: MEMSYS_L2_INVALIDATE_EVICT flags=0x%x",
		     params_size >= 4 ? *flags : 0);

	*out_status = 0; /* NV_OK */
	*out_params_size = 0;
}

static void gsp_rm_handle_gr_get_fecs_trace_hw_enable(struct gsp_emu_context *ctx,
						      u32 sequence,
						      const u8 *req_params, u32 params_size,
						      u32 *out_status,
						      u8 *out_params, u32 *out_params_size)
{
	/* NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE (0x20800a38)
	 * Input:  grRouteInfo (aligned 8), bEnable (NvBool) - actually input is route info only
	 * Output: bEnable (NvBool) - returns current enable state
	 * The guest queries whether FECS context switch tracing is enabled.
	 * For vGPU, tracing is not supported, return disabled (0).
	 */
	struct {
		u64 grRouteInfo_base;
		u64 grRouteInfo_size;
		u64 grRouteInfo_alignment;
		u32 grRouteInfo_addressSpace;
		u32 grRouteInfo_cpuCacheAttrib;
		u8  bEnable;
	} *out = (void *)out_params;

	/* Input parsing (optional) - grRouteInfo is 28 bytes aligned to 8 = 32 bytes */
	/* struct NV2080_CTRL_INTERNAL_GR_FECS_TRACE_HW_ENABLE_PARAMS {
	 *     NV_DECLARE_ALIGNED(NV2080_CTRL_GR_ROUTE_INFO grRouteInfo, 8);
	 *     NvBool bEnable;
	 * };
	 * NV2080_CTRL_GR_ROUTE_INFO = { NvU64 base, size, alignment; NvU32 addressSpace, cpuCacheAttrib; }
	 */

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: GR_GET_FECS_TRACE_HW_ENABLE params_size=%u", params_size);

	/* Return disabled (0) - no FECS tracing on vGPU */
	memset(out, 0, sizeof(*out));
	out->bEnable = 0;

	*out_status = 0; /* NV_OK */
	*out_params_size = sizeof(*out);
}

static void gsp_rm_handle_system_map_shared_data(struct gsp_emu_context *ctx,
						 u32 sequence,
						 const u8 *req_params, u32 params_size,
						 u32 *out_status,
						 u8 *out_params, u32 *out_params_size)
{
	/* NV0073_CTRL_CMD_SYSTEM_MAP_SHARED_DATA (0x730151)
	 * Input:  memDescInfo (base, size, alignment, addressSpace, cpuCacheAttrib), subDeviceInstance, bMap
	 * Output: none (just status)
	 * The guest maps/unmaps shared system memory.
	 * For vGPU, we acknowledge success - no actual mapping needed.
	 */
	struct {
		u64 base;
		u64 size;
		u64 alignment;
		u32 addressSpace;
		u32 cpuCacheAttrib;
		u32 subDeviceInstance;
		u8  bMap;
	} __attribute__((packed)) *in = (void *)req_params;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: SYSTEM_MAP_SHARED_DATA base=0x%llx size=0x%llx subDev=%u bMap=%u",
		     in ? in->base : 0, in ? in->size : 0,
		     in ? in->subDeviceInstance : 0, in ? in->bMap : 0);

	*out_status = 0; /* NV_OK */
	*out_params_size = 0;
}

/*
 * NVA06C_CTRL_CMD_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS (0xa06c010a)
 *
 * Sent at channel-group allocation: each runqueue's CE fault method buffer
 * is handed to the GSP as a memdesc {base (GPU physical address), size,
 * alignment, addressSpace, cpuCacheAttrib} plus a bar2Addr, so the GSP can
 * bind them into the engine context. Keep the addresses (guest structures
 * we otherwise cannot enumerate) and log them once.
 *
 * Params layout (88 bytes): memdesc[0] @0, memdesc[1] @32 (each = u64 base,
 * u64 size, u64 alignment, u32 addressSpace, u32 cpuCacheAttrib),
 * bar2Addr[2] @64, numValidEntries @80.
 */
#define GSPRM_CTRL_PROMOTE_FAULT_METHOD_BUFFERS	0xa06c010a
#define GSPRM_FAULT_MBUFS_MAX			8
#define GSPRM_FAULT_MBUFS_PER			2

struct gsp_rm_fault_mbuf {
	u32	h_object;
	u32	h_client;
	u32	n;
	struct {
		u64	base;
		u64	size;
		u32	address_space;
	} e[GSPRM_FAULT_MBUFS_PER];
};

static struct gsp_rm_fault_mbuf gsp_rm_fault_mbufs[GSPRM_FAULT_MBUFS_MAX];
static u32 gsp_rm_fault_mbufs_n;

static void
gsp_rm_handle_promote_fault_method_buffers(struct gsp_emu_context *ctx,
					   const struct gsp_rm_control_req *req,
					   const u8 *p, u32 len)
{
	struct gsp_rm_fault_mbuf *m = NULL;
	u32 i, n = 0;

	if (len < 84)
		return;

	memcpy(&n, p + 80, 4);
	if (n > GSPRM_FAULT_MBUFS_PER)
		n = GSPRM_FAULT_MBUFS_PER;

	for (i = 0; i < gsp_rm_fault_mbufs_n; i++) {
		if (gsp_rm_fault_mbufs[i].h_object == req->hObject) {
			m = &gsp_rm_fault_mbufs[i];
			break;
		}
	}
	if (!m && gsp_rm_fault_mbufs_n < GSPRM_FAULT_MBUFS_MAX)
		m = &gsp_rm_fault_mbufs[gsp_rm_fault_mbufs_n++];
	if (!m)
		return;

	m->h_object = req->hObject;
	m->h_client = req->hClient;
	m->n = n;
	for (i = 0; i < n; i++) {
		memcpy(&m->e[i].base, p + 32 * i, 8);
		memcpy(&m->e[i].size, p + 32 * i + 8, 8);
		memcpy(&m->e[i].address_space, p + 32 * i + 24, 4);
		if (gsp_sema_dbg)
			nv_vgpu_info(ctx->vgpu,
				     "vgsp-rm: FAULT_METHOD_BUFFER hObject=0x%x hClient=0x%x buf[%u] base=0x%llx size=%llu ap=%u",
				     req->hObject, req->hClient, i,
				     m->e[i].base, m->e[i].size,
				     m->e[i].address_space);
	}
}

/*
 * Bounded logging for the P3 forward counters: the first four fallbacks (so a
 * session that cannot forward is visible) and every 64th event. Never called
 * when the parameter is off.
 */
static void gsp_rm_ctrl_forward_note(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u64 events = rm->ctrl_fwd_ok + rm->ctrl_fwd_fallback;

	if ((rm->ctrl_fwd_fallback && rm->ctrl_fwd_fallback <= 4) ||
	    (events && (events % 64) == 0))
		nv_vgpu_info(ctx->vgpu,
			     "vgsp-rm: RM_CONTROL forward: ok=%llu fallback=%llu timeout=%llu",
			     rm->ctrl_fwd_ok, rm->ctrl_fwd_fallback,
			     rm->ctrl_fwd_timeout);
}

/*
 * P3 read-only control forwarding (gsp_phys_ctrl_forward=1; default off).
 *
 * A small whitelist of side-effect-free queries is sent to the live backend
 * session with the guest's emulated internal handles translated to the
 * snapshot's adopted server handles; the server's echoed params become the
 * guest reply. Everything else - the vGPU-specific controls, anything
 * mutating, unknown IDs, an unreadable/absent snapshot, unmappable handles,
 * a multi-element request, a backend error/timeout or a malformed reply -
 * returns false and the emulated per-command handler runs unchanged
 * (fail-open).
 *
 * gsp_phys_backend_rpc() returns with gsp_phys_rm_lock released and the reply
 * in a kmalloc'd copy; the status queue is only written afterwards by
 * gsp_rm_send(), never while a backend wait is pending (doc 11 section 6.5).
 */
static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx,
				const struct gsp_rm_control_req *req,
				const u8 *req_params, u8 *reply,
				u32 reply_data_len, u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsp_phys_backend_static_config snap;
	struct gsp_rm_control_req fwd_hdr;
	u8 *fwd_req;
	void *rsp = NULL;
	u32 rsp_len = 0;
	u32 out_status;
	u32 out_params_size;
	int ret;

	/* Wave 1 whitelist: compile-time side-effect-free queries only. */
	switch (req->cmd) {
	case TU104_CTRL_BUS_GET_PCI_BAR_INFO:
	case TU104_CTRL_BUS_GET_INFO_V2:
	case TU104_CTRL_FB_GET_INFO_V2:
	case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO:
		break;
	default:
		return false;
	}

	/* The capture hands us only the first queue element, so a request that
	 * does not fit one element (paramsSize+24 <= GSPRM_MAX_SINGLE_ELEM_DATA)
	 * cannot be reassembled and stays emulated. */
	if (req->paramsSize > GSPRM_MAX_SINGLE_ELEM_DATA - GSP_RM_CTRL_HDR_SIZE)
		goto fallback;

	/* A valid snapshot implies a READY session and the adopted handles. */
	if (gsp_phys_backend_static_get(&snap))
		goto fallback;

	/* Translate the guest's emulated internal handles; any other handle
	 * domain stays emulated. */
	if (req->hClient != TU104_H_INTERNAL_CLIENT ||
	    (req->hObject != TU104_H_INTERNAL_DEVICE &&
	     req->hObject != TU104_H_INTERNAL_SUBDEVICE))
		goto fallback;

	fwd_req = kmalloc(GSP_RM_CTRL_HDR_SIZE + req->paramsSize, GFP_KERNEL);
	if (!fwd_req)
		goto fallback;
	memset(&fwd_hdr, 0, sizeof(fwd_hdr));
	fwd_hdr.hClient = snap.h_internal_client;
	fwd_hdr.hObject = req->hObject == TU104_H_INTERNAL_DEVICE ?
			  snap.h_internal_device : snap.h_internal_subdevice;
	fwd_hdr.cmd = req->cmd;
	fwd_hdr.paramsSize = req->paramsSize;
	fwd_hdr.flags = req->flags;
	memcpy(fwd_req, &fwd_hdr, GSP_RM_CTRL_HDR_SIZE);
	if (req->paramsSize)
		memcpy(fwd_req + GSP_RM_CTRL_HDR_SIZE, req_params,
		       req->paramsSize);

	ret = gsp_phys_backend_rpc(GSPRM_RPC_FUNC_GSP_RM_CONTROL, fwd_req,
				   GSP_RM_CTRL_HDR_SIZE + req->paramsSize,
				   &rsp, &rsp_len, 500);
	kfree(fwd_req);
	if (ret) {
		if (ret == -ETIMEDOUT)
			rm->ctrl_fwd_timeout++;
		goto fallback;
	}

	/*
	 * The reply is rpc_gsp_rm_control_v03_00: status at +12, paramsSize
	 * at +16, echoed params at +24 - the same 24-byte prefix the emulated
	 * handler builds below. The RPC-level rpc_result was already checked
	 * by gsp_phys_backend_rpc().
	 */
	if (!rsp || rsp_len < GSP_RM_CTRL_HDR_SIZE)
		goto fallback;
	memcpy(&out_status, (u8 *)rsp + 12, 4);
	memcpy(&out_params_size, (u8 *)rsp + 16, 4);
	if (out_status != 0 ||
	    out_params_size > rsp_len - GSP_RM_CTRL_HDR_SIZE ||
	    out_params_size > reply_data_len - GSP_RM_CTRL_HDR_SIZE)
		goto fallback;

	/* Build the guest reply exactly as the emulated handler does: the
	 * guest's own handles/cmd and the server's status/params. */
	memcpy(reply + 0, &req->hClient, 4);
	memcpy(reply + 4, &req->hObject, 4);
	memcpy(reply + 8, &req->cmd, 4);
	memcpy(reply + 12, &out_status, 4);
	memcpy(reply + 16, &out_params_size, 4);
	memcpy(reply + 20, &req->flags, 4);
	if (out_params_size)
		memcpy(reply + GSP_RM_CTRL_HDR_SIZE,
		       (u8 *)rsp + GSP_RM_CTRL_HDR_SIZE, out_params_size);
	kfree(rsp);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0,
			reply, GSP_RM_CTRL_HDR_SIZE + out_params_size)) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: RM_CONTROL forward reply send failed cmd=0x%x",
			    req->cmd);
		rm->ctrl_fwd_fallback++;
	} else {
		rm->ctrl_fwd_ok++;
	}
	gsp_rm_ctrl_forward_note(ctx);
	return true;

fallback:
	kfree(rsp);
	rm->ctrl_fwd_fallback++;
	gsp_rm_ctrl_forward_note(ctx);
	return false;
}

static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
				     const u8 *elem, u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE +
			     sizeof(struct gsprm_rpc_header);
	struct gsp_rm_control_req req;
	u32 reply_data_len;
	u8 *reply;
	u32 out_status = 0;
	u32 out_params_size = 0;
	u64 t0 = ktime_get_ns();

	memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE);

	if (gsp_sema_dbg)
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: GSP_RM_CONTROL cmd=0x%x hClient=0x%x hObject=0x%x paramsSize=%u flags=0x%x",
			     req.cmd, req.hClient, req.hObject, req.paramsSize,
			     req.flags);

	/* Allocate reply: control header + max possible params */
	reply_data_len = GSP_RM_CTRL_HDR_SIZE + max(req.paramsSize, (u32)4096);
	reply = kzalloc(reply_data_len, GFP_KERNEL);
	if (!reply) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: RM_CONTROL reply alloc failed");
		return;
	}

	/*
	 * P3: when enabled, forward a whitelist of read-only queries to the
	 * live backend. The helper builds and sends the reply itself; a false
	 * return means nothing was sent and the emulated switch below runs
	 * byte-identically to the parameter-off path.
	 */
	if (gsp_phys_ctrl_forward_enabled() &&
	    gsp_rm_ctrl_forward(ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE,
				reply, reply_data_len, sequence)) {
		kfree(reply);
		return;
	}

	switch (req.cmd) {
	case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE:
		gsp_rm_handle_fifo_device_table(ctx, sequence, &req,
			req_data + GSP_RM_CTRL_HDR_SIZE,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */
		gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE,
					       req.paramsSize);
		out_status = 0;
		out_params_size = 0;
		break;

	case GSPRM_CTRL_PROMOTE_FAULT_METHOD_BUFFERS:
		gsp_rm_handle_promote_fault_method_buffers(
			ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE,
			req.paramsSize);
		out_status = 0;
		out_params_size = req.paramsSize;
		break;

	case TU104_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE:
		gsp_rm_handle_intr_kernel_table(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO:
		gsp_rm_handle_constructed_falcon_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE:
		gsp_rm_handle_internal_device_table(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case 0x20800a41: /* INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP */
		gsp_rm_handle_user_register_access_map(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS:
		gsp_rm_handle_fifo_get_num_channels(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_BUS_GET_PCI_BAR_INFO:
		gsp_rm_handle_bus_get_pci_bar_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_CLK_DOMAIN_MASK:
	case TU104_CTRL_CLK_DOMAIN_DATA:
	case TU104_CTRL_CLK_DOMAIN_SEL:
		gsp_rm_handle_clk_domain_info(ctx, req.cmd, req.paramsSize,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO:
		gsp_rm_handle_display_get_static_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION:
		gsp_rm_handle_display_get_ip_version(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE:
		gsp_rm_handle_ce_get_fault_method_buffer_size(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_CE_GET_PHYSICAL_CAPS:
		gsp_rm_handle_ce_physical_caps(ctx,
			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_CE_GET_CE_PCE_MASK:
		gsp_rm_handle_ce_pce_mask(ctx,
			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_CE_GET_ALL_CAPS:
	case TU104_CTRL_CE_GET_ALL_PHYSICAL_CAPS:
		gsp_rm_handle_ce_all_caps(ctx,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO:
		gsp_rm_handle_gmmu_get_static_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_STATIC_KGR_GET_GLOBAL_SM_ORDER:
		gsp_rm_handle_kgr_get_global_sm_order(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_STATIC_KGR_GET_INFO:
		gsp_rm_handle_kgr_get_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO:
		gsp_rm_handle_kgr_get_context_buffers_info(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_FB_GET_INFO_V2:
		gsp_rm_handle_fb_get_info_v2(ctx,
			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_BUS_GET_INFO_V2:
		gsp_rm_handle_bus_get_info_v2(ctx,
			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS:
		gsp_rm_handle_kgr_get_floorsweeping_masks(ctx, sequence,
			&out_status,
			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT:
		gsp_rm_handle_memsys_l2_invalidate_evict(ctx,
			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
			&out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE:
		gsp_rm_handle_gr_get_fecs_trace_hw_enable(ctx, sequence,
			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
			&out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case TU104_CTRL_SYSTEM_MAP_SHARED_DATA:
		gsp_rm_handle_system_map_shared_data(ctx, sequence,
			req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
			&out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
		break;

	case 0x2080012b: { /* NV2080_CTRL_CMD_GPU_PROMOTE_CTX */
		/*
		 * The stock guest promotes its GR virtual contexts through this
		 * control during channel setup. The emulation answers SUCCESS
		 * (the physical backend owns execution), but the guest's entry
		 * shapes are the ground truth for what a real client sends:
		 * capture them (all fields, all entries) so the physical ladder
		 * can be driven with the exact same promote shape.
		 */
		const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE;
		u32 engine_type = 0, h_cli = 0, ch_id = 0, h_chan_cli = 0;
		u32 h_obj = 0, h_virt = 0, entry_count = 0, i, ecap;

		if (req.paramsSize >= 48) {
			memcpy(&engine_type, pp + 0, 4);
			memcpy(&h_cli, pp + 4, 4);
			memcpy(&ch_id, pp + 8, 4);
			memcpy(&h_chan_cli, pp + 12, 4);
			memcpy(&h_obj, pp + 16, 4);
			memcpy(&h_virt, pp + 20, 4);
			{
				u64 va = 0, sz = 0;

				memcpy(&va, pp + 24, 8);
				memcpy(&sz, pp + 32, 8);
				memcpy(&entry_count, pp + 40, 4);
				ecap = min_t(u32, entry_count, 16u);
				nv_vgpu_info(ctx->vgpu,
					     "vgsp-rm: PROMOTE_CTX engine=0x%x hClient=0x%x chId=%u hChanClient=0x%x hObject=0x%x hVirtMemory=0x%x va=0x%llx size=0x%llx entryCount=%u",
					     engine_type, h_cli, ch_id,
					     h_chan_cli, h_obj, h_virt, va,
					     sz, entry_count);
			}
			for (i = 0; i < ecap &&
			     48u + (i + 1u) * 32u <= req.paramsSize; i++) {
				const u8 *e = pp + 48 + i * 32;
				u64 pa = 0, eva = 0, esz = 0;
				u32 attr = 0;
				u16 bid = 0;

				memcpy(&pa, e + 0, 8);
				memcpy(&eva, e + 8, 8);
				memcpy(&esz, e + 16, 8);
				memcpy(&attr, e + 24, 4);
				memcpy(&bid, e + 28, 2);
				nv_vgpu_info(ctx->vgpu,
					     "vgsp-rm:   promote[%u] bufferId=%u bInitialize=%u bNonmapped=%u physAttr=0x%x pa=0x%llx va=0x%llx size=0x%llx",
					     i, bid, e[30], e[31], attr, pa,
					     eva, esz);
			}
		}
		out_status = 0;
		out_params_size = req.paramsSize;
		if (req.paramsSize &&
		    req.paramsSize + GSP_RM_CTRL_HDR_SIZE <=
				GSPRM_MAX_SINGLE_ELEM_DATA) {
			u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE;
			u32 init_entries = 0, k;

			/*
			 * PROMOTE_CTX is an in/out control: the client allocates
			 * one NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS, fills the
			 * entries that still need work (bInitialize=1 for a PA
			 * initialization; the prepare helpers also emit VA
			 * promotion entries), sends it, and reads the SAME
			 * buffer back after the call. The reply decides whether
			 * the client marks the ctx buffer initialized and stops
			 * re-promoting it (kernel_graphics_object.c,
			 * kgrobjPromoteContext_IMPL):
			 *
			 *   status = pRmApi->Control(..., &params, ...);
			 *   if (status == NV_OK) {
			 *       for (i = 0; i < entryCount; ++i) {
			 *           if (!params.promoteEntry[i].bInitialize)
			 *               continue;
			 *           kgrctxMarkCtxBufferInitialized(...,
			 *               params.promoteEntry[i].bufferId);
			 *       }
			 *   }
			 *
			 * The flag is therefore the server's "context buffer is
			 * initialized" acknowledgement: an entry the client
			 * asked to initialize must come back with bInitialize
			 * STILL SET, or the client skips the mark, re-prepares
			 * the same buffer on every pass and the context never
			 * becomes usable. The physical backend owns the actual
			 * residency and the emulation answers SUCCESS, so the
			 * correct reply is a verbatim echo - the request's
			 * flags already carry the values the client checks.
			 * (fa9296af briefly cleared the flag believing the
			 * check was inverted; that made the client re-promote
			 * forever. This is the corrected semantics.)
			 */
			memcpy(rp, pp, req.paramsSize);
			if (req.paramsSize >= 48) {
				u32 ec = 0;

				memcpy(&ec, pp + 40, 4);
				for (k = 0; k < ec &&
				     48u + (k + 1u) * 32u <= req.paramsSize;
				     k++) {
					if (rp[48 + k * 32 + 30])
						init_entries++;
				}
			}
			if (init_entries)
				nv_vgpu_info(ctx->vgpu,
					     "vgsp-rm: PROMOTE_CTX reply confirms %u initialize entr%s (bInitialize kept; client will mark ctx buffers initialized)",
					     init_entries,
					     init_entries == 1 ? "y" : "ies");
		}
		break;
	}

	default:
		/* Unknown control command - return success with zeroed params.
		 * Do NOT echo the request: pure queries pass uninitialized
		 * stack structs as input (e.g. DISPLAY_GET_IP_VERSION), so an
		 * echo hands the driver its own stack garbage as the answer. */
		if (gsp_sema_dbg)
			nv_vgpu_info(ctx->vgpu, "vgsp-rm: RM_CONTROL unknown cmd=0x%x -> generic SUCCESS",
				     req.cmd);
		/* Dump small request params so mapping/alloc controls that
		 * carry PAs can be identified (rate-limited). */
		if (gsp_sema_dbg && req.paramsSize && req.paramsSize <= 96 &&
		    rm->ctrl_param_dumps < 400) {
			rm->ctrl_param_dumps++;
			nv_vgpu_info(ctx->vgpu,
				     "vgsp-rm:   ctrl 0x%x params[0..%u]=%*phN",
				     req.cmd, req.paramsSize, req.paramsSize,
				     req_data + GSP_RM_CTRL_HDR_SIZE);
		}
		out_status = 0;
		out_params_size = req.paramsSize;
		break;

	case TU104_CTRL_I2C_TRANSACTION:
		/* I2C transactions have variable-length FINN-serialized
		 * params: an all-zero reply fails the driver's DeserializeUp
		 * and the calling ioctl never completes (nvidia-smi hang).
		 * Echo the request's serialized params (valid stream) and
		 * report NOT_SUPPORTED - headless vGPU has no DDC/I2C bus. */
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: I2C_TRANSACTION -> NOT_SUPPORTED (echo)");
		out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */
		out_params_size = req.paramsSize;
		if (req.paramsSize &&
		    req.paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA)
			memcpy(reply + GSP_RM_CTRL_HDR_SIZE,
			       req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize);
		break;
	}

	/* Build the reply header */
	memcpy(reply + 0, &req.hClient, 4);
	memcpy(reply + 4, &req.hObject, 4);
	memcpy(reply + 8, &req.cmd, 4);
	memcpy(reply + 12, &out_status, 4);
	memcpy(reply + 16, &out_params_size, 4);
	memcpy(reply + 20, &req.flags, 4);

	{
		u64 dt_us = div_u64(ktime_get_ns() - t0, 1000);

		if (gsp_sema_dbg)
			nv_vgpu_info(ctx->vgpu,
				     "vgsp-rm: RM_CONTROL reply cmd=0x%x hClient=0x%x hObject=0x%x in=%u out=%u status=0x%x us=%llu",
				     req.cmd, req.hClient, req.hObject,
				     req.paramsSize, out_params_size,
				     out_status, dt_us);
	}

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0,
			reply, GSP_RM_CTRL_HDR_SIZE + out_params_size))
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: RM_CONTROL reply send failed cmd=0x%x",
			    req.cmd);
	else
		rm->rpcs_replied++;

	kfree(reply);
}

/*
 * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK.
 *
 * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the
 * request's params back with status=0 so the driver's free rpc_message is a
 * complete well-formed record. (The poll match is function-only; the echo
 * keeps the post-poll result struct self-consistent.)
 */
static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
			       u32 sequence)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const struct gsprm_rpc_header *hdr =
		(const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
	const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr);
	u8 params[16]; /* NVOS00_PARAMETERS: hRoot, hObjectParent, hObjectOld, status */
	u32 hObjOld = 0, hRoot = 0;
	u32 removed = 0, i;

	if (hdr->length >= sizeof(*hdr) + sizeof(params))
		memcpy(params, req, sizeof(params));
	else
		memset(params, 0, sizeof(params));

	memcpy(&hRoot, params + 0, 4);
	memcpy(&hObjOld, params + 8, 4);
	memset(params + 12, 0, 4); /* status = NV_OK */

	gsp_sema_service_channel_del(hObjOld);

	/*
	 * Retire event records that refer to the object being freed. Posting an
	 * event for a freed (hClient, hEvent) pair makes the guest RM assert in
	 * CliGetEventInfo (kernel_gsp.c:466) and drop the record; a freed
	 * client's notifier events (hSrcResource 0) linger across runs and the
	 * fallback wake-up nudge would hit them. Also retire a client's events
	 * when its root object is freed (hObjOld == hRoot).
	 */
	for (i = 0; i < rm->events_n; i++) {
		if (!rm->events[i].valid)
			continue;
		if (rm->events[i].h_event == hObjOld ||
		    (hObjOld && hObjOld == hRoot &&
		     rm->events[i].h_client == hRoot)) {
			rm->events[i].valid = false;
			removed++;
		}
	}
	if (removed)
		nv_vgpu_info(ctx->vgpu,
			     "vgsp-rm: retired %u event record(s) on FREE hObjOld=0x%x hRoot=0x%x",
			     removed, hObjOld, hRoot);

	if (gsp_sema_dbg)
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u",
			     hObjOld, sequence, rm->cmd_read, rm->stat_write);

	/* Small delay before FREE/UNLOADING replies: mitigates an intermittent
	 * driver-side reply-matching race in the cleanup path. */
	udelay(2000);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_FREE, sequence, 0,
			params, sizeof(params)))
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: FREE reply send failed");
	else
		rm->rpcs_replied++;
}

/*
 * Handle one command-queue record (first element). For async init RPCs we
 * just consume. Synchronous RPCs get a reply; per-function handlers are
 * implemented iteratively (unknown ones get a generic success reply).
 */
static bool gsp_rm_alloc_interesting(u32 class_id)
{
	/* Boring classes already decoded: root/device/subdevice */
	switch (class_id) {
	case 0x00:	/* NV01_ROOT */
	case 0x80:	/* NV01_DEVICE_0 */
	case 0x2080:	/* NV20_SUBDEVICE_0 */
	case 0x2081:	/* NV2081_BINAPI */
		return false;
	default:
		return true;
	}
}

/*
 * SET_PAGE_DIRECTORY (54) / UNSET_PAGE_DIRECTORY (79)
 *
 * The driver builds the page tables of the GPU VA spaces it owns and hands the
 * top-level directory to the GPU here. The parameters are
 *   { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId;
 *     u32 subDeviceId; u32 pasid; }
 * (flags bits 1:0: 0 = vidmem aperture, 1 = sysmem coherent,
 * 2 = sysmem non-coherent).
 *
 * Recording the directory lets the progress service translate that VA space's
 * addresses directly (four-level walk), which is how the client's semaphore
 * waits are completed; it also removes the need for heuristic discovery.
 */
static void gsp_rm_handle_set_page_directory(struct gsp_emu_context *ctx,
					     const u8 *elem, u32 sequence,
					     u32 length)
{
	const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
		      sizeof(struct gsprm_rpc_header);
	struct sema_vaspace_desc desc;
	u64 phys = 0;
	u32 num_entries = 0, flags = 0, h_vas = 0, ch_id = 0, subdev = 0, pasid = 0;

	/*
	 * Wire layout: a 16-byte message header (hClient, hObject, ...)
	 * followed by the control's own parameters
	 * { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId;
	 *   u32 subDeviceId; u32 pasid; }.
	 */
	if (length >= 44) {
		memcpy(&phys, p + 16, 8);
		memcpy(&num_entries, p + 24, 4);
		memcpy(&flags, p + 28, 4);
		memcpy(&h_vas, p + 32, 4);
		memcpy(&ch_id, p + 36, 4);
		memcpy(&subdev, p + 40, 4);
		memcpy(&pasid, p + 44, 4);
	}

	{
		char dump[200];
		int o = 0, i;
		u32 dlen = min_t(u32, length, 80);

		for (i = 0; i < (int)dlen; i += 4) {
			u32 v = 0;
			memcpy(&v, p + i, 4);
			o += snprintf(dump + o, sizeof(dump) - o, "%08x ",
				      v);
		}
		nv_vgpu_info(ctx->vgpu,
			     "vgsp-rm: SET_PAGE_DIRECTORY len=%u payload: %s",
			     length, dump);
	}
	nv_vgpu_info(ctx->vgpu,
		     "vgsp-rm: SET_PAGE_DIRECTORY pdb=0x%llx entries=%u flags=0x%x hVASpace=0x%x chId=0x%x subDev=%u pasid=%u len=%u",
		     phys, num_entries, flags, h_vas, ch_id, subdev, pasid,
		     length);

	if (phys && num_entries) {
		memset(&desc, 0, sizeof(desc));
		desc.h_vaspace = h_vas;
		desc.pdb = phys;
		desc.num_entries = num_entries;
		desc.b_vidmem = (flags & 0x3) == 0;
		gsp_sema_service_vaspace_add(&desc);

		/* Show the first entries of the directory: they are the ground
		 * truth for the walk. */
		{
			u64 peek[4] = {0};
			char b[128];
			int i, o = 0, rc = -ENODEV;

			if (desc.b_vidmem) {
				u8 *base = ctx->vgpu->bars[1].vaddr;

				/*
				 * Bound against the emulated vidmem shadow
				 * size, not the physical BAR1 aperture: vidmem
				 * offsets above the aperture (e.g. 262M in the
				 * 512M turing type) are legitimate (ISSUE-0002
				 * r75b).
				 */
				if (base && phys + sizeof(peek) <=
					    nv_vgpu_fb_emul_size(ctx->vgpu)) {
					memcpy(peek, base + phys, sizeof(peek));
					rc = 0;
				}
			} else {
				rc = gsp_rm_read_dev(ctx->vgpu, phys, peek,
						     sizeof(peek));
			}
			if (!rc) {
				for (i = 0; i < 4; i++)
					o += snprintf(b + o, sizeof(b) - o,
						      "%016llx ", peek[i]);
				nv_vgpu_info(ctx->vgpu,
					     "vgsp-rm:   directory[0..3]: %s", b);
			}
		}
	}

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY, sequence, 0,
			NULL, 0))
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: SET_PAGE_DIRECTORY reply send failed");
}

static void gsp_rm_handle_unset_page_directory(struct gsp_emu_context *ctx,
					       const u8 *elem, u32 sequence,
					       u32 length)
{
	const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
		      sizeof(struct gsprm_rpc_header);
	u32 h_vas = 0;

	if (length >= 4)
		memcpy(&h_vas, p, 4);

	nv_vgpu_info(ctx->vgpu,
		     "vgsp-rm: UNSET_PAGE_DIRECTORY hVASpace=0x%x len=%u",
		     h_vas, length);
	gsp_sema_service_vaspace_del(h_vas);

	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY, sequence, 0,
			NULL, 0))
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: UNSET_PAGE_DIRECTORY reply send failed");
}

static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	const struct gsprm_rpc_header *hdr =
		(const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
	u32 function = hdr->function;
	u32 sequence = hdr->sequence;

	rm->rpcs_seen++;

	switch (function) {
	case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO:
	case GSPRM_RPC_FUNC_SET_REGISTRY:
		/* Async init RPCs - fire and forget, no reply expected.
		 * Guest was treating SUCCESS replies as "unexpected events". */
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: async RPC %s(%u) seq=%u len=%u -> NO REPLY (fire-and-forget)",
			     gsp_rm_rpc_name(function), function, sequence,
			     hdr->length);
		rm->rpcs_replied++;
		return;

	case GSPRM_RPC_FUNC_CONTINUATION_RECORD:
		/* CONTINUATION_RECORD: driver expects a reply. Send SUCCESS. */
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: async RPC %s(%u) seq=%u len=%u -> SUCCESS reply",
			     gsp_rm_rpc_name(function), function, sequence,
			     hdr->length);
		if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0))
			nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function);
		else
			rm->rpcs_replied++;
		return;

	case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO:
		gsp_rm_handle_set_guest_system_info(ctx, elem, sequence);
		return;

	case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO:
		gsp_rm_handle_get_static_info(ctx, sequence);
		return;

	case GSPRM_RPC_FUNC_GSP_RM_ALLOC:
		gsp_rm_handle_rm_alloc(ctx, elem, sequence);
		return;

	case GSPRM_RPC_FUNC_GSP_RM_CONTROL:
		gsp_rm_handle_rm_control(ctx, elem, sequence);
		return;

	case GSPRM_RPC_FUNC_FREE:
		gsp_rm_handle_free(ctx, elem, sequence);
		return;

	case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY:
		gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length);
		return;

	case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY:
		gsp_rm_handle_unset_page_directory(ctx, elem, sequence, hdr->length);
		return;

	case GSPRM_RPC_FUNC_UPDATE_BAR_PDE:
		/* UPDATE_BAR_PDE: guest updates BAR page directory entries.
		 * For vGPU, we just acknowledge success. */
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC UPDATE_BAR_PDE(%u) seq=%u len=%u -> SUCCESS",
			     function, sequence, hdr->length);
		if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0))
			nv_vgpu_err(ctx->vgpu, "vgsp-rm: UPDATE_BAR_PDE reply send failed");
		else
			rm->rpcs_replied++;
		return;

	case GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER:
		/* UNLOADING_GUEST_DRIVER: log for unload-cycle debugging */
		nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC UNLOADING_GUEST_DRIVER(%u) seq=%u cmd_read=%u stat_write=%u stat_seq=%u",
			     function, sequence,
			     rm->cmd_read, rm->stat_write, rm->stat_seq);
		/* Small delay before UNLOADING replies: mitigates an intermittent
		 * driver-side reply-matching race in the cleanup path. */
		udelay(2000);
		if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0))
			nv_vgpu_err(ctx->vgpu, "vgsp-rm: UNLOADING reply send failed");
		else
			rm->rpcs_replied++;
		return;

	default:
		/* Synchronous RPC: echo function+sequence with success so the
		 * driver's rpcRecvPoll completes. */
		rm->unknown_rpcs++;
		if (gsp_sema_dbg)
			nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)",
				     gsp_rm_rpc_name(function), function,
				     sequence, hdr->length, rm->unknown_rpcs);
		if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0))
			nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function);
		else
			rm->rpcs_replied++;
		return;
	}
}

/*
 * Poll the command queue TX writePtr; consume and handle new records.
 * A record spans elemCount consecutive queue elements.
 *
 * msgq pointers are SLOT INDICES that wrap at msgCount (msgq.c:
 * "writePtr += n; if (>= msgCount) -= msgCount"). Empty when
 * writePtr == readPtr. A 62->0 transition is a legitimate wrap, NOT
 * queue death.
 */
static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	struct gsprm_msgq_tx_header tx;
	int ret;

	ret = gsp_rm_read_buf(ctx, rm->cmd_off, &tx, sizeof(tx));
	if (ret) {
		/* VM may be gone or device closed; stay quiet, retry later */
		return;
	}

	/* A healthy driver ALWAYS has writePtr < msgCount. Anything else
	 * means the queue header is garbage (buffer freed on unload/crash
	 * without an observable GSP shutdown) - declare the emulated GSP
	 * dead so the next probe's WPR2-up check passes and it re-boots. */
	if (tx.writePtr >= rm->cmd_msgcount) {
		nv_vgpu_warn(ctx->vgpu, "vgsp-rm: insane cmd writePtr %u (msgCount=%u), GSP boot declared dead",
			     tx.writePtr, rm->cmd_msgcount);
		gsp_emu_clear_boot_state(ctx, "stale GSP-RM queue header");
		gsp_rm_emu_reset(ctx);
		return;
	}

	/* Consume records until we catch up with the (wrapped) writePtr */
	while (rm->cmd_read != tx.writePtr) {
		u8 *elem;
		u64 elem_off = rm->cmd_off + rm->entry_off +
			       (u64)(rm->cmd_read % rm->cmd_msgcount) * GSPRM_QUEUE_ELEMENT_SIZE;
		u32 signature, elem_count;
		const struct gsprm_rpc_header *hdr;

		elem = kmalloc(GSPRM_QUEUE_ELEMENT_SIZE, GFP_KERNEL);
		if (!elem)
			return;
		ret = gsp_rm_read_buf(ctx, elem_off, elem, GSPRM_QUEUE_ELEMENT_SIZE);
		if (ret) {
			kfree(elem);
			return;
		}

		hdr = (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
		signature = hdr->signature;
		memcpy(&elem_count, elem + 0x28, 4);

		if (signature != GSPRM_RPC_SIGNATURE || elem_count == 0 ||
		    elem_count > rm->cmd_msgcount) {
			/* Garbage element: could be teardown (driver stopped
			 * writing valid elements after UNLOADING_GUEST_DRIVER/
			 * FREE sequence) or genuine queue death. Don't declare
			 * death here - stop polling quietly and let the
			 * Booter-unload kick (SEC2 CPUCTL) trigger the reset
			 * via gsp_emu_clear_boot_state. This allows the
			 * driver's cleanup frees to complete. */
			if (gsp_sema_dbg)
				nv_vgpu_info(ctx->vgpu, "vgsp-rm: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, stopping poll (teardown?)",
					     rm->cmd_read, signature, elem_count,
					     hdr->function);
			kfree(elem);
			return;
		}

		gsp_rm_handle_element(ctx, elem);
		kfree(elem);

		/* consume the whole record (all its elements), wrapping
		 * at msgCount like the driver's own writePtr */
		rm->cmd_read = (rm->cmd_read + elem_count) % rm->cmd_msgcount;
	}

	/* Publish our command-queue read position into the status queue RX
	 * header (MSGQ_FLAGS_SWAP_RX: read pointers live in the peer's queue). */
	gsp_rm_write_buf(ctx, rm->stat_off + 0x20,
			 &rm->cmd_read, sizeof(rm->cmd_read));
}

/* ============================================================
 * Public interface
 * ============================================================ */

int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm;

	rm = kzalloc(sizeof(*rm), GFP_KERNEL);
	if (!rm)
		return -ENOMEM;

	rm->vgpu = vgpu;
	mutex_init(&rm->stat_lock);
	ctx->rm_emu = rm;
	return 0;
}

void gsp_rm_emu_fini(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;

	if (!rm)
		return;
	kfree(rm->ptes);
	kfree(rm);
	ctx->rm_emu = NULL;
}

/*
 * Reset the GSP-RM emulation state on GSP ENGINE reset (guest driver reboots
 * the GSP on every probe/unload). The queue buffer GPAs from the previous
 * boot are stale (freed by the guest) - drop all parsed state so the next
 * Booter completion re-parses the chain.
 */
void gsp_rm_emu_reset(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;

	if (!rm || !rm->parsed)
		return;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: reset (rpcs=%llu replied=%llu unknown=%llu)",
		     rm->rpcs_seen, rm->rpcs_replied, rm->unknown_rpcs);

	/*
	 * Serialize with the status-queue producers (the sema service posts
	 * events from its own thread; an unsynchronized reset here raced a
	 * post and desynced the write pointer/sequence - see the PART 27
	 * "Bad sequence number" incident class).
	 */
	mutex_lock(&rm->stat_lock);
	kfree(rm->ptes);
	rm->ptes = NULL;
	rm->parsed = false;
	rm->active = false;
	rm->init_done_sent = false;
	rm->libos_args_gpa = 0;
	rm->gsp_args_gpa = 0;
	rm->cmd_read = 0;
	rm->stat_write = 0;
	rm->stat_seq = 0;
	mutex_unlock(&rm->stat_lock);
	/*
	 * Fresh parse-retry budget for the next boot: a stale timestamp from
	 * the previous probe would delay the first retry on the new boot,
	 * and every retry tick counts against the guest's 4 s
	 * GspStatusQueueInit wait.
	 */
	rm->last_parse_retry = 0;
	memset(&rm->mq, 0, sizeof(rm->mq));
	rm->cmd_gpa = 0;
	rm->stat_gpa = 0;
	rm->cmd_size = 0;
	rm->stat_size = 0;
	rm->cmd_msgcount = 0;
	rm->stat_msgcount = 0;
}

/*
 * Called when the guest's Booter Load completes. The GSP MAILBOX0/1 now hold
 * the libos init args GPA; parse the queue chain, link the status queue and
 * emit GSP_INIT_DONE so the driver's kgspWaitForRmInitDone completes.
 */
void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;
	u32 mb_lo, mb_hi;
	u32 b_is_d3hot = 0;
	int ret;

	if (!rm || rm->parsed)
		return;

	mb_lo = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0);
	mb_hi = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0 + 4);
	rm->libos_args_gpa = ((u64)mb_hi << 32) | mb_lo;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: libos args at gpa=0x%llx (mailbox 0x%x:0x%x)",
		     rm->libos_args_gpa, mb_hi, mb_lo);

	if (!rm->libos_args_gpa) {
		pr_warn_ratelimited("vgsp-rm: libos args GPA is 0, will retry\n");
		return;
	}

	ret = gsp_rm_parse_queue_chain(ctx);
	if (ret) {
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: queue chain parse failed: %d", ret);
		return;
	}

	ret = gsp_rm_link_status_queue(ctx);
	if (ret)
		return;

	rm->parsed = true;
	rm->active = true;

	/* Emit GSP_INIT_DONE (function 0x1001, sequence 0, init_done_v17_00) */
	ret = gsp_rm_send(ctx, GSPRM_RPC_EVENT_GSP_INIT_DONE, 0, 0,
			  &b_is_d3hot, sizeof(b_is_d3hot));
	if (ret) {
		/*
		 * Re-arm the parse-retry path: parsed/active were set above,
		 * and the poll loop only re-attempts while !parsed. A failed
		 * INIT_DONE send with parsed=true would leave the guest's
		 * kgspWaitForRmInitDone with no retry at all - exactly the
		 * 4 s GspStatusQueueInit timeout class.
		 */
		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_INIT_DONE send failed: %d (will retry)",
			    ret);
		rm->parsed = false;
		rm->active = false;
		return;
	}
	rm->init_done_sent = true;

	nv_vgpu_info(ctx->vgpu, "vgsp-rm: GSP_INIT_DONE emitted, RPC loop active");
}

void gsp_rm_emu_poll(struct gsp_emu_context *ctx)
{
	struct gsp_rm_emu *rm = ctx->rm_emu;

	if (!rm)
		return;

	/* Retry chain parsing if the first attempt at BOOTER_DONE failed
	 * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE.
	 * Cadence matters: the guest's GspStatusQueueInit gives the link only
	 * 4 s, so retries are throttled just enough to keep a persistent
	 * failure from spamming dmesg (the failure log is ratelimited), not
	 * at 250 ms which burns a fifth of the guest's budget per attempt. */
	if (!rm->parsed) {
		if (ctx->boot_stage == 2 &&
		    time_after(jiffies, rm->last_parse_retry + HZ / 20)) {
			rm->last_parse_retry = jiffies;
			gsp_rm_emu_on_booter_done(ctx);
		}
		return;
	}

	if (!rm->active)
		return;

	gsp_rm_poll_commands(ctx);
}