kernel: msgq writePtr wrap semantics + retry backoff + CE fault buffer size

CRITICAL protocol fix (msgq.c): writePtr/readPtr are slot indices that
WRAP at msgCount ('writePtr += n; if (>= msgCount) -= msgCount'), NOT
monotonic counters. '62 -> 0' is a legitimate wrap, not queue death.

- gsp_rm_poll_commands: consume while (cmd_read != writePtr), advance
  cmd_read = (cmd_read + elem_count) % msgcount. Death detection now
  only: writePtr >= msgCount (garbage header) / bad element signature /
  insane elemCount. Removes false deaths on healthy wraps that killed
  the emulator mid-session (fn 10 FREE timeouts).
- gsp_rm_send: published status writePtr now wraps too (driver's
  free-space/slot math breaks on unbounded values >= 2*msgCount).
- gsp_rm_emu_poll: back off on_booter_done retries to 250ms (was every
  100us poll -> thousands of 'will retry' lines/sec, dmesg unusable);
  retry log now pr_warn_ratelimited.
- CE_GET_FAULT_METHOD_BUFFER_SIZE(0x20802a08) = 12288 (GV100 formula
  inherited by Turing: 135*(13*5+2) page-aligned, 5 PCEs). Zero reply
  failed fault method buffer allocation (NV_ERR_NO_MEMORY) killing
  channel-group creation.

3 files changed, 72 insertions, 25 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 5ad557f..665f073 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -251,7 +251,11 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
 
 	kfree(buf);
 
-	rm->stat_write += elem_count;
+	/* Advance the write pointer with the same wrap semantics as the
+	 * driver's msgq (slot index in [0, msgCount) - the peer's free-space
+	 * and slot math breaks on unbounded values). stat_seq stays
+	 * monotonic per-record. */
+	rm->stat_write = (rm->stat_write + elem_count) % rm->stat_msgcount;
 	rm->stat_seq++;
 
 	/* bump status TX header writePtr (offset 0x10) */
@@ -981,6 +985,30 @@ static void gsp_rm_handle_display_get_ip_version(struct gsp_emu_context *ctx,
 	nv_vgpu_info(ctx->vgpu, "GSPRM: DISPLAY_GET_IP_VERSION -> 0x%x", val32);
 }
 
+/*
+ * CE_GET_FAULT_METHOD_BUFFER_SIZE (0x20802a08)
+ *
+ * kchangrpInit allocates one fault method buffer per runqueue of this
+ * size; a zero reply makes the allocation fail (NV_ERR_NO_MEMORY) and
+ * kills channel-group creation. GV100 formula inherited by Turing:
+ * 135 * (13*totalCes + 2) page-aligned; TU104 = 5 PCEs -> 12288.
+ */
+static void gsp_rm_handle_ce_get_fault_method_buffer_size(
+						struct gsp_emu_context *ctx,
+						u32 sequence,
+						u32 *out_status,
+						u8 *out_params,
+						u32 *out_params_size)
+{
+	u32 val32 = TU104_CE_FAULT_METHOD_BUFFER_SIZE;
+
+	memcpy(out_params, &val32, 4);
+	*out_params_size = 4;
+	*out_status = 0;
+
+	nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_FAULT_METHOD_BUFFER_SIZE -> %u", val32);
+}
+
 static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
 				     const u8 *elem, u32 sequence)
 {
@@ -1063,6 +1091,12 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
 			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
 		break;
 
+	case TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE:
+		gsp_rm_handle_ce_get_fault_method_buffer_size(ctx, sequence,
+			&out_status,
+			reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+		break;
+
 	default:
 		/* Unknown control command - return success with zeroed params.
 		 * Do NOT echo the request: pure queries pass uninitialized
@@ -1163,6 +1197,11 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
 /*
  * Poll the command queue TX writePtr; consume and handle new records.
  * A record spans elemCount consecutive queue elements.
+ *
+ * msgq pointers are SLOT INDICES that wrap at msgCount (msgq.c:
+ * "writePtr += n; if (>= msgCount) -= msgCount"). Empty when
+ * writePtr == readPtr. A 62->0 transition is a legitimate wrap, NOT
+ * queue death.
  */
 static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
 {
@@ -1176,24 +1215,19 @@ static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
 		return;
 	}
 
-	if (tx.writePtr == rm->cmd_read)
-		return;
-
-	if (tx.writePtr < rm->cmd_read ||
-	    (tx.writePtr - rm->cmd_read) > rm->cmd_msgcount) {
-		/* The writePtr is monotonic for the life of a boot. A backward
-		 * or insane jump means the queue buffer was torn down - the
-		 * guest driver unloaded (or crashed) without an observable
-		 * GSP shutdown. Declare the emulated GSP dead: clear WPR2 /
-		 * RISCV / boot stages so the next probe's WPR2-up check
-		 * passes and it re-boots from scratch. */
-		nv_vgpu_warn(ctx->vgpu, "GSPRM: cmd writePtr jumped %u -> %u, GSP boot declared dead",
-			     rm->cmd_read, tx.writePtr);
-		gsp_emu_clear_boot_state(ctx, "stale GSP-RM queue");
+	/* A healthy driver ALWAYS has writePtr < msgCount. Anything else
+	 * means the queue header is garbage (buffer freed on unload/crash
+	 * without an observable GSP shutdown) - declare the emulated GSP
+	 * dead so the next probe's WPR2-up check passes and it re-boots. */
+	if (tx.writePtr >= rm->cmd_msgcount) {
+		nv_vgpu_warn(ctx->vgpu, "GSPRM: insane cmd writePtr %u (msgCount=%u), GSP boot declared dead",
+			     tx.writePtr, rm->cmd_msgcount);
+		gsp_emu_clear_boot_state(ctx, "stale GSP-RM queue header");
 		return;
 	}
 
-	while (rm->cmd_read < tx.writePtr) {
+	/* Consume records until we catch up with the (wrapped) writePtr */
+	while (rm->cmd_read != tx.writePtr) {
 		u8 *elem;
 		u64 elem_gpa = gsp_rm_off_to_gpa(rm, rm->cmd_off + rm->entry_off +
 			       (u64)(rm->cmd_read % rm->cmd_msgcount) * GSPRM_QUEUE_ELEMENT_SIZE);
@@ -1216,10 +1250,10 @@ static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
 		if (signature != GSPRM_RPC_SIGNATURE || elem_count == 0 ||
 		    elem_count > rm->cmd_msgcount) {
 			/* The driver always writes a valid element BEFORE
-			 * bumping writePtr. An advanced writePtr with garbage
-			 * content means the queue buffer is stale (freed and
-			 * reused by the guest after unload/crash) - declare
-			 * the emulated GSP dead so the next probe re-boots. */
+			 * bumping writePtr. Garbage content means the queue
+			 * buffer is stale (freed and reused by the guest
+			 * after unload/crash) - declare the emulated GSP
+			 * dead so the next probe re-boots. */
 			nv_vgpu_warn(ctx->vgpu, "GSPRM: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, GSP boot declared dead",
 				     rm->cmd_read, signature, elem_count,
 				     hdr->function);
@@ -1231,8 +1265,9 @@ static void gsp_rm_poll_commands(struct gsp_emu_context *ctx)
 		gsp_rm_handle_element(ctx, elem);
 		kfree(elem);
 
-		/* consume the whole record (all its elements) */
-		rm->cmd_read += elem_count;
+		/* consume the whole record (all its elements), wrapping
+		 * at msgCount like the driver's own writePtr */
+		rm->cmd_read = (rm->cmd_read + elem_count) % rm->cmd_msgcount;
 	}
 
 	/* Publish our command-queue read position into the status queue RX
@@ -1327,7 +1362,7 @@ void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx)
 		     rm->libos_args_gpa, mb_hi, mb_lo);
 
 	if (!rm->libos_args_gpa) {
-		nv_vgpu_warn(ctx->vgpu, "GSPRM: libos args GPA is 0, will retry");
+		pr_warn_ratelimited("GSPRM: libos args GPA is 0, will retry\n");
 		return;
 	}
 
@@ -1364,10 +1399,15 @@ void gsp_rm_emu_poll(struct gsp_emu_context *ctx)
 		return;
 
 	/* Retry chain parsing if the first attempt at BOOTER_DONE failed
-	 * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE. */
+	 * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE.
+	 * Back off hard: at 100us poll cadence an unthrottled retry spams
+	 * dmesg into unusability when the driver stalls mid-bootstrap. */
 	if (!rm->parsed) {
-		if (ctx->boot_stage == 2)
+		if (ctx->boot_stage == 2 &&
+		    time_after(jiffies, rm->last_parse_retry + HZ / 4)) {
+			rm->last_parse_retry = jiffies;
 			gsp_rm_emu_on_booter_done(ctx);
+		}
 		return;
 	}
 
diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h
index b1a7259..b4e84ef 100644
--- a/kernel/gsp_rm_emulation.h
+++ b/kernel/gsp_rm_emulation.h
@@ -110,6 +110,8 @@ struct gsp_rm_emu {
 	bool		parsed;		/* queue chain parsed + status queue linked */
 	bool		active;
 
+	unsigned long	last_parse_retry;	/* jiffies of last on_booter_done retry */
+
 	u64		libos_args_gpa;	/* from GSP MAILBOX0/1 */
 	u64		gsp_args_gpa;	/* RMARGS region pa */
 
diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h
index d05d453..c6dfb56 100644
--- a/kernel/gsp_rm_tu104.h
+++ b/kernel/gsp_rm_tu104.h
@@ -699,6 +699,11 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = {
 #define TU104_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO	0x20800a01
 #define TU104_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION	0x20800a4b
 #define TU104_CTRL_I2C_TRANSACTION			0x402c0105
+#define TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE	0x20802a08
+
+/* CE fault method buffer size (ceGetFaultMethodBufferSize_GV100 formula,
+ * Turing inherits: 135 * (13*totalCes + 2), page-aligned; TU104 has 5 PCEs) */
+#define TU104_CE_FAULT_METHOD_BUFFER_SIZE		12288
 
 /* Turing display IP version (DISP v04.00 per DispIpHal halspec) */
 #define TU104_DISPLAY_IP_VERSION				0x04000000