vgsp-rm: harden the boot-time queue link against the 4 s GspStatusQueueInit window

The guest's first RmInitAdapter intermittently fails: GspStatusQueueInit
gives the status-queue link only 4 s (message_queue_cpu.c: timeoutUs =
4000000) and the first boot attempt can burn it, after which the RM tears
down, leaks the device and retries; the surviving retry leaves the GR/ctx
state degraded, which is the persistent cudaMalloc stall class.

Changes:
- gsp_rm_emu_poll: parse-retry cadence 250 ms -> 50 ms (the 250 ms tick
  consumed a fifth of the guest's budget per attempt).
- gsp_rm_emu_reset: clear last_parse_retry so a new boot gets a fresh
  retry budget; take stat_lock while clearing queue cursors so a sema
  event post cannot race the reset.
- gsp_rm_emu_on_booter_done: on GSP_INIT_DONE send failure re-arm the
  retry path (parsed/active were already set, so the poll loop would
  never retry).
- gsp_rm_parse_queue_chain: free the previous ptes before re-alloc so
  retries and post-reset re-parses do not leak.
- gsp_rm_post_channel_event: never post events across a queue reset
  (parsed gate).

guest_kprobes.sh: add rpcs/rpcp transport probes (the direct RPC-wait
symbol is notrace in 570.144) and ccset entry/return probes for
rmapiControlCacheSet so a stall names the control whose reply breaks the
guest's control-cache bookkeeping.

2 files changed, 58 insertions, 4 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index f2656bb..983fecb 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -612,6 +612,10 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src)
 	rm = ctx->rm_emu;
 	if (!rm || !h_src)
 		return -ENODEV;
+	/* Never post across a queue reset: the guest has freed the old queue
+	 * and re-parses on the next Booter completion. */
+	if (!rm->parsed)
+		return -EAGAIN;
 
 	for (i = 0; i < rm->events_n; i++) {
 		if (!rm->events[i].valid || rm->events[i].h_src != h_src)
@@ -869,6 +873,8 @@ static int gsp_rm_parse_queue_chain(struct gsp_emu_context *ctx)
 	}
 
 	/* 4. Read the page table (RmPhysAddr per 4K page of the shared buffer) */
+	/* A retry (or a post-reset re-parse) must not leak the previous table. */
+	kfree(rm->ptes);
 	rm->ptes = kmalloc_array(rm->mq.pageTableEntryCount, sizeof(u64),
 				 GFP_KERNEL);
 	if (!rm->ptes) {
@@ -3598,6 +3604,13 @@ void gsp_rm_emu_reset(struct gsp_emu_context *ctx)
 	nv_vgpu_info(ctx->vgpu, "vgsp-rm: reset (rpcs=%llu replied=%llu unknown=%llu)",
 		     rm->rpcs_seen, rm->rpcs_replied, rm->unknown_rpcs);
 
+	/*
+	 * Serialize with the status-queue producers (the sema service posts
+	 * events from its own thread; an unsynchronized reset here raced a
+	 * post and desynced the write pointer/sequence - see the PART 27
+	 * "Bad sequence number" incident class).
+	 */
+	mutex_lock(&rm->stat_lock);
 	kfree(rm->ptes);
 	rm->ptes = NULL;
 	rm->parsed = false;
@@ -3608,6 +3621,14 @@ void gsp_rm_emu_reset(struct gsp_emu_context *ctx)
 	rm->cmd_read = 0;
 	rm->stat_write = 0;
 	rm->stat_seq = 0;
+	mutex_unlock(&rm->stat_lock);
+	/*
+	 * Fresh parse-retry budget for the next boot: a stale timestamp from
+	 * the previous probe would delay the first retry on the new boot,
+	 * and every retry tick counts against the guest's 4 s
+	 * GspStatusQueueInit wait.
+	 */
+	rm->last_parse_retry = 0;
 	memset(&rm->mq, 0, sizeof(rm->mq));
 	rm->cmd_gpa = 0;
 	rm->stat_gpa = 0;
@@ -3661,7 +3682,17 @@ void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx)
 	ret = gsp_rm_send(ctx, GSPRM_RPC_EVENT_GSP_INIT_DONE, 0, 0,
 			  &b_is_d3hot, sizeof(b_is_d3hot));
 	if (ret) {
-		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_INIT_DONE send failed: %d", ret);
+		/*
+		 * Re-arm the parse-retry path: parsed/active were set above,
+		 * and the poll loop only re-attempts while !parsed. A failed
+		 * INIT_DONE send with parsed=true would leave the guest's
+		 * kgspWaitForRmInitDone with no retry at all - exactly the
+		 * 4 s GspStatusQueueInit timeout class.
+		 */
+		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_INIT_DONE send failed: %d (will retry)",
+			    ret);
+		rm->parsed = false;
+		rm->active = false;
 		return;
 	}
 	rm->init_done_sent = true;
@@ -3678,11 +3709,13 @@ void gsp_rm_emu_poll(struct gsp_emu_context *ctx)
 
 	/* Retry chain parsing if the first attempt at BOOTER_DONE failed
 	 * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE.
-	 * Back off hard: at 100us poll cadence an unthrottled retry spams
-	 * dmesg into unusability when the driver stalls mid-bootstrap. */
+	 * Cadence matters: the guest's GspStatusQueueInit gives the link only
+	 * 4 s, so retries are throttled just enough to keep a persistent
+	 * failure from spamming dmesg (the failure log is ratelimited), not
+	 * at 250 ms which burns a fifth of the guest's budget per attempt. */
 	if (!rm->parsed) {
 		if (ctx->boot_stage == 2 &&
-		    time_after(jiffies, rm->last_parse_retry + HZ / 4)) {
+		    time_after(jiffies, rm->last_parse_retry + HZ / 20)) {
 			rm->last_parse_retry = jiffies;
 			gsp_rm_emu_on_booter_done(ctx);
 		}
diff --git a/tools/guest-diag/guest_kprobes.sh b/tools/guest-diag/guest_kprobes.sh
index cf34833..d3db7b3 100644
--- a/tools/guest-diag/guest_kprobes.sh
+++ b/tools/guest-diag/guest_kprobes.sh
@@ -85,6 +85,27 @@ add_probe uvsv uvm_gpu_tracking_semaphore_is_value_completed 'sem=%di value=%si'
 echo "-:kprobes/uvsvr" >> "$TR/kprobe_events" 2>/dev/null
 echo "-:uvsvr" >> "$TR/kprobe_events" 2>/dev/null
 echo 'r:uvsvr uvm_gpu_tracking_semaphore_is_value_completed $retval' >> "$TR/kprobe_events" 2>/dev/null
+# RPC wait tracing: _issueRpcAndWait is notrace in 570.144; probe the
+# send/poll transport instead. An rpcp entry whose return is ~4s later is
+# the timed-out RPC that fails RmInitAdapter (observed: first init aborts,
+# second succeeds).
+echo "-:kprobes/rpcs" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:rpcs" >> "$TR/kprobe_events" 2>/dev/null
+echo 'p:rpcs _rpcSendMessage_VGPUGSP rpc=%si' >> "$TR/kprobe_events" 2>/dev/null
+echo "-:kprobes/rpcp" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:rpcp" >> "$TR/kprobe_events" 2>/dev/null
+echo 'p:rpcp rpcRecvPoll_IMPL gpu=%di' >> "$TR/kprobe_events" 2>/dev/null
+echo "-:kprobes/rpcpr" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:rpcpr" >> "$TR/kprobe_events" 2>/dev/null
+echo 'r:rpcpr rpcRecvPoll_IMPL $retval' >> "$TR/kprobe_events" 2>/dev/null
+# Control-cache set tracing: retval 0x3b (INVALID_PARAMETER) names the cmd/size
+# whose reply breaks the guest's cache bookkeeping.
+echo "-:kprobes/ccset" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:ccset" >> "$TR/kprobe_events" 2>/dev/null
+echo 'p:ccset rmapiControlCacheSet hCli=%di hObj=%si cmd=%dx size=%r8' >> "$TR/kprobe_events" 2>/dev/null
+echo "-:kprobes/ccsetr" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:ccsetr" >> "$TR/kprobe_events" 2>/dev/null
+echo 'r:ccsetr rmapiControlCacheSet $retval' >> "$TR/kprobe_events" 2>/dev/null
 
 echo 1 > "$TR/events/kprobes/enable" 2>/dev/null || true
 # The client's spin floods the trace with uvmt/uvsv events; a one-shot uvbp