From 7127c04d038074a3753a3c38dc11fa1fddf81b16 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Thu, 8 Oct 2026 09:38:39 -0400
Subject: vgsp-rm: harden the boot-time queue link against the 4 s
 GspStatusQueueInit window

The guest's first RmInitAdapter intermittently fails: GspStatusQueueInit
gives the status-queue link only 4 s (message_queue_cpu.c: timeoutUs =
4000000) and the first boot attempt can burn it, after which the RM tears
down, leaks the device and retries; the surviving retry leaves the GR/ctx
state degraded, which is the persistent cudaMalloc stall class.

Changes:
- gsp_rm_emu_poll: parse-retry cadence 250 ms -> 50 ms (the 250 ms tick
  consumed a fifth of the guest's budget per attempt).
- gsp_rm_emu_reset: clear last_parse_retry so a new boot gets a fresh
  retry budget; take stat_lock while clearing queue cursors so a sema
  event post cannot race the reset.
- gsp_rm_emu_on_booter_done: on GSP_INIT_DONE send failure re-arm the
  retry path (parsed/active were already set, so the poll loop would
  never retry).
- gsp_rm_parse_queue_chain: free the previous ptes before re-alloc so
  retries and post-reset re-parses do not leak.
- gsp_rm_post_channel_event: never post events across a queue reset
  (parsed gate).

guest_kprobes.sh: add rpcs/rpcp transport probes (the direct RPC-wait
symbol is notrace in 570.144) and ccset entry/return probes for
rmapiControlCacheSet so a stall names the control whose reply breaks the
guest's control-cache bookkeeping.
---
 tools/guest-diag/guest_kprobes.sh | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

(limited to 'tools')

diff --git a/tools/guest-diag/guest_kprobes.sh b/tools/guest-diag/guest_kprobes.sh
index cf34833..d3db7b3 100644
--- a/tools/guest-diag/guest_kprobes.sh
+++ b/tools/guest-diag/guest_kprobes.sh
@@ -85,6 +85,27 @@ add_probe uvsv uvm_gpu_tracking_semaphore_is_value_completed 'sem=%di value=%si'
 echo "-:kprobes/uvsvr" >> "$TR/kprobe_events" 2>/dev/null
 echo "-:uvsvr" >> "$TR/kprobe_events" 2>/dev/null
 echo 'r:uvsvr uvm_gpu_tracking_semaphore_is_value_completed $retval' >> "$TR/kprobe_events" 2>/dev/null
+# RPC wait tracing: _issueRpcAndWait is notrace in 570.144; probe the
+# send/poll transport instead. An rpcp entry whose return is ~4s later is
+# the timed-out RPC that fails RmInitAdapter (observed: first init aborts,
+# second succeeds).
+echo "-:kprobes/rpcs" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:rpcs" >> "$TR/kprobe_events" 2>/dev/null
+echo 'p:rpcs _rpcSendMessage_VGPUGSP rpc=%si' >> "$TR/kprobe_events" 2>/dev/null
+echo "-:kprobes/rpcp" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:rpcp" >> "$TR/kprobe_events" 2>/dev/null
+echo 'p:rpcp rpcRecvPoll_IMPL gpu=%di' >> "$TR/kprobe_events" 2>/dev/null
+echo "-:kprobes/rpcpr" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:rpcpr" >> "$TR/kprobe_events" 2>/dev/null
+echo 'r:rpcpr rpcRecvPoll_IMPL $retval' >> "$TR/kprobe_events" 2>/dev/null
+# Control-cache set tracing: retval 0x3b (INVALID_PARAMETER) names the cmd/size
+# whose reply breaks the guest's cache bookkeeping.
+echo "-:kprobes/ccset" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:ccset" >> "$TR/kprobe_events" 2>/dev/null
+echo 'p:ccset rmapiControlCacheSet hCli=%di hObj=%si cmd=%dx size=%r8' >> "$TR/kprobe_events" 2>/dev/null
+echo "-:kprobes/ccsetr" >> "$TR/kprobe_events" 2>/dev/null
+echo "-:ccsetr" >> "$TR/kprobe_events" 2>/dev/null
+echo 'r:ccsetr rmapiControlCacheSet $retval' >> "$TR/kprobe_events" 2>/dev/null
 
 echo 1 > "$TR/events/kprobes/enable" 2>/dev/null || true
 # The client's spin floods the trace with uvmt/uvsv events; a one-shot uvbp
-- 
cgit v1.2.3