From ba95c995a719289f40fedda081df91f576b52c2a Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Mon, 28 Sep 2026 12:10:17 -0400
Subject: guest-diag: nmi_catch.sh - sysrq-l sampler that keeps nvidia kernel
 stacks

The guest RM's expiring waits are kernel busy-polls and all of their
functions are notrace, so ftrace/kprobes cannot name them. sysrq-l NMI
backtraces can: this sampler triggers them and keeps only the Comm and
[nvidia] frame lines per round, so the wait/teardown stacks survive dmesg
ring wraps. Caught the adapter-shutdown scrubber stack
(channelReadChannelMemdesc <- ceutilsUpdateProgress <- scrubberDestruct <-
memmgrDestroyInternalChannels <- gpuStatePreUnload <- nv_shutdown_adapter).
---
 tools/guest-diag/nmi_catch.sh | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 tools/guest-diag/nmi_catch.sh

diff --git a/tools/guest-diag/nmi_catch.sh b/tools/guest-diag/nmi_catch.sh
new file mode 100644
index 0000000..3afa895
--- /dev/null
+++ b/tools/guest-diag/nmi_catch.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+# nmi_catch.sh [rounds] [sleep] - trigger sysrq-l NMI backtraces and append a
+# compact summary of the nvidia frames seen in each round (guest side, root).
+#
+# The guest RM's expiring waits are kernel busy-polls (their internals are
+# `notrace`, so ftrace/kprobes cannot see them). sysrq-l prints each CPU's
+# current kernel stack, and this script keeps only the lines that carry an
+# `[nvidia]` frame plus the owning `Comm:` line, so the wait/teardown stack
+# survives even when the dmesg ring wraps.
+#
+# Run: sudo sh nmi_catch.sh 42 6 &        (42 rounds every 6 s)
+# Read: /tmp/nmi2.log   (sorted unique Comm/nvidia-frame lines)
+rm -f /tmp/nmi2.raw
+i=0
+R=${1:-40}
+S=${2:-6}
+echo 1 > /proc/sys/kernel/sysrq
+while [ $i -lt "$R" ]; do
+    echo l > /proc/sysrq-trigger
+    sleep "$S"
+    dmesg | grep -aB20 '\[nvidia\]' | grep -aE 'Comm: |\[nvidia\]' >> /tmp/nmi2.raw 2>/dev/null
+    i=$((i+1))
+done
+sort -u /tmp/nmi2.raw > /tmp/nmi2.log 2>/dev/null
+echo DONE >> /tmp/nmi2.log
+wc -l /tmp/nmi2.log
\ No newline at end of file
-- 
cgit v1.2.3