guest-diag: one-shot gdb/state samplers for timed post-control captures

gdb_now.sh attaches once at a host-chosen moment (final RM control) and dumps
all threads; oneshot_gdb.sh triggers on user-running state; state_log.sh
records state/syscall/wchan so kernel busy-waits can be told apart from real
userspace spins.

3 files changed, 77 insertions, 0 deletions

diff --git a/tools/guest-diag/gdb_now.sh b/tools/guest-diag/gdb_now.sh
new file mode 100644
index 0000000..b58fe3d
--- /dev/null
+++ b/tools/guest-diag/gdb_now.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# gdb_now.sh <pid> - attach gdb once immediately (root), dump all threads,
+# detach, CONT. Used when the host has seen the final RM control, so the
+# attach is timed to the post-control window instead of state sampling.
+PID="$1"
+LOG="${2:-/tmp/gdbbt2.log}"
+: > "$LOG"
+{
+    echo "=== FORCED $(date +%T.%3N) pid=$PID ==="
+    for t in /proc/$PID/task/*; do
+        echo "--- thread $(basename $t): $(awk '{print $3}' $t/stat 2>/dev/null) $(cat $t/comm 2>/dev/null) syscall=$(cat $t/syscall 2>/dev/null | cut -d' ' -f1-2)"
+        cat "$t/stack" 2>/dev/null | sed 's/^/    kstack: /'
+    done
+    grep -E "libcuda|nvidia" /proc/$PID/maps | head -4
+} >> "$LOG" 2>&1
+timeout 90 gdb -p "$PID" -batch \
+    -ex "set pagination off" -ex "set height 0" -ex "set confirm off" \
+    -ex "info threads" \
+    -ex "thread apply all bt 12" \
+    -ex "thread apply all info registers rip rsp rbp rbx rax rdi rsi rdx rcx r8 r9" \
+    -ex "thread apply all x/10i \$pc" \
+    >> "$LOG" 2>&1
+echo "=== GDB DONE $(date +%T.%3N) ===" >> "$LOG"
+kill -CONT "$PID" 2>/dev/null
diff --git a/tools/guest-diag/oneshot_gdb.sh b/tools/guest-diag/oneshot_gdb.sh
new file mode 100644
index 0000000..9161ee4
--- /dev/null
+++ b/tools/guest-diag/oneshot_gdb.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+# oneshot_gdb.sh <pid> - catch the client's USERSPACE spin (state R and
+# /proc/<pid>/syscall == "running"; a kernel busy-wait shows its syscall
+# number). Fires gdb ONCE, dumps all threads, detaches, CONTs.
+PID="$1"
+LOG=/tmp/gdbbt.log
+: > "$LOG"
+cir=0
+win=0          # count of user-running samples since start
+samples=0
+while kill -0 "$PID" 2>/dev/null; do
+    samples=$((samples+1))
+    st=$(awk '{print $3}' /proc/$PID/stat 2>/dev/null)
+    sy=$(cat /proc/$PID/syscall 2>/dev/null)
+    if [ "$st" = "R" ] && [ "$sy" = "running" ]; then
+        cir=$((cir+1)); win=$((win+1))
+    else
+        cir=0
+    fi
+    if [ "$cir" -ge 4 ] || { [ "$samples" -ge 32 ] && [ "$win" -ge 12 ]; }; then
+        {
+            echo "=== TRIGGER $(date +%T.%3N) pid=$PID cir=$cir win=$win samples=$samples ==="
+            for t in /proc/$PID/task/*; do
+                echo "--- thread $(basename $t): $(awk '{print $3}' $t/stat 2>/dev/null) $(cat $t/comm 2>/dev/null) syscall=$(cat $t/syscall 2>/dev/null | cut -d' ' -f1-2)"
+            done
+            grep -E "libcuda|nvidia" /proc/$PID/maps | head -4
+        } >> "$LOG"
+        timeout 90 gdb -p "$PID" -batch \
+            -ex "set pagination off" -ex "set height 0" -ex "set confirm off" \
+            -ex "info threads" \
+            -ex "thread apply all bt 12" \
+            -ex "thread apply all info registers rip rsp rbp rbx rax rdi rsi rdx rcx r8 r9" \
+            -ex "thread apply all x/10i \$pc" \
+            >> "$LOG" 2>&1
+        echo "=== GDB DONE $(date +%T.%3N) ===" >> "$LOG"
+        kill -CONT "$PID" 2>/dev/null
+        exit 0
+    fi
+    sleep 0.25
+done
+echo "=== process gone after $samples samples ===" >> "$LOG"
diff --git a/tools/guest-diag/state_log.sh b/tools/guest-diag/state_log.sh
new file mode 100644
index 0000000..de3d6ed
--- /dev/null
+++ b/tools/guest-diag/state_log.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+# state_log.sh <pid> - 200ms sampler of state/syscall/wchan for the whole run.
+PID="$1"
+: > /tmp/state.log
+while kill -0 "$PID" 2>/dev/null; do
+    st=$(awk '{print $3}' /proc/$PID/stat 2>/dev/null)
+    sc=$(cat /proc/$PID/syscall 2>/dev/null | cut -d' ' -f1-3)
+    wc=$(cat /proc/$PID/wchan 2>/dev/null)
+    echo "$(date +%T.%3N) st=$st sc=[$sc] wchan=$wc" >> /tmp/state.log
+    sleep 0.2
+done
+echo "$(date +%T.%3N) GONE" >> /tmp/state.log