guest-diag: per-thread kernel stack sampler for the CUDA client

The failing wait can live in a libcuda worker thread (strace without -f
misses it); sample every /proc/<pid>/task/*/stack + wchan while the client
runs.

2 files changed, 37 insertions, 0 deletions

diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 5e331e5..58ebdb3 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -36,6 +36,7 @@ push_b64 "$HERE/guest_pmap.py"  /tmp/guest_pmap.py
 push_b64 "$HERE/guest_pmap_win.py" /tmp/guest_pmap_win.py
 push_b64 "$HERE/guest_pin_stream.py" /tmp/guest_pin_stream.py
 push_b64 "$HERE/track_waits.sh" /tmp/track_waits.sh
+push_b64 "$HERE/stack_sampler.sh" /tmp/stack_sampler.sh
 push_b64 "$HERE/guest_kprobes.sh" /tmp/kprobes.sh
 
 echo "[*] compiling in guest..."
diff --git a/tools/guest-diag/stack_sampler.sh b/tools/guest-diag/stack_sampler.sh
new file mode 100755
index 0000000..f210422
--- /dev/null
+++ b/tools/guest-diag/stack_sampler.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+# stack_sampler.sh - per-thread kernel stack sampler for the CUDA client.
+#
+# The client's failing wait is not always visible in strace: libcuda runs
+# worker threads and the main thread may only see the error. Sample every
+# thread's wchan + kernel stack so the wait that expires is captured.
+#
+# Usage: sh /tmp/stack_sampler.sh [samples] [sleep]
+N="${1:-900}"
+SLEEP="${2:-0.2}"
+
+PID=""
+for i in $(seq 1 80); do
+    PID=$(pgrep -x cdt | head -1)
+    [ -n "$PID" ] && break
+    sleep 0.25
+done
+[ -n "$PID" ] || { echo "no cdt" > /tmp/stacks.log; exit 1; }
+
+: > /tmp/stacks.log
+i=0
+while [ "$i" -lt "$N" ]; do
+    kill -0 "$PID" 2>/dev/null || { echo "GONE" >> /tmp/stacks.log; break; }
+    T=$(date +%T.%3N)
+    for t in /proc/$PID/task/*; do
+        [ -d "$t" ] || continue
+        tid=${t##*/}
+        st=$(awk '{print $3}' "$t/stat" 2>/dev/null)
+        wc=$(cat "$t/wchan" 2>/dev/null)
+        echo "$T tid=$tid state=$st wchan=$wc" >> /tmp/stacks.log
+        cat "$t/stack" >> /tmp/stacks.log 2>/dev/null
+    done
+    i=$((i + 1))
+    sleep "$SLEEP"
+done
+echo "DONE" >> /tmp/stacks.log
\ No newline at end of file