#!/bin/sh
# stack_sampler.sh - per-thread kernel stack sampler for the CUDA client.
#
# The client's failing wait is not always visible in strace: libcuda runs
# worker threads and the main thread may only see the error. Sample every
# thread's wchan + kernel stack so the wait that expires is captured.
#
# Usage: sh /tmp/stack_sampler.sh [samples] [sleep]
N="${1:-900}"
SLEEP="${2:-0.2}"

PID=""
for i in $(seq 1 80); do
    PID=$(pgrep -x cdt | head -1)
    [ -n "$PID" ] && break
    sleep 0.25
done
[ -n "$PID" ] || { echo "no cdt" > /tmp/stacks.log; exit 1; }

: > /tmp/stacks.log
i=0
while [ "$i" -lt "$N" ]; do
    kill -0 "$PID" 2>/dev/null || { echo "GONE" >> /tmp/stacks.log; break; }
    T=$(date +%T.%3N)
    for t in /proc/$PID/task/*; do
        [ -d "$t" ] || continue
        tid=${t##*/}
        st=$(awk '{print $3}' "$t/stat" 2>/dev/null)
        wc=$(cat "$t/wchan" 2>/dev/null)
        echo "$T tid=$tid state=$st wchan=$wc" >> /tmp/stacks.log
        cat "$t/stack" >> /tmp/stacks.log 2>/dev/null
    done
    i=$((i + 1))
    sleep "$SLEEP"
done
echo "DONE" >> /tmp/stacks.log