blob: f210422bd157190480323502efe35d70958fa8b0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
|
#!/bin/sh
# stack_sampler.sh - per-thread kernel stack sampler for the CUDA client.
#
# The client's failing wait is not always visible in strace: libcuda runs
# worker threads and the main thread may only see the error. Sample every
# thread's wchan + kernel stack so the wait that expires is captured.
#
# Usage: sh /tmp/stack_sampler.sh [samples] [sleep]
N="${1:-900}"
SLEEP="${2:-0.2}"
PID=""
for i in $(seq 1 80); do
PID=$(pgrep -x cdt | head -1)
[ -n "$PID" ] && break
sleep 0.25
done
[ -n "$PID" ] || { echo "no cdt" > /tmp/stacks.log; exit 1; }
: > /tmp/stacks.log
i=0
while [ "$i" -lt "$N" ]; do
kill -0 "$PID" 2>/dev/null || { echo "GONE" >> /tmp/stacks.log; break; }
T=$(date +%T.%3N)
for t in /proc/$PID/task/*; do
[ -d "$t" ] || continue
tid=${t##*/}
st=$(awk '{print $3}' "$t/stat" 2>/dev/null)
wc=$(cat "$t/wchan" 2>/dev/null)
echo "$T tid=$tid state=$st wchan=$wc" >> /tmp/stacks.log
cat "$t/stack" >> /tmp/stacks.log 2>/dev/null
done
i=$((i + 1))
sleep "$SLEEP"
done
echo "DONE" >> /tmp/stacks.log
|
