1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
#!/bin/bash
# track_waits.sh - guest-side wait sampler: while a CUDA client (cdt) runs,
# repeatedly dump the per-channel completion counters (values), the tracker
# wait entries (trkinfo.py, every 2nd pass) and the push pages the host could
# not resolve (read_va.py), so the last signal before a failure is recorded.
#
# Written for the guest (no host paths). Run as root in the guest:
# sudo /tmp/track_waits.sh <pid> [samples] [sleep]
# Logs: /tmp/trk.log, /tmp/cnt.log, /tmp/push.log
set -u
PID="${1:?usage: track_waits.sh <pid> [samples] [sleep]}"
N="${2:-600}"
SLEEP="${3:-1}"
PAGE_VA="${PAGE_VA:-0x20400f000}"
COUNT="${COUNT:-20}"
STRIDE="${STRIDE:-16}"
OFFSET="${OFFSET:-0xec0}"
PUSH="${PUSH:-1}"
: > /tmp/trk.log
: > /tmp/cnt.log
: > /tmp/push.log
python3 - "$PID" "$PAGE_VA" "$COUNT" "$STRIDE" "$OFFSET" <<'PY' >> /tmp/cnt.log 2>&1 &
import os, sys, time
pid = int(sys.argv[1]); page = int(sys.argv[2], 16)
count = int(sys.argv[3]); stride = int(sys.argv[4], 16); off = int(sys.argv[5], 16)
pm = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY)
mem = os.open('/proc/%d/mem' % pid, os.O_RDONLY)
while True:
ts = time.strftime('%T')
vals = []
for i in range(count):
va = page + off + i * stride
try:
os.lseek(pm, (va // 4096) * 8, 0)
e = int.from_bytes(os.read(pm, 8), 'little')
present = (e >> 63) & 1
if present:
v = int.from_bytes(os.pread(mem, 4, va), 'little')
vals.append('%d:0x%x' % (i, v))
else:
vals.append('%d:-' % i)
except OSError:
vals.append('%d:ERR' % i)
print('%s %s' % (ts, ' '.join(vals)), flush=True)
time.sleep(0.5)
PY
CNTPID=$!
for i in $(seq 1 "$N"); do
TS=$(date +%T.%3N)
kill -0 "$PID" 2>/dev/null || { echo "PROCESS GONE at $TS (sample $i)" >> /tmp/trk.log; break; }
if [ $((i % 2)) -eq 1 ] && [ "${GDB:-1}" = "1" ]; then
echo "=== $TS sample $i ===" >> /tmp/trk.log
timeout 8 gdb -p "$PID" -batch -x /tmp/trkinfo.py 2>&1 \
| grep -vE "^\[|Reading symbols|^0x[0-9a-f]+ in |warning:" >> /tmp/trk.log
fi
if [ "$PUSH" = "1" ]; then
{
echo "=== $TS push pages ==="
python3 /tmp/guest_pmap.py "$PID" 0x201000000 0x203400000 0x203400020 2>&1
for va in 0x201000000 0x203400000 0x203400020; do
echo "--- $va"
timeout 5 python3 /tmp/read_va.py "$PID" "$va" 64 --u32 2>&1
done
} >> /tmp/push.log
fi
sleep "$SLEEP"
done
kill "$CNTPID" 2>/dev/null
echo "sampler done" >> /tmp/trk.log
|
