blob: 45d08ccc55444acce72b56664203fc2ec4b701de (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
#!/bin/bash
# pdwatch_cycle.sh - TASK 2: watch every write landing in the root PD pages
# 0x200000..0x204000 (and the 0x208000 table) through a full cdt2 run, with
# the walk state sampled during the CP park and after completion.
set -uo pipefail
HERE=/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/guest-diag
REPO=/home/rmcore/Git/integdev_gpu_drv/open-iov
VMEXEC="$REPO/scripts/vgpu_vm.py"
OUT=/tmp/opencode
SER=/tmp/vgpu-guest-serial.log
LOG=$OUT/pdwatch_cycle.log
exec > >(tee -a "$LOG") 2>&1
echo "[*] $(date +%T) vmrestart"
timeout 200 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }
for i in $(seq 1 30); do
timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP && { echo " guest up"; break; }
sleep 5
done
echo "[*] $(date +%T) params: sema_dbg=Y, dst watch 0x200000 + 0x208000"
sudo sh -c "echo Y > /sys/module/open_iov_vfio/parameters/sema_dbg; \
echo N > /sys/module/open_iov_vfio/parameters/sema_fake_rel; \
echo N > /sys/module/open_iov_vfio/parameters/sema_skip_ops; \
echo 64 > /sys/module/open_iov_vfio/parameters/sema_ops_log; \
echo 0 > /sys/module/open_iov_vfio/parameters/sema_copy_watch; \
echo 0 > /sys/module/open_iov_vfio/parameters/sema_copy_pat; \
echo 0x200000 > /sys/module/open_iov_vfio/parameters/sema_copy_dst; \
echo 0x208000 > /sys/module/open_iov_vfio/parameters/sema_copy_dst2; \
echo 8000 > /sys/module/open_iov_vfio/parameters/sema_ce_grace_ms"
echo "[*] $(date +%T) push tools"
timeout 600 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || echo " push failed"
echo "[*] $(date +%T) nvidia modules + kprobes"
timeout 120 "$VMEXEC" exec "nvidia-smi -L >/dev/null 2>&1; lsmod | grep -q nvidia_uvm && echo UVM_LOADED" 2>/dev/null | tail -1
timeout 60 "$VMEXEC" exec "sh /tmp/kprobes.sh" 2>/dev/null | grep -E "ok|FAIL" | head -8
: > "$OUT/sem_stream.txt"
( setsid nohup env SEMFIX_COUNTER_RETRY=10 python3 "$HERE/pin_gr_stream.py" > "$OUT/pw_pin.log" 2>&1 & )
sleep 3
( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 python3 "$HERE/semfix_uvmt.py" 0.5 > "$OUT/pw_semfix.log" 2>&1 & )
sleep 3
sudo dmesg -C 2>/dev/null
: > "$SER" 2>/dev/null || true
echo "[*] $(date +%T) launching cdt2 (CP pause 240)"
timeout 40 "$VMEXEC" exec \
"echo '=== PDWATCH RUN START ===' > /dev/ttyS0; rm -f /tmp/cdt2.out; nohup sh -c 'CDT_CP_PAUSE=240 CDT_WAIT=60 /tmp/cdt2 2>&1 | tee /tmp/cdt2.out | stdbuf -oL tee /dev/ttyS0 >/dev/null' >/dev/null 2>&1 & echo STARTED" 2>/dev/null | tail -1
for i in $(seq 1 60); do
sleep 5
grep -qa "parked 240" "$SER" 2>/dev/null && { echo " parked after ~$((i*5))s"; break; }
done
echo "[*] $(date +%T) shadow dumps during park"
for a in 0x200000 0x201000 0x202000 0x203000 0x208000; do
echo " -- $a"
sudo sh -c "echo '$a,64' > /sys/module/open_iov_vfio/parameters/sema_vidmem_dump" 2>/dev/null
sleep 0.2
done
sudo dmesg | grep -a "sema-vidmem\|sema-walk-probe\|sema-svc: VA space" | tail -40 > "$OUT/pdwatch_park_dmesg.txt"
cat "$OUT/pdwatch_park_dmesg.txt"
DEV=$(grep -ao 'device ptr = 0x[0-9a-f]*' "$SER" | tail -1 | cut -d= -f2 | tr -d ' ')
echo " devptr=$DEV"
if [ -n "${DEV:-}" ]; then
STG=$(printf '0x%x' $((DEV+0x3c00100)))
echo " staging=$STG"
echo " -- walk_probe staging"; sudo sh -c "echo $STG > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1
echo " -- walk_probe devptr"; sudo sh -c "echo $DEV > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1
fi
echo " -- copy-dst lines so far"
sudo dmesg | grep -aE "sema-copy-dst|sema-svc: applied|sema-256" | tail -30
echo "[*] $(date +%T) waiting for the roundtrip"
for i in $(seq 1 90); do
sleep 5
grep -qa "roundtrip\|ALL OK\|FAILED rc=" "$SER" 2>/dev/null && { echo " result after ~$((i*5))s"; break; }
done
sleep 8
echo "[*] $(date +%T) post-run: walk state + copy-dst log"
cp "$SER" "$OUT/pdwatch_serial.log" 2>/dev/null
sudo dmesg > "$OUT/pdwatch_dmesg.log" 2>/dev/null
if [ -n "${STG:-}" ]; then
echo " -- post walk_probe staging"; sudo sh -c "echo $STG > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1
sudo dmesg | tail -12
fi
echo " -- copy-dst summary"
grep -aE "sema-copy-dst|sema-grce: inline write|sema-256|sema-lda" "$OUT/pdwatch_dmesg.log" | tail -60
echo " -- stats"
grep -a "rebuild #" "$OUT/pdwatch_dmesg.log" | tail -2
echo "[*] $(date +%T) done"
tail -3 "$SER" 2>/dev/null | strings | tail -3
|
