diff options
| -rwxr-xr-x | tools/guest-diag/pdwatch_cycle.sh | 98 |
1 files changed, 98 insertions, 0 deletions
diff --git a/tools/guest-diag/pdwatch_cycle.sh b/tools/guest-diag/pdwatch_cycle.sh new file mode 100755 index 0000000..45d08cc --- /dev/null +++ b/tools/guest-diag/pdwatch_cycle.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# pdwatch_cycle.sh - TASK 2: watch every write landing in the root PD pages +# 0x200000..0x204000 (and the 0x208000 table) through a full cdt2 run, with +# the walk state sampled during the CP park and after completion. +set -uo pipefail +HERE=/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/guest-diag +REPO=/home/rmcore/Git/integdev_gpu_drv/open-iov +VMEXEC="$REPO/scripts/vgpu_vm.py" +OUT=/tmp/opencode +SER=/tmp/vgpu-guest-serial.log +LOG=$OUT/pdwatch_cycle.log + +exec > >(tee -a "$LOG") 2>&1 + +echo "[*] $(date +%T) vmrestart" +timeout 200 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; } + +for i in $(seq 1 30); do + timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP && { echo " guest up"; break; } + sleep 5 +done + +echo "[*] $(date +%T) params: sema_dbg=Y, dst watch 0x200000 + 0x208000" +sudo sh -c "echo Y > /sys/module/open_iov_vfio/parameters/sema_dbg; \ +echo N > /sys/module/open_iov_vfio/parameters/sema_fake_rel; \ +echo N > /sys/module/open_iov_vfio/parameters/sema_skip_ops; \ +echo 64 > /sys/module/open_iov_vfio/parameters/sema_ops_log; \ +echo 0 > /sys/module/open_iov_vfio/parameters/sema_copy_watch; \ +echo 0 > /sys/module/open_iov_vfio/parameters/sema_copy_pat; \ +echo 0x200000 > /sys/module/open_iov_vfio/parameters/sema_copy_dst; \ +echo 0x208000 > /sys/module/open_iov_vfio/parameters/sema_copy_dst2; \ +echo 8000 > /sys/module/open_iov_vfio/parameters/sema_ce_grace_ms" + +echo "[*] $(date +%T) push tools" +timeout 600 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || echo " push failed" + +echo "[*] $(date +%T) nvidia modules + kprobes" +timeout 120 "$VMEXEC" exec "nvidia-smi -L >/dev/null 2>&1; lsmod | grep -q nvidia_uvm && echo UVM_LOADED" 2>/dev/null | tail -1 +timeout 60 "$VMEXEC" exec "sh /tmp/kprobes.sh" 2>/dev/null | grep -E "ok|FAIL" | head -8 + +: > "$OUT/sem_stream.txt" +( setsid nohup env SEMFIX_COUNTER_RETRY=10 python3 "$HERE/pin_gr_stream.py" > "$OUT/pw_pin.log" 2>&1 & ) +sleep 3 +( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 python3 "$HERE/semfix_uvmt.py" 0.5 > "$OUT/pw_semfix.log" 2>&1 & ) +sleep 3 + +sudo dmesg -C 2>/dev/null +: > "$SER" 2>/dev/null || true + +echo "[*] $(date +%T) launching cdt2 (CP pause 240)" +timeout 40 "$VMEXEC" exec \ + "echo '=== PDWATCH RUN START ===' > /dev/ttyS0; rm -f /tmp/cdt2.out; nohup sh -c 'CDT_CP_PAUSE=240 CDT_WAIT=60 /tmp/cdt2 2>&1 | tee /tmp/cdt2.out | stdbuf -oL tee /dev/ttyS0 >/dev/null' >/dev/null 2>&1 & echo STARTED" 2>/dev/null | tail -1 + +for i in $(seq 1 60); do + sleep 5 + grep -qa "parked 240" "$SER" 2>/dev/null && { echo " parked after ~$((i*5))s"; break; } +done + +echo "[*] $(date +%T) shadow dumps during park" +for a in 0x200000 0x201000 0x202000 0x203000 0x208000; do + echo " -- $a" + sudo sh -c "echo '$a,64' > /sys/module/open_iov_vfio/parameters/sema_vidmem_dump" 2>/dev/null + sleep 0.2 +done +sudo dmesg | grep -a "sema-vidmem\|sema-walk-probe\|sema-svc: VA space" | tail -40 > "$OUT/pdwatch_park_dmesg.txt" +cat "$OUT/pdwatch_park_dmesg.txt" + +DEV=$(grep -ao 'device ptr = 0x[0-9a-f]*' "$SER" | tail -1 | cut -d= -f2 | tr -d ' ') +echo " devptr=$DEV" +if [ -n "${DEV:-}" ]; then + STG=$(printf '0x%x' $((DEV+0x3c00100))) + echo " staging=$STG" + echo " -- walk_probe staging"; sudo sh -c "echo $STG > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1 + echo " -- walk_probe devptr"; sudo sh -c "echo $DEV > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1 +fi +echo " -- copy-dst lines so far" +sudo dmesg | grep -aE "sema-copy-dst|sema-svc: applied|sema-256" | tail -30 + +echo "[*] $(date +%T) waiting for the roundtrip" +for i in $(seq 1 90); do + sleep 5 + grep -qa "roundtrip\|ALL OK\|FAILED rc=" "$SER" 2>/dev/null && { echo " result after ~$((i*5))s"; break; } +done + +sleep 8 +echo "[*] $(date +%T) post-run: walk state + copy-dst log" +cp "$SER" "$OUT/pdwatch_serial.log" 2>/dev/null +sudo dmesg > "$OUT/pdwatch_dmesg.log" 2>/dev/null +if [ -n "${STG:-}" ]; then + echo " -- post walk_probe staging"; sudo sh -c "echo $STG > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1 + sudo dmesg | tail -12 +fi +echo " -- copy-dst summary" +grep -aE "sema-copy-dst|sema-grce: inline write|sema-256|sema-lda" "$OUT/pdwatch_dmesg.log" | tail -60 +echo " -- stats" +grep -a "rebuild #" "$OUT/pdwatch_dmesg.log" | tail -2 +echo "[*] $(date +%T) done" +tail -3 "$SER" 2>/dev/null | strings | tail -3 |