guest-diag: PD-watch cycle (dst-keyed root-PD watch + park-time shadow dumps)

Reproducible TASK 2 harness: fresh boot, sema_dbg=Y with
sema_copy_dst=0x200000 / sema_copy_dst2=0x208000, cdt2 parked at
CDT_CP_PAUSE so the root PD/PT pages can be dumped and walk_probe run
while the client is live. Prints the offending write data (prov= names
the translation layer) and the post-run walk state.

1 files changed, 98 insertions, 0 deletions

diff --git a/tools/guest-diag/pdwatch_cycle.sh b/tools/guest-diag/pdwatch_cycle.sh
new file mode 100755
index 0000000..45d08cc
--- /dev/null
+++ b/tools/guest-diag/pdwatch_cycle.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+# pdwatch_cycle.sh - TASK 2: watch every write landing in the root PD pages
+# 0x200000..0x204000 (and the 0x208000 table) through a full cdt2 run, with
+# the walk state sampled during the CP park and after completion.
+set -uo pipefail
+HERE=/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/guest-diag
+REPO=/home/rmcore/Git/integdev_gpu_drv/open-iov
+VMEXEC="$REPO/scripts/vgpu_vm.py"
+OUT=/tmp/opencode
+SER=/tmp/vgpu-guest-serial.log
+LOG=$OUT/pdwatch_cycle.log
+
+exec > >(tee -a "$LOG") 2>&1
+
+echo "[*] $(date +%T) vmrestart"
+timeout 200 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }
+
+for i in $(seq 1 30); do
+    timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP && { echo "    guest up"; break; }
+    sleep 5
+done
+
+echo "[*] $(date +%T) params: sema_dbg=Y, dst watch 0x200000 + 0x208000"
+sudo sh -c "echo Y > /sys/module/open_iov_vfio/parameters/sema_dbg; \
+echo N > /sys/module/open_iov_vfio/parameters/sema_fake_rel; \
+echo N > /sys/module/open_iov_vfio/parameters/sema_skip_ops; \
+echo 64 > /sys/module/open_iov_vfio/parameters/sema_ops_log; \
+echo 0 > /sys/module/open_iov_vfio/parameters/sema_copy_watch; \
+echo 0 > /sys/module/open_iov_vfio/parameters/sema_copy_pat; \
+echo 0x200000 > /sys/module/open_iov_vfio/parameters/sema_copy_dst; \
+echo 0x208000 > /sys/module/open_iov_vfio/parameters/sema_copy_dst2; \
+echo 8000 > /sys/module/open_iov_vfio/parameters/sema_ce_grace_ms"
+
+echo "[*] $(date +%T) push tools"
+timeout 600 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || echo "    push failed"
+
+echo "[*] $(date +%T) nvidia modules + kprobes"
+timeout 120 "$VMEXEC" exec "nvidia-smi -L >/dev/null 2>&1; lsmod | grep -q nvidia_uvm && echo UVM_LOADED" 2>/dev/null | tail -1
+timeout 60 "$VMEXEC" exec "sh /tmp/kprobes.sh" 2>/dev/null | grep -E "ok|FAIL" | head -8
+
+: > "$OUT/sem_stream.txt"
+( setsid nohup env SEMFIX_COUNTER_RETRY=10 python3 "$HERE/pin_gr_stream.py" > "$OUT/pw_pin.log" 2>&1 & )
+sleep 3
+( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 python3 "$HERE/semfix_uvmt.py" 0.5 > "$OUT/pw_semfix.log" 2>&1 & )
+sleep 3
+
+sudo dmesg -C 2>/dev/null
+: > "$SER" 2>/dev/null || true
+
+echo "[*] $(date +%T) launching cdt2 (CP pause 240)"
+timeout 40 "$VMEXEC" exec \
+  "echo '=== PDWATCH RUN START ===' > /dev/ttyS0; rm -f /tmp/cdt2.out; nohup sh -c 'CDT_CP_PAUSE=240 CDT_WAIT=60 /tmp/cdt2 2>&1 | tee /tmp/cdt2.out | stdbuf -oL tee /dev/ttyS0 >/dev/null' >/dev/null 2>&1 & echo STARTED" 2>/dev/null | tail -1
+
+for i in $(seq 1 60); do
+    sleep 5
+    grep -qa "parked 240" "$SER" 2>/dev/null && { echo "    parked after ~$((i*5))s"; break; }
+done
+
+echo "[*] $(date +%T) shadow dumps during park"
+for a in 0x200000 0x201000 0x202000 0x203000 0x208000; do
+  echo "  -- $a"
+  sudo sh -c "echo '$a,64' > /sys/module/open_iov_vfio/parameters/sema_vidmem_dump" 2>/dev/null
+  sleep 0.2
+done
+sudo dmesg | grep -a "sema-vidmem\|sema-walk-probe\|sema-svc: VA space" | tail -40 > "$OUT/pdwatch_park_dmesg.txt"
+cat "$OUT/pdwatch_park_dmesg.txt"
+
+DEV=$(grep -ao 'device ptr = 0x[0-9a-f]*' "$SER" | tail -1 | cut -d= -f2 | tr -d ' ')
+echo "    devptr=$DEV"
+if [ -n "${DEV:-}" ]; then
+  STG=$(printf '0x%x' $((DEV+0x3c00100)))
+  echo "    staging=$STG"
+  echo "  -- walk_probe staging"; sudo sh -c "echo $STG > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1
+  echo "  -- walk_probe devptr";  sudo sh -c "echo $DEV > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1
+fi
+echo "  -- copy-dst lines so far"
+sudo dmesg | grep -aE "sema-copy-dst|sema-svc: applied|sema-256" | tail -30
+
+echo "[*] $(date +%T) waiting for the roundtrip"
+for i in $(seq 1 90); do
+    sleep 5
+    grep -qa "roundtrip\|ALL OK\|FAILED rc=" "$SER" 2>/dev/null && { echo "    result after ~$((i*5))s"; break; }
+done
+
+sleep 8
+echo "[*] $(date +%T) post-run: walk state + copy-dst log"
+cp "$SER" "$OUT/pdwatch_serial.log" 2>/dev/null
+sudo dmesg > "$OUT/pdwatch_dmesg.log" 2>/dev/null
+if [ -n "${STG:-}" ]; then
+  echo "  -- post walk_probe staging"; sudo sh -c "echo $STG > /sys/module/open_iov_vfio/parameters/walk_probe"; sleep 1
+  sudo dmesg | tail -12
+fi
+echo "  -- copy-dst summary"
+grep -aE "sema-copy-dst|sema-grce: inline write|sema-256|sema-lda" "$OUT/pdwatch_dmesg.log" | tail -60
+echo "  -- stats"
+grep -a "rebuild #" "$OUT/pdwatch_dmesg.log" | tail -2
+echo "[*] $(date +%T) done"
+tail -3 "$SER" 2>/dev/null | strings | tail -3