#!/bin/bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2026 Open-IOV contributors
# pdwatch_cycle.sh - TASK 2: watch every write landing in the root PD pages
# 0x200000..0x204000 (and the 0x208000 table) through a full cdt2 run, with
# the walk state sampled during the CP park and after completion.
set -uo pipefail
HERE=/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/guest-diag
REPO=/home/rmcore/Git/integdev_gpu_drv/open-iov
VMEXEC="$REPO/scripts/vgpu_vm.py"
OUT=/tmp/opencode
SER=/tmp/vgpu-guest-serial.log
LOG=$OUT/pdwatch_cycle.log

exec > >(tee -a "$LOG") 2>&1

echo "[*] $(date +%T) vmrestart"
timeout 200 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }

for i in $(seq 1 30); do
    timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP && { echo "    guest up"; break; }
    sleep 5
done

echo "[*] $(date +%T) params: sema_dbg=Y, dst watch 0x200000 + 0x208000"
sudo sh -c "echo Y > /sys/module/vgsp/parameters/sema_dbg; \
echo N > /sys/module/vgsp/parameters/sema_fake_rel; \
echo N > /sys/module/vgsp/parameters/sema_skip_ops; \
echo 64 > /sys/module/vgsp/parameters/sema_ops_log; \
echo 0 > /sys/module/vgsp/parameters/sema_copy_watch; \
echo 0 > /sys/module/vgsp/parameters/sema_copy_pat; \
echo 0x200000 > /sys/module/vgsp/parameters/sema_copy_dst; \
echo 0x208000 > /sys/module/vgsp/parameters/sema_copy_dst2; \
echo 8000 > /sys/module/vgsp/parameters/sema_ce_grace_ms"

echo "[*] $(date +%T) push tools"
timeout 600 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || echo "    push failed"

echo "[*] $(date +%T) nvidia modules + kprobes"
timeout 120 "$VMEXEC" exec "nvidia-smi -L >/dev/null 2>&1; lsmod | grep -q nvidia_uvm && echo UVM_LOADED" 2>/dev/null | tail -1
timeout 60 "$VMEXEC" exec "sh /tmp/kprobes.sh" 2>/dev/null | grep -E "ok|FAIL" | head -8

: > "$OUT/sem_stream.txt"
( setsid nohup env SEMFIX_COUNTER_RETRY=10 python3 "$HERE/pin_gr_stream.py" > "$OUT/pw_pin.log" 2>&1 & )
sleep 3
( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 python3 "$HERE/semfix_uvmt.py" 0.5 > "$OUT/pw_semfix.log" 2>&1 & )
sleep 3

sudo dmesg -C 2>/dev/null
: > "$SER" 2>/dev/null || true

echo "[*] $(date +%T) launching cdt2 (CP pause 240)"
timeout 40 "$VMEXEC" exec \
  "echo '=== PDWATCH RUN START ===' > /dev/ttyS0; rm -f /tmp/cdt2.out; nohup sh -c 'CDT_CP_PAUSE=240 CDT_WAIT=60 /tmp/cdt2 2>&1 | tee /tmp/cdt2.out | stdbuf -oL tee /dev/ttyS0 >/dev/null' >/dev/null 2>&1 & echo STARTED" 2>/dev/null | tail -1

for i in $(seq 1 60); do
    sleep 5
    grep -qa "parked 240" "$SER" 2>/dev/null && { echo "    parked after ~$((i*5))s"; break; }
done

echo "[*] $(date +%T) shadow dumps during park"
for a in 0x200000 0x201000 0x202000 0x203000 0x208000; do
  echo "  -- $a"
  sudo sh -c "echo '$a,64' > /sys/module/vgsp/parameters/sema_vidmem_dump" 2>/dev/null
  sleep 0.2
done
sudo dmesg | grep -a "sema-vidmem\|sema-walk-probe\|sema-svc: VA space" | tail -40 > "$OUT/pdwatch_park_dmesg.txt"
cat "$OUT/pdwatch_park_dmesg.txt"

DEV=$(grep -ao 'device ptr = 0x[0-9a-f]*' "$SER" | tail -1 | cut -d= -f2 | tr -d ' ')
echo "    devptr=$DEV"
if [ -n "${DEV:-}" ]; then
  STG=$(printf '0x%x' $((DEV+0x3c00100)))
  echo "    staging=$STG"
  echo "  -- walk_probe staging"; sudo sh -c "echo $STG > /sys/module/vgsp/parameters/walk_probe"; sleep 1
  echo "  -- walk_probe devptr";  sudo sh -c "echo $DEV > /sys/module/vgsp/parameters/walk_probe"; sleep 1
fi
echo "  -- copy-dst lines so far"
sudo dmesg | grep -aE "sema-copy-dst|sema-svc: applied|sema-256" | tail -30

echo "[*] $(date +%T) waiting for the roundtrip"
for i in $(seq 1 90); do
    sleep 5
    grep -qa "roundtrip\|ALL OK\|FAILED rc=" "$SER" 2>/dev/null && { echo "    result after ~$((i*5))s"; break; }
done

sleep 8
echo "[*] $(date +%T) post-run: walk state + copy-dst log"
cp "$SER" "$OUT/pdwatch_serial.log" 2>/dev/null
sudo dmesg > "$OUT/pdwatch_dmesg.log" 2>/dev/null
if [ -n "${STG:-}" ]; then
  echo "  -- post walk_probe staging"; sudo sh -c "echo $STG > /sys/module/vgsp/parameters/walk_probe"; sleep 1
  sudo dmesg | tail -12
fi
echo "  -- copy-dst summary"
grep -aE "sema-copy-dst|sema-grce: inline write|sema-256|sema-lda" "$OUT/pdwatch_dmesg.log" | tail -60
echo "  -- stats"
grep -a "rebuild #" "$OUT/pdwatch_dmesg.log" | tail -2
echo "[*] $(date +%T) done"
tail -3 "$SER" 2>/dev/null | strings | tail -3