tools: cuda_cycle.sh - one-command CUDA bring-up attempt

Restart the VM, push the guest tools, start cdt holding on failure, wait
for the 20 GR channels, register/pin the completion counters and arm
scanerr.py, so each iteration is a single command. scanerr now loops over
scanner hits until a hit carries a non-zero RM error record.

1 files changed, 56 insertions, 0 deletions

diff --git a/tools/guest-diag/cuda_cycle.sh b/tools/guest-diag/cuda_cycle.sh
new file mode 100755
index 0000000..2246322
--- /dev/null
+++ b/tools/guest-diag/cuda_cycle.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# cuda_cycle.sh - one full CUDA bring-up attempt: restart the VM, push the
+# guest tools, start cdt holding on failure, wait for the 20 GR channels,
+# register/pin the completion counters, and arm the RM-error scanner so the
+# next forced 999 names its record.
+#
+# Usage: tools/guest-diag/cuda_cycle.sh [hold-seconds]
+# Output: guest /tmp/cdt.out, /tmp/scanerr.out; host sema-svc log in dmesg.
+set -uo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$HERE/../.."
+VMEXEC="$REPO/scripts/vgpu_vm.py"
+HOLD="${1:-600}"
+
+echo "[*] vmrestart"
+timeout 150 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }
+
+echo "[*] waiting for guest"
+for i in $(seq 1 20); do
+    if timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP; then
+        break
+    fi
+    sleep 5
+done
+
+echo "[*] pushing guest tools"
+timeout 240 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || { echo "push failed"; exit 1; }
+
+echo "[*] starting cdt (CDT_HOLD=$HOLD)"
+timeout 30 "$VMEXEC" exec \
+    "rm -f /tmp/cdt.out /tmp/scanerr.out; CDT_HOLD=$HOLD nohup /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
+    >/dev/null 2>&1
+
+echo "[*] waiting for 20 GR channels"
+for i in $(seq 1 24); do
+    sleep 8
+    echo 1 | sudo tee /sys/module/open_iov_vfio/parameters/chan_dump >/dev/null
+    sleep 1
+    n=$(sudo dmesg -T | grep "sema-chan-dump" | grep "eng=0" | wc -l)
+    if [ "$n" -ge 20 ]; then
+        echo "    GR channels up (poll $i)"
+        break
+    fi
+done
+
+echo "[*] injecting counters"
+timeout 150 "$HERE/inject_counters.sh" 2>/dev/null | grep -c registered
+
+PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null | grep -E '^[0-9]+$' | head -1)
+echo "[*] cdt pid=$PID; arming scanerr"
+timeout 30 "$VMEXEC" exec \
+    "nohup gdb -p $PID -batch -x /tmp/scanerr.py >/tmp/scanerr.out 2>&1 & echo ARMED" \
+    >/dev/null 2>&1
+
+echo "[*] cycle armed: watch /tmp/cdt.out and /tmp/scanerr.out in the guest"