tools: cuda_cycle.sh - one-command CUDA bring-up attempt

Restart the VM, push the guest tools, start cdt holding on failure, wait
for the 20 GR channels, register/pin the completion counters and arm
scanerr.py, so each iteration is a single command. scanerr now loops over
scanner hits until a hit carries a non-zero RM error record.

2 files changed, 83 insertions, 21 deletions

diff --git a/tools/guest-diag/cuda_cycle.sh b/tools/guest-diag/cuda_cycle.sh
new file mode 100755
index 0000000..2246322
--- /dev/null
+++ b/tools/guest-diag/cuda_cycle.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# cuda_cycle.sh - one full CUDA bring-up attempt: restart the VM, push the
+# guest tools, start cdt holding on failure, wait for the 20 GR channels,
+# register/pin the completion counters, and arm the RM-error scanner so the
+# next forced 999 names its record.
+#
+# Usage: tools/guest-diag/cuda_cycle.sh [hold-seconds]
+# Output: guest /tmp/cdt.out, /tmp/scanerr.out; host sema-svc log in dmesg.
+set -uo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$HERE/../.."
+VMEXEC="$REPO/scripts/vgpu_vm.py"
+HOLD="${1:-600}"
+
+echo "[*] vmrestart"
+timeout 150 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }
+
+echo "[*] waiting for guest"
+for i in $(seq 1 20); do
+    if timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP; then
+        break
+    fi
+    sleep 5
+done
+
+echo "[*] pushing guest tools"
+timeout 240 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || { echo "push failed"; exit 1; }
+
+echo "[*] starting cdt (CDT_HOLD=$HOLD)"
+timeout 30 "$VMEXEC" exec \
+    "rm -f /tmp/cdt.out /tmp/scanerr.out; CDT_HOLD=$HOLD nohup /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
+    >/dev/null 2>&1
+
+echo "[*] waiting for 20 GR channels"
+for i in $(seq 1 24); do
+    sleep 8
+    echo 1 | sudo tee /sys/module/open_iov_vfio/parameters/chan_dump >/dev/null
+    sleep 1
+    n=$(sudo dmesg -T | grep "sema-chan-dump" | grep "eng=0" | wc -l)
+    if [ "$n" -ge 20 ]; then
+        echo "    GR channels up (poll $i)"
+        break
+    fi
+done
+
+echo "[*] injecting counters"
+timeout 150 "$HERE/inject_counters.sh" 2>/dev/null | grep -c registered
+
+PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null | grep -E '^[0-9]+$' | head -1)
+echo "[*] cdt pid=$PID; arming scanerr"
+timeout 30 "$VMEXEC" exec \
+    "nohup gdb -p $PID -batch -x /tmp/scanerr.py >/tmp/scanerr.out 2>&1 & echo ARMED" \
+    >/dev/null 2>&1
+
+echo "[*] cycle armed: watch /tmp/cdt.out and /tmp/scanerr.out in the guest"
diff --git a/tools/guest-diag/scanerr.py b/tools/guest-diag/scanerr.py
index c3af2f5..96c4170 100755
--- a/tools/guest-diag/scanerr.py
+++ b/tools/guest-diag/scanerr.py
@@ -1,8 +1,8 @@
 # gdb script: at the libcuda error-scanner entry (FUN_0054fa00), walk the
 # context list and dump every non-zero RM error record (magic words,
-# errorClass at +0x8, shorts at +0xc/+0xe). This names the source of a
-# forced 999: the scanner returns 999 when param_2 != 0 and any record's
-# class fields are non-zero.
+# errorClass at +0x8, shorts at +0xc/+0xe). The scanner is called often with
+# a zeroed record; keep going until a hit has a non-zero record (that is the
+# one a forced 999 comes from) and print it with a backtrace.
 # Run: sudo gdb -p <cdt-pid> -batch -x /tmp/scanerr.py
 import gdb
 
@@ -20,11 +20,6 @@ if base is None:
 gdb.execute("set pagination off")
 gdb.execute("set confirm off")
 gdb.execute("break *0x%x" % (base + 0x44fa00))
-try:
-    gdb.execute("continue")
-except gdb.error as e:
-    print("CONTINUE %s" % e)
-print("HIT")
 
 
 def rd64(a):
@@ -41,12 +36,21 @@ def rd32(a):
         return None
 
 
-try:
-    p1 = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
-    p2 = int(gdb.parse_and_eval("$rsi")) & 0xffffffff
-    print("param_1=0x%x param_2=%d" % (p1, p2))
+for hit in range(400):
+    try:
+        gdb.execute("continue")
+    except gdb.error as e:
+        print("CONTINUE_END %s" % e)
+        break
+    try:
+        p1 = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
+        p2 = int(gdb.parse_and_eval("$rsi")) & 0xffffffff
+    except Exception:
+        print("REGS_GONE")
+        break
     pl = rd64(p1)
     n = 0
+    found = 0
     while pl and n < 64:
         mgr = rd64(pl + 0x1291 * 8)
         if mgr:
@@ -56,18 +60,20 @@ try:
                 if rec:
                     vals = [rd32(rec + i) or 0 for i in range(0, 0x28, 4)]
                     if any(vals):
-                        print("ENTRY %d ctx=0x%x mgr=0x%x rec=0x%x rec=%s" %
-                              (n, pl, mgr, rec,
+                        print("HIT %d param_2=%d ENTRY %d ctx=0x%x mgr=0x%x rec=0x%x rec=%s" %
+                              (hit, p2, n, pl, mgr, rec,
                                ' '.join('0x%08x' % v for v in vals)))
+                        found = 1
         pl = rd64(pl + 0x128e * 8)
         n += 1
-    print("WALKED %d" % n)
-except Exception as e:
-    print("dump_fail %s" % e)
-try:
-    gdb.execute("bt 8", to_string=True)
-except Exception:
-    pass
+    if found:
+        print("FOUND")
+        try:
+            print(gdb.execute("bt 12", to_string=True))
+        except Exception:
+            pass
+        break
+print("DONE")
 try:
     gdb.execute("detach")
 except Exception: