guest-diag: run_cycle gdb-launch mode + bpz18 early path trace

2 files changed, 96 insertions, 2 deletions

diff --git a/tools/guest-diag/bpz18.py b/tools/guest-diag/bpz18.py
new file mode 100644
index 0000000..ed82084
--- /dev/null
+++ b/tools/guest-diag/bpz18.py
@@ -0,0 +1,82 @@
+# gdb script: trace the path through FUN_0042aec0 (GPU arch/config init) in
+# the guest, launched as `gdb --args /tmp/cdt` so early device-init code is
+# caught.
+#
+# Host: c48=6 c4c=0x27 c50=7 c54=5 e94=0x18. Guest reaches same c48..c54 but
+# e94=0, so some branch inside FUN_0042aec0 diverges. This logs hits on:
+#   0x32aec0 FUN_0042aec0 entry
+#   0x32b0b7 after chip-info store
+#   0x32c911 the 0xc4c dispatch
+#   0x32e8be case index 1 body (sets e94=0x18)
+#   0x32e46a default target of the dispatch
+#   0x2e3e80 FUN_003e3e80 (terminator; dumps ds fields)
+import gdb
+import time
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+gdb.execute("set environment CDT_PAUSE=60")
+gdb.execute("set environment CDT_HOLD=900")
+gdb.execute("catch load libcuda.so")
+gdb.execute("run")
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    raise SystemExit
+
+SITES = [
+    (0x32aec0, "F42AEC0_ENTRY"),
+    (0x32b0b7, "AFTER_CHIPINFO"),
+    (0x32c911, "C4C_DISPATCH"),
+    (0x32e8be, "CASE1_e94_0x18"),
+    (0x32e46a, "DISPATCH_DEFAULT"),
+]
+for off, name in SITES:
+    gdb.execute("break *0x%x" % (base + off))
+CALLER_RET = base + 0xd0eae1
+gdb.execute("break *0x%x" % (base + 0x2e3e80))
+
+def rd(addr, n=4, fmt="unsigned int"):
+    try:
+        return int(gdb.parse_and_eval("*(%s*)0x%x" % (fmt, addr))) & ((1 << (8 * n)) - 1)
+    except Exception:
+        return None
+
+t0 = time.time()
+for hit in range(120):
+    try:
+        gdb.execute("continue")
+    except gdb.error as e:
+        print("CONTINUE_END %s" % e)
+        break
+    rip = int(gdb.parse_and_eval("$rip")) & 0xffffffffffffffff
+    off = rip - base
+    if off == 0x2e3e80:
+        ret = rd(int(gdb.parse_and_eval("$rsp")), 8, "unsigned long")
+        if ret != CALLER_RET:
+            continue
+        rdi = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
+        ds = rd(rdi + 0x40, 8, "unsigned long")
+        print("TERMINATOR FUN_003e3e80 dev=0x%x ds=0x%x t=%.3f" % (rdi, ds or 0, time.time() - t0))
+        for o in [0xc48, 0xc4c, 0xc50, 0xc54, 0xe94, 0x4354, 0x426c]:
+            print("  ds+0x%x = 0x%x" % (o, rd(ds + o) or 0))
+        break
+    name = dict((o, n) for o, n in SITES).get(off, "?")
+    rcx = int(gdb.parse_and_eval("$rcx")) & 0xffffffffffffffff
+    eax = rd("$eax") if False else (int(gdb.parse_and_eval("$eax")) & 0xffffffff)
+    extra = ""
+    if off == 0x32c911:
+        extra = " c4c=0x%x arch=0x%x" % (rd(rcx + 0xc4c) or 0, rd(rcx + 0xc50) or 0)
+    print("HIT t=%.3f +0x%x %s eax=0x%x rcx=0x%x%s" %
+          (time.time() - t0, off, name, eax, rcx, extra))
+
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
diff --git a/tools/guest-diag/run_cycle.sh b/tools/guest-diag/run_cycle.sh
index 71db054..b97928e 100755
--- a/tools/guest-diag/run_cycle.sh
+++ b/tools/guest-diag/run_cycle.sh
@@ -89,6 +89,16 @@ elif [ "$TRACE" = "gdb" ]; then
         sleep 8
     done
     [ "$ARMED" = 1 ] || echo "    WARN: gdb arm failed 3x"
+elif [ "$TRACE" = "gdb-launch" ]; then
+    # Launch cdt INSIDE gdb so device-init code that runs before the
+    # CDT_PAUSE (cuInit/cuDeviceGet) is visible to the breakpoints.
+    timeout 30 "$VMEXEC" exec "rm -f /tmp/cdt.out /tmp/cdtgdb.out; sync; echo CLEARED" >/dev/null 2>&1
+    timeout 30 "$VMEXEC" exec \
+        "rm -f /tmp/cdt.out /tmp/cdtgdb.out; nohup sh -c 'gdb -batch -x /tmp/${GDBSCRIPT:-bp999v2.py} --args /tmp/cdt > /tmp/cdtgdb.out 2>&1' & echo STARTED" \
+        >/dev/null 2>&1
+    sleep 3
+    PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null | grep -E '^[0-9]+$' | head -1)
+    echo "    launched under gdb; cdt pid=${PID:-?} script=${GDBSCRIPT:-bp999v2.py}"
 else
     timeout 30 "$VMEXEC" exec \
         "rm -f /tmp/cdt.out; CDT_HOLD=$HOLD nohup /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
@@ -96,9 +106,11 @@ else
 fi
 
 echo "[*] waiting for the run to finish (watch cdt.out)"
+POLL_FILE=/tmp/cdt.out
+[ "$TRACE" = "gdb-launch" ] && POLL_FILE=/tmp/cdtgdb.out
 for i in $(seq 1 90); do
     sleep 2
-    OUT_TXT=$(timeout 20 "$VMEXEC" exec "cat /tmp/cdt.out 2>/dev/null" 2>/dev/null | tail -6)
+    OUT_TXT=$(timeout 20 "$VMEXEC" exec "cat $POLL_FILE 2>/dev/null" 2>/dev/null | tail -6)
     if echo "$OUT_TXT" | grep -q "FAILED rc="; then
         echo "    failure observed after ~$((i*2))s"
         break
@@ -106,7 +118,7 @@ for i in $(seq 1 90); do
 done
 
 echo "[*] snapshotting guest dmesg"
-timeout 60 "$VMEXEC" exec "dmesg > /tmp/guestdmesg.log; cp /tmp/cdt.out /tmp/cdt.out.snap; [ -f /tmp/strace.log ] && cp /tmp/strace.log /tmp/strace.log.snap; [ -f /tmp/bpwait.out ] && cp /tmp/bpwait.out /tmp/bpwait.out.snap; echo DONE" >/dev/null 2>&1
+timeout 60 "$VMEXEC" exec "dmesg > /tmp/guestdmesg.log; cp /tmp/cdt.out /tmp/cdt.out.snap; [ -f /tmp/strace.log ] && cp /tmp/strace.log /tmp/strace.log.snap; [ -f /tmp/bpwait.out ] && cp /tmp/bpwait.out /tmp/bpwait.out.snap; [ -f /tmp/cdtgdb.out ] && cp /tmp/cdtgdb.out /tmp/bpwait.out.snap; echo DONE" >/dev/null 2>&1
 sleep 1
 sudo kill "$DMESG_PID" 2>/dev/null