guest-diag: post-pump ctx-init status probes (bpz/bpz2 gdb scripts)

bpz.py probes the four known rc=999 stores on the cuCtxCreate path plus the
scanner force sites; the live run eliminated all of them and showed the
engine event pump (FUN_0031a3b0) returning success. bpz2.py probes every
post-pump status store in the ctx bring-up (FUN_00374240) and the function's
final return, printing the first non-zero status with a backtrace.

3 files changed, 174 insertions, 0 deletions

diff --git a/tools/guest-diag/bpz.py b/tools/guest-diag/bpz.py
new file mode 100644
index 0000000..53d46a6
--- /dev/null
+++ b/tools/guest-diag/bpz.py
@@ -0,0 +1,92 @@
+# gdb script: name the silent-rollback 999 site in the cuCtxCreate path.
+#
+# Sites (file vaddr of libcuda.so.570.144; runtime = base + off):
+#   0x222e99  FUN_00322190: cmp $0x88,%eax (work-submission offset delta
+#             computed from two RM helper calls); next insn jumps to the
+#             999 store when delta > 0x88.
+#   0x223402  FUN_00322190: rc=999 (delta > 0x88)
+#   0x223452  FUN_00322190: rc=999 (neither GR nor CE engine pointer found)
+#   0x2231ab  FUN_00322190: store of FUN_0031a3b0 (engine event pump) return
+#   0x277840  FUN_00377680: rc=999 (context-register helper failed)
+#   0x27640b  FUN_00374240: rc=999 (malloc(0x40) failed)
+#   0x44fb80  FUN_0054fa00: scanner forced 999
+#   0x44fb6c  FUN_0054fa00: scanner return 0x2cf
+#
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bpz.py
+import gdb
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    print("NO_LIBCUDA")
+    raise SystemExit
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+
+SITES = [
+    (0x222e99, "SIZEDELTA"),
+    (0x223402, "SIZE999"),
+    (0x223452, "ENGINE999"),
+    (0x2231ab, "AFTER_PUMP"),
+    (0x277840, "CTXREF999"),
+    (0x27640b, "MALLOC999"),
+    (0x44fb80, "SCAN999"),
+    (0x44fb6c, "SCAN2cf"),
+]
+for off, name in SITES:
+    gdb.execute("break *0x%x" % (base + off))
+
+for hit in range(80):
+    try:
+        gdb.execute("continue")
+    except gdb.error as e:
+        print("CONTINUE_END %s" % e)
+        break
+    try:
+        rip = int(gdb.parse_and_eval("$rip")) & 0xffffffffffffffff
+        rax = int(gdb.parse_and_eval("$rax")) & 0xffffffffffffffff
+        r14 = int(gdb.parse_and_eval("$r14")) & 0xffffffffffffffff
+        r12 = int(gdb.parse_and_eval("$r12")) & 0xffffffffffffffff
+    except Exception as e:
+        print("REGS_GONE %s" % e)
+        break
+    off = rip - base
+    name = dict(SITES).get(off, "?")
+    print("=== HIT %d rip=0x%x off=0x%x %s rax=0x%x r14=0x%x r12=0x%x" %
+          (hit, rip, off, name, rax, r14, r12))
+    try:
+        if off in (0x222e99, 0x223402):
+            print("  delta(eax)=0x%x u32(r14)=0x%x u32(r14+0x14)=0x%x" %
+                  (rax & 0xffffffff,
+                   int(gdb.parse_and_eval("*(unsigned int *)0x%x" % r14)),
+                   int(gdb.parse_and_eval("*(unsigned int *)0x%x" % (r14 + 0x14)))))
+        if off == 0x223452:
+            print("  ctx+0x9830=0x%x ctx+0x9838=0x%x" %
+                  (int(gdb.parse_and_eval("*(unsigned long *)0x%x" % (r12 + 0x9830))),
+                   int(gdb.parse_and_eval("*(unsigned long *)0x%x" % (r12 + 0x9838)))))
+    except Exception as e:
+        print("  deref_fail %s" % e)
+    try:
+        print(gdb.execute("bt 10", to_string=True))
+    except Exception as e:
+        print("bt_fail %s" % e)
+
+for t in gdb.selected_inferior().threads():
+    t.switch()
+    print("== tid %s" % t.num)
+    try:
+        print(gdb.execute("bt 6", to_string=True).rstrip())
+    except Exception as e:
+        print("bt_fail %s" % e)
+
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
diff --git a/tools/guest-diag/bpz2.py b/tools/guest-diag/bpz2.py
new file mode 100644
index 0000000..2d399ff
--- /dev/null
+++ b/tools/guest-diag/bpz2.py
@@ -0,0 +1,80 @@
+# gdb script: capture the post-pump failure call in FUN_00374240 (ctx init).
+#
+# From the previous run: the engine event pump FUN_0031a3b0 returned 0
+# (0x2231ab) and the four known 999 stores did NOT fire, so the rc=999 comes
+# from a call AFTER the pump on the setup path. Every one of those calls in
+# FUN_00374240 stores its status into r14d and jumps to 0x274680 on failure,
+# and the function returns r14d at 0x27428a. Break at each store site and at
+# the final return; print the status and a backtrace when it is non-zero.
+#
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bpz2.py
+import gdb
+import time
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    print("NO_LIBCUDA")
+    raise SystemExit
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+
+SITES = [
+    (0x2231ab, "AFTER_PUMP"),
+    (0x2233f7, "TAIL_SCAN"),
+    (0x27537d, "RET_CTXENG"),
+    (0x27539d, "RET_A4090"),
+    (0x2753da, "RET_35fb00"),
+    (0x275444, "RET_a5c50"),
+    (0x275474, "RET_ENG60"),
+    (0x27549d, "RET_ENG68"),
+    (0x2754c6, "RET_ENG70"),
+    (0x2754f6, "RET_ENG78"),
+    (0x275541, "RET_ENGLOOP"),
+    (0x27558c, "RET_ENG2f8"),
+    (0x2755bf, "RET_ENG300"),
+    (0x27428a, "CTX_RET"),
+    (0x277840, "CTXREF999"),
+    (0x44fb80, "SCAN999"),
+    (0x44fb6c, "SCAN2cf"),
+]
+for off, name in SITES:
+    gdb.execute("break *0x%x" % (base + off))
+
+t0 = time.time()
+for hit in range(200):
+    try:
+        gdb.execute("continue")
+    except gdb.error as e:
+        print("CONTINUE_END %s" % e)
+        break
+    try:
+        rip = int(gdb.parse_and_eval("$rip")) & 0xffffffffffffffff
+        rax = int(gdb.parse_and_eval("$rax")) & 0xffffffffffffffff
+        r14 = int(gdb.parse_and_eval("$r14")) & 0xffffffffffffffff
+    except Exception as e:
+        print("REGS_GONE %s" % e)
+        break
+    off = rip - base
+    name = dict(SITES).get(off, "?")
+    st = rax & 0xffffffff
+    print("HIT %d t=%.3f off=0x%x %s rax=0x%x r14d=0x%x" %
+          (hit, time.time() - t0, off, name, rax, r14 & 0xffffffff))
+    if st != 0 or off in (0x27428a, 0x277840, 0x44fb80, 0x44fb6c):
+        print("  >>> NONZERO/TERMINAL %s status=0x%x" % (name, st))
+        try:
+            print(gdb.execute("bt 12", to_string=True))
+        except Exception as e:
+            print("bt_fail %s" % e)
+
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 7114b03..2bfb2a9 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -28,6 +28,8 @@ push_b64 "$HERE/dumpctx.py"     /tmp/dumpctx.py
 push_b64 "$HERE/trkinfo.py"     /tmp/trkinfo.py
 push_b64 "$HERE/bp999.py"       /tmp/bp999.py
 push_b64 "$HERE/bp999v2.py"     /tmp/bp999v2.py
+push_b64 "$HERE/bpz.py"         /tmp/bpz.py
+push_b64 "$HERE/bpz2.py"        /tmp/bpz2.py
 push_b64 "$HERE/bpwait.py"      /tmp/bpwait.py
 push_b64 "$HERE/ctx999.py"      /tmp/ctx999.py
 push_b64 "$HERE/scanerr.py"     /tmp/scanerr.py