guest-diag: bpz3 - probe every r14d store in the ctx bring-up (85 sites)

Run #2: all post-pump call sites up to 0x2755bf returned 0, but FUN_00374240
returned r14d=0x3e7 (999) 554 ms later. bpz3 arms every mov %eax,%r14d store
in FUN_00374240 plus the alternate epilogue and the single direct 999 store,
printing the first non-zero status with a backtrace.

2 files changed, 152 insertions, 0 deletions

diff --git a/tools/guest-diag/bpz3.py b/tools/guest-diag/bpz3.py
new file mode 100644
index 0000000..ae90f86
--- /dev/null
+++ b/tools/guest-diag/bpz3.py
@@ -0,0 +1,151 @@
+# gdb script: find the exact site that sets r14d=999 in FUN_00374240.
+#
+# Run #2 showed every post-pump call site up to 0x2755bf returning 0, then
+# FUN_00374240 returning r14d=0x3e7 at 0x27428a 554 ms later. This scripts
+# breaks on EVERY `mov %eax,%r14d` store in FUN_00374240 (85 sites, generated
+# from the disassembly) plus the alternate epilogue at 0x27498f, printing the
+# stored status; the first non-zero store is the failing call/check.
+#
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bpz3.py
+import gdb
+import time
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    print("NO_LIBCUDA")
+    raise SystemExit
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+
+SITES = [
+    (0x2746f7, "STORE_2746f7"),
+    (0x274724, "STORE_274724"),
+    (0x2747b4, "STORE_2747b4"),
+    (0x274ab0, "STORE_274ab0"),
+    (0x274be5, "STORE_274be5"),
+    (0x274d59, "STORE_274d59"),
+    (0x274f3f, "STORE_274f3f"),
+    (0x274f87, "STORE_274f87"),
+    (0x27503a, "STORE_27503a"),
+    (0x27505c, "STORE_27505c"),
+    (0x275075, "STORE_275075"),
+    (0x275208, "STORE_275208"),
+    (0x27526e, "STORE_27526e"),
+    (0x275294, "STORE_275294"),
+    (0x2752d7, "STORE_2752d7"),
+    (0x2752fd, "STORE_2752fd"),
+    (0x27532c, "STORE_27532c"),
+    (0x275355, "STORE_275355"),
+    (0x27537d, "STORE_27537d"),
+    (0x27539d, "STORE_27539d"),
+    (0x2753da, "STORE_2753da"),
+    (0x275444, "STORE_275444"),
+    (0x275474, "STORE_275474"),
+    (0x27549d, "STORE_27549d"),
+    (0x2754c6, "STORE_2754c6"),
+    (0x2754f6, "STORE_2754f6"),
+    (0x275541, "STORE_275541"),
+    (0x27558c, "STORE_27558c"),
+    (0x2755bf, "STORE_2755bf"),
+    (0x2756cd, "STORE_2756cd"),
+    (0x27572b, "STORE_27572b"),
+    (0x275774, "STORE_275774"),
+    (0x275799, "STORE_275799"),
+    (0x2757c3, "STORE_2757c3"),
+    (0x2757e2, "STORE_2757e2"),
+    (0x275804, "STORE_275804"),
+    (0x27581f, "STORE_27581f"),
+    (0x275834, "STORE_275834"),
+    (0x275849, "STORE_275849"),
+    (0x27585e, "STORE_27585e"),
+    (0x275873, "STORE_275873"),
+    (0x2758aa, "STORE_2758aa"),
+    (0x2758eb, "STORE_2758eb"),
+    (0x275900, "STORE_275900"),
+    (0x275a53, "STORE_275a53"),
+    (0x275b3f, "STORE_275b3f"),
+    (0x275bc7, "STORE_275bc7"),
+    (0x275bdf, "STORE_275bdf"),
+    (0x275bf2, "STORE_275bf2"),
+    (0x275c7b, "STORE_275c7b"),
+    (0x275ccc, "STORE_275ccc"),
+    (0x275cec, "STORE_275cec"),
+    (0x275d11, "STORE_275d11"),
+    (0x275d2b, "STORE_275d2b"),
+    (0x275e6c, "STORE_275e6c"),
+    (0x275f9b, "STORE_275f9b"),
+    (0x275fb2, "STORE_275fb2"),
+    (0x2760c8, "STORE_2760c8"),
+    (0x2760fe, "STORE_2760fe"),
+    (0x276149, "STORE_276149"),
+    (0x276165, "STORE_276165"),
+    (0x276181, "STORE_276181"),
+    (0x27619d, "STORE_27619d"),
+    (0x27622c, "STORE_27622c"),
+    (0x27627d, "STORE_27627d"),
+    (0x27641e, "STORE_27641e"),
+    (0x27652a, "STORE_27652a"),
+    (0x27654c, "STORE_27654c"),
+    (0x276579, "STORE_276579"),
+    (0x2765ca, "STORE_2765ca"),
+    (0x276618, "STORE_276618"),
+    (0x2768b0, "STORE_2768b0"),
+    (0x2768cf, "STORE_2768cf"),
+    (0x27692c, "STORE_27692c"),
+    (0x27695f, "STORE_27695f"),
+    (0x2769f5, "STORE_2769f5"),
+    (0x276a40, "STORE_276a40"),
+    (0x276b3d, "STORE_276b3d"),
+    (0x276b88, "STORE_276b88"),
+    (0x276cf4, "STORE_276cf4"),
+    (0x276d69, "STORE_276d69"),
+    (0x27716b, "STORE_27716b"),
+    (0x2772a3, "STORE_2772a3"),
+    (0x2772d1, "STORE_2772d1"),
+    (0x27734d, "STORE_27734d"),
+    (0x27640b, "MALLOC999"),
+    (0x27428a, "CTX_RET"),
+    (0x27498f, "ALT_EPI"),
+]
+
+for off, name in SITES:
+    gdb.execute("break *0x%x" % (base + off))
+
+t0 = time.time()
+for hit in range(400):
+    try:
+        gdb.execute("continue")
+    except gdb.error as e:
+        print("CONTINUE_END %s" % e)
+        break
+    try:
+        rip = int(gdb.parse_and_eval("$rip")) & 0xffffffffffffffff
+        rax = int(gdb.parse_and_eval("$rax")) & 0xffffffffffffffff
+        r14 = int(gdb.parse_and_eval("$r14")) & 0xffffffffffffffff
+    except Exception as e:
+        print("REGS_GONE %s" % e)
+        break
+    off = rip - base
+    name = dict(SITES).get(off, "?")
+    st = rax & 0xffffffff
+    print("HIT %d t=%.3f off=0x%x %s rax=0x%x r14d=0x%x" %
+          (hit, time.time() - t0, off, name, rax, r14 & 0xffffffff))
+    if st != 0 or off in (0x27428a, 0x27498f):
+        print("  >>> NONZERO/TERMINAL %s status=0x%x" % (name, st))
+        try:
+            print(gdb.execute("bt 12", to_string=True))
+        except Exception as e:
+            print("bt_fail %s" % e)
+
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 2bfb2a9..8600318 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -30,6 +30,7 @@ push_b64 "$HERE/bp999.py"       /tmp/bp999.py
 push_b64 "$HERE/bp999v2.py"     /tmp/bp999v2.py
 push_b64 "$HERE/bpz.py"         /tmp/bpz.py
 push_b64 "$HERE/bpz2.py"        /tmp/bpz2.py
+push_b64 "$HERE/bpz3.py"        /tmp/bpz3.py
 push_b64 "$HERE/bpwait.py"      /tmp/bpwait.py
 push_b64 "$HERE/ctx999.py"      /tmp/ctx999.py
 push_b64 "$HERE/scanerr.py"     /tmp/scanerr.py