guest-diag: bpz6/invd - read the live arch/impl and test the WAR-ucode lookup

Run #5 named the failing hook: idx=3 FUN_005d7e70 ('membar_war_bug2022401')
returned 999; its only source is FUN_0035a4d0(0x13, dev+0xc50, dev+0xc54)
returning NULL or the ucode load failing. bpz6 reads the live device state
arch/impl at the failure and evaluates the lookup in-process, scanning
(arch,rev) pairs to show which WAR-ucode descriptors exist; invd.py is the
standalone same-probe for a parked cdt.

3 files changed, 197 insertions, 0 deletions

diff --git a/tools/guest-diag/bpz6.py b/tools/guest-diag/bpz6.py
new file mode 100644
index 0000000..7742da2
--- /dev/null
+++ b/tools/guest-diag/bpz6.py
@@ -0,0 +1,103 @@
+# gdb script: at the membar-WAR hook failure, read the device arch/impl and
+# test the FUN_0035a4d0(0x13, arch, rev) descriptor lookup in-process.
+#
+# Run #5: hook idx=3 (FUN_005d7e70) returned 0x3e7; its only 999 source is
+# `FUN_0035a4d0(0x13, *(u32*)(dev+0xc50), *(u32*)(dev+0xc54)) == 0` or the
+# ucode load failing. Break on the hook return; when the status is non-zero,
+# eval the lookup with the live values and scan nearby (arch, rev) pairs.
+#
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bpz6.py
+import gdb
+import time
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    raise SystemExit
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+for off in (0x4d20fe, 0x4d2101):
+    gdb.execute("break *0x%x" % (base + off))
+
+f = base + 0x35a4d0
+
+
+def r64(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned long *)0x%x" % a)) & 0xffffffffffffffff
+    except Exception:
+        return None
+
+
+def r32(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned int *)0x%x" % a)) & 0xffffffff
+    except Exception:
+        return None
+
+
+def lookup(chip, rev):
+    try:
+        return int(gdb.parse_and_eval(
+            "((void*(*)(long,int,int))0x%x)(0x13,%d,%d)" % (f, chip, rev))) & 0xffffffffffffffff
+    except Exception as e:
+        print("  call_fail chip=0x%x rev=0x%x: %s" % (chip, rev, e))
+        return None
+
+
+t0 = time.time()
+for hit in range(40):
+    try:
+        gdb.execute("continue")
+    except gdb.error as e:
+        print("CONTINUE_END %s" % e)
+        break
+    rip = int(gdb.parse_and_eval("$rip")) & 0xffffffffffffffff
+    off = rip - base
+    if off == 0x4d20fe:
+        try:
+            rax = int(gdb.parse_and_eval("$rax")) & 0xffffffffffffffff
+            print("CALL t=%.3f target_off=0x%x" %
+                  (time.time() - t0, (int(gdb.parse_and_eval(
+                      "*(unsigned long *)0x%x" % (rax + 8))) - base) & 0xffffffffffffffff))
+        except Exception as e:
+            print("call_dump_fail %s" % e)
+        continue
+    rax = int(gdb.parse_and_eval("$rax")) & 0xffffffffffffffff
+    print("RET t=%.3f eax=0x%x" % (time.time() - t0, rax & 0xffffffff))
+    if (rax & 0xffffffff) == 0:
+        continue
+    try:
+        ctx = int(gdb.parse_and_eval("$rbx")) & 0xffffffffffffffff
+        st = r64(ctx + 0x40)
+        chip = r32(st + 0xc50)
+        rev = r32(st + 0xc54)
+        print("ctx=0x%x state=0x%x state+0xc50=0x%x (arch=0x%x) +0xc54=0x%x" %
+              (ctx, st or 0, r64(st + 0xc50) or 0, chip or 0, rev or 0))
+        v = lookup(chip, rev)
+        print("lookup(0x13,0x%x,0x%x) = 0x%x" % (chip, rev, v or 0))
+        if v:
+            print("  desc+0x10=0x%x desc+0x18=0x%x" %
+                  (r64(v + 0x10) or 0, r64(v + 0x18) or 0))
+        print("scan:")
+        for c in (0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0x160, 0x170):
+            for rv in range(0, 4):
+                vv = lookup(c, rv)
+                if vv:
+                    print("  chip=0x%x rev=0x%x -> 0x%x size=0x%x" %
+                          (c, rv, vv, r64(vv + 0x18) or 0))
+    except Exception as e:
+        print("live_probe_fail %s" % e)
+    break
+
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
diff --git a/tools/guest-diag/invd.py b/tools/guest-diag/invd.py
new file mode 100644
index 0000000..1fc8491
--- /dev/null
+++ b/tools/guest-diag/invd.py
@@ -0,0 +1,92 @@
+# gdb script: read the device arch/impl and test the membar-WAR descriptor
+# lookup (FUN_0035a4d0) live on the parked cdt.
+#
+# Run #5: hook idx=3 (FUN_005d7e70, "membar_war_bug2022401") returned 999,
+# whose only source is `FUN_0035a4d0(0x13, *(u32*)(dev+0xc50),
+# *(u32*)(dev+0xc54)) == NULL` or the ucode load failing. This script prints
+# the device state's +0xc50/+0xc54 the way the failing code reads them, calls
+# the lookup with those values, and scans nearby (arch, rev) pairs to show
+# which exist in libcuda's WAR-microcode table.
+#
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/invd.py
+import gdb
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    raise SystemExit
+
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+
+
+def r64(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned long *)0x%x" % a)) & 0xffffffffffffffff
+    except Exception as e:
+        return None
+
+
+def r32(a):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned int *)0x%x" % a)) & 0xffffffff
+    except Exception as e:
+        return None
+
+
+dev = r64(base + 0x44120c8)
+print("dev array [0]=0x%x" % (dev or 0))
+stack = []
+if dev:
+    for off in (0x0, 0x8, 0x10, 0x40):
+        v = r64(dev + off)
+        print("  dev+0x%x = 0x%x" % (off, v or 0))
+    cand = r64(dev + 8)
+    if cand:
+        st = r64(cand + 0x40)
+        print("  [dev+8]+0x40 = 0x%x" % (st or 0))
+        if st:
+            q = r64(st + 0xc50)
+            print("  state+0xc50 = 0x%x (arch=%s impl=%s)" %
+                  (q or 0, (q or 0) & 0xffffffff, (q or 0) >> 32))
+            print("  state+0xc54 = 0x%x" % (r32(st + 0xc54) or 0))
+
+f = base + 0x35a4d0
+
+
+def lookup(chip, rev):
+    try:
+        v = int(gdb.parse_and_eval("((void*(*)(long,int,int))0x%x)(0x13,%d,%d)" %
+                                   (f, chip, rev))) & 0xffffffffffffffff
+        return v
+    except Exception as e:
+        print("  call_fail chip=0x%x rev=0x%x: %s" % (chip, rev, e))
+        return None
+
+
+if dev and r64(dev + 8) and r64(r64(dev + 8) + 0x40):
+    st = r64(r64(dev + 8) + 0x40)
+    chip = r32(st + 0xc50)
+    rev = r32(st + 0xc54)
+    v = lookup(chip, rev)
+    print("lookup(0x13,0x%x,0x%x) = 0x%x" % (chip, rev, v or 0))
+    if v:
+        print("  desc+0x10=0x%x desc+0x18=0x%x" %
+              (r64(v + 0x10) or 0, r64(v + 0x18) or 0))
+print("scan (chip,rev) -> descriptor:")
+for chip in (0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0x160, 0x170):
+    for rev in range(0, 6):
+        v = lookup(chip, rev)
+        if v:
+            print("  chip=0x%x rev=0x%x -> 0x%x size=0x%x" %
+                  (chip, rev, v, r64(v + 0x18) or 0))
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index e4c1222..e10c8ff 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -33,6 +33,8 @@ push_b64 "$HERE/bpz2.py"        /tmp/bpz2.py
 push_b64 "$HERE/bpz3.py"        /tmp/bpz3.py
 push_b64 "$HERE/bpz4.py"        /tmp/bpz4.py
 push_b64 "$HERE/bpz5.py"        /tmp/bpz5.py
+push_b64 "$HERE/bpz6.py"        /tmp/bpz6.py
+push_b64 "$HERE/invd.py"        /tmp/invd.py
 push_b64 "$HERE/bpwait.py"      /tmp/bpwait.py
 push_b64 "$HERE/ctx999.py"      /tmp/ctx999.py
 push_b64 "$HERE/scanerr.py"     /tmp/scanerr.py