gsp_rm: retire event records on FREE; guest-diag: bp999 dumps the RM error record

A freed client's notifier events (hSrcResource 0) stayed valid in the event
table, so the fallback wake-up nudge kept posting POST_EVENT records for a
(hClient, hEvent) pair the guest RM no longer knows; every one asserts in
CliGetEventInfo (kernel_gsp.c:466) and is dropped (171 failures in the
15:24 window of the last run, zero after this fix).  FREE now retires the
matching event record (and a client's records when its root object is freed).

bp999.py now walks rdi -> ctx -> ctx[0x1291] -> +0x10 -> +0x10 and prints the
RM error notification record, and push_guest_tools.sh ships it to the guest.

1 files changed, 47 insertions, 1 deletions

diff --git a/tools/guest-diag/bp999.py b/tools/guest-diag/bp999.py
index db83d7d..ff7a23f 100644
--- a/tools/guest-diag/bp999.py
+++ b/tools/guest-diag/bp999.py
@@ -1,5 +1,13 @@
 # gdb script: break on the libcuda instructions that produce CUDA errors
-# 719 (0x2cf) and 999 (0x3e7), then print the stack that led there.
+# 719 (0x2cf) and 999 (0x3e7), then print the stack that led there and the
+# error record the scanner reads (FUN_0054fa00 walks ctx[0x1291] -> +0x10 ->
+# malloc'd reg struct -> +0x10 -> RM error notification buffer):
+#   ctx  = *(void**)param_1
+#   mgr  = *(void**)(ctx + 0x1291*8)
+#   reg  = *(void**)(mgr + 0x10)
+#   rec  = *(void**)(reg + 0x10)     <- RM-written error record
+# The record dwords are {magic0, magic1, errorClass, ...}: errorClass is what
+# the scanner maps to a CUDA error (or 999 when its force flag param_2 != 0).
 # Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bp999.py
 import gdb
 
@@ -30,6 +38,44 @@ try:
     print(gdb.execute("info registers rdi rsi rdx rax rbx rbp rip", to_string=True))
 except Exception as e:
     print("regs_fail %s" % e)
+
+
+def rd64(addr):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned long *)0x%x" % addr)) & 0xffffffffffffffff
+    except Exception:
+        return None
+
+
+def rd32(addr):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned int *)0x%x" % addr)) & 0xffffffff
+    except Exception:
+        return None
+
+
+try:
+    p1 = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
+    p2 = int(gdb.parse_and_eval("$rsi")) & 0xffffffff
+    print("param_1=0x%x param_2=0x%x" % (p1, p2))
+    ctx = rd64(p1)
+    print("ctx=0x%x" % (ctx or 0))
+    if ctx:
+        mgr = rd64(ctx + 0x1291 * 8)
+        print("mgr=0x%x" % (mgr or 0))
+        if mgr:
+            for mo in (0x0, 0x8, 0x10, 0x18, 0x38):
+                print("  mgr[+0x%x]=0x%x" % (mo, rd64(mgr + mo) or 0))
+            reg = rd64(mgr + 0x10)
+            print("reg=0x%x" % (reg or 0))
+            if reg:
+                rec = rd64(reg + 0x10)
+                print("rec=0x%x" % (rec or 0))
+                if rec:
+                    for i in range(0, 0x28, 4):
+                        print("  rec[+0x%02x]=0x%08x" % (i, rd32(rec + i) or 0))
+except Exception as e:
+    print("record_dump_fail %s" % e)
 try:
     gdb.execute("detach")
 except Exception: