gsp_rm: retire event records on FREE; guest-diag: bp999 dumps the RM error record

A freed client's notifier events (hSrcResource 0) stayed valid in the event
table, so the fallback wake-up nudge kept posting POST_EVENT records for a
(hClient, hEvent) pair the guest RM no longer knows; every one asserts in
CliGetEventInfo (kernel_gsp.c:466) and is dropped (171 failures in the
15:24 window of the last run, zero after this fix).  FREE now retires the
matching event record (and a client's records when its root object is freed).

bp999.py now walks rdi -> ctx -> ctx[0x1291] -> +0x10 -> +0x10 and prints the
RM error notification record, and push_guest_tools.sh ships it to the guest.

3 files changed, 74 insertions, 2 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 7300a8f..706bd81 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -2712,18 +2712,43 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
 		(const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
 	const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr);
 	u8 params[16]; /* NVOS00_PARAMETERS: hRoot, hObjectParent, hObjectOld, status */
-	u32 hObjOld = 0;
+	u32 hObjOld = 0, hRoot = 0;
+	u32 removed = 0, i;
 
 	if (hdr->length >= sizeof(*hdr) + sizeof(params))
 		memcpy(params, req, sizeof(params));
 	else
 		memset(params, 0, sizeof(params));
 
+	memcpy(&hRoot, params + 0, 4);
 	memcpy(&hObjOld, params + 8, 4);
 	memset(params + 12, 0, 4); /* status = NV_OK */
 
 	gsp_sema_service_channel_del(hObjOld);
 
+	/*
+	 * Retire event records that refer to the object being freed. Posting an
+	 * event for a freed (hClient, hEvent) pair makes the guest RM assert in
+	 * CliGetEventInfo (kernel_gsp.c:466) and drop the record; a freed
+	 * client's notifier events (hSrcResource 0) linger across runs and the
+	 * fallback wake-up nudge would hit them. Also retire a client's events
+	 * when its root object is freed (hObjOld == hRoot).
+	 */
+	for (i = 0; i < rm->events_n; i++) {
+		if (!rm->events[i].valid)
+			continue;
+		if (rm->events[i].h_event == hObjOld ||
+		    (hObjOld && hObjOld == hRoot &&
+		     rm->events[i].h_client == hRoot)) {
+			rm->events[i].valid = false;
+			removed++;
+		}
+	}
+	if (removed)
+		nv_vgpu_info(ctx->vgpu,
+			     "GSPRM: retired %u event record(s) on FREE hObjOld=0x%x hRoot=0x%x",
+			     removed, hObjOld, hRoot);
+
 	nv_vgpu_info(ctx->vgpu, "GSPRM: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u",
 		     hObjOld, sequence, rm->cmd_read, rm->stat_write);
 
diff --git a/tools/guest-diag/bp999.py b/tools/guest-diag/bp999.py
index db83d7d..ff7a23f 100644
--- a/tools/guest-diag/bp999.py
+++ b/tools/guest-diag/bp999.py
@@ -1,5 +1,13 @@
 # gdb script: break on the libcuda instructions that produce CUDA errors
-# 719 (0x2cf) and 999 (0x3e7), then print the stack that led there.
+# 719 (0x2cf) and 999 (0x3e7), then print the stack that led there and the
+# error record the scanner reads (FUN_0054fa00 walks ctx[0x1291] -> +0x10 ->
+# malloc'd reg struct -> +0x10 -> RM error notification buffer):
+#   ctx  = *(void**)param_1
+#   mgr  = *(void**)(ctx + 0x1291*8)
+#   reg  = *(void**)(mgr + 0x10)
+#   rec  = *(void**)(reg + 0x10)     <- RM-written error record
+# The record dwords are {magic0, magic1, errorClass, ...}: errorClass is what
+# the scanner maps to a CUDA error (or 999 when its force flag param_2 != 0).
 # Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bp999.py
 import gdb
 
@@ -30,6 +38,44 @@ try:
     print(gdb.execute("info registers rdi rsi rdx rax rbx rbp rip", to_string=True))
 except Exception as e:
     print("regs_fail %s" % e)
+
+
+def rd64(addr):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned long *)0x%x" % addr)) & 0xffffffffffffffff
+    except Exception:
+        return None
+
+
+def rd32(addr):
+    try:
+        return int(gdb.parse_and_eval("*(unsigned int *)0x%x" % addr)) & 0xffffffff
+    except Exception:
+        return None
+
+
+try:
+    p1 = int(gdb.parse_and_eval("$rdi")) & 0xffffffffffffffff
+    p2 = int(gdb.parse_and_eval("$rsi")) & 0xffffffff
+    print("param_1=0x%x param_2=0x%x" % (p1, p2))
+    ctx = rd64(p1)
+    print("ctx=0x%x" % (ctx or 0))
+    if ctx:
+        mgr = rd64(ctx + 0x1291 * 8)
+        print("mgr=0x%x" % (mgr or 0))
+        if mgr:
+            for mo in (0x0, 0x8, 0x10, 0x18, 0x38):
+                print("  mgr[+0x%x]=0x%x" % (mo, rd64(mgr + mo) or 0))
+            reg = rd64(mgr + 0x10)
+            print("reg=0x%x" % (reg or 0))
+            if reg:
+                rec = rd64(reg + 0x10)
+                print("rec=0x%x" % (rec or 0))
+                if rec:
+                    for i in range(0, 0x28, 4):
+                        print("  rec[+0x%02x]=0x%08x" % (i, rd32(rec + i) or 0))
+except Exception as e:
+    print("record_dump_fail %s" % e)
 try:
     gdb.execute("detach")
 except Exception:
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 516c64a..607cce4 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -26,6 +26,7 @@ push_b64 "$HERE/ioctl_trace.c"   /tmp/iot.c
 push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c
 push_b64 "$HERE/dumpctx.py"     /tmp/dumpctx.py
 push_b64 "$HERE/trkinfo.py"     /tmp/trkinfo.py
+push_b64 "$HERE/bp999.py"       /tmp/bp999.py
 
 echo "[*] compiling in guest..."
 "$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \