guest-diag: print tracking-sem state (index, queued, payload) on change

The final cuCtxCreate stall is a UVM tracking semaphore whose completed
value stays 0; the pool payload is the ground truth. semfix now prints
idx/+0x10/+0x28/payload for every tracking sem it sees, whenever a field
changes, so the last stall before a failure is visible in its log.

1 files changed, 40 insertions, 0 deletions

diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py
index 347da5e..1fc6b01 100644
--- a/tools/guest-diag/semfix_watch.py
+++ b/tools/guest-diag/semfix_watch.py
@@ -343,6 +343,33 @@ def pin_bulk(va, pa, nbytes):
     return False
 
 
+def sem_state(q, sem):
+    """(index, queued, payload) for a tracking-semaphore kptr, or None.
+
+    payload is the u32 the client's update_completed_value reads: the pool
+    page at the semaphore's index, through the CPU mapping the struct names.
+    Printed when it changes so the last stall before a failure is visible.
+    """
+    vals = q.read64(sem, 11)
+    if len(vals) < 2 or not is_kern(vals[0]):
+        return None
+    page = vals[0]
+    pv = q.read64(page)
+    if not pv or not is_kern(pv[0]):
+        return None
+    mem = pv[0]
+    vas = q.read64(mem + 0x10, 16)
+    cpu = next((v for v in vas if is_kern(v)), None)
+    idx = vals[1] & 0xffff
+    w10 = vals[2] if len(vals) > 2 else 0
+    queued = vals[5] if len(vals) > 5 else 0
+    payload = None
+    if cpu:
+        w = q.read64(cpu + idx * 4)
+        payload = w[0] if w else None
+    return (idx, w10, queued, payload)
+
+
 def resolve_sem_pool(q, sem):
     page = q.read64(sem)
     if not page or not page[0]:
@@ -408,6 +435,7 @@ def main():
     seen_pb = set()
     seen_ch = set()
     pinned_ranges = set()
+    sem_last = {}
 
     def pin_range_once(gpu_vas, cpu, gpa, size=0):
         key = tuple(gpu_vas), cpu, size
@@ -419,6 +447,18 @@ def main():
     print("semfix: watching (interval %.1fs)" % interval, flush=True)
     while True:
         sems, pbs, chans = guest_trace()
+        for sem in sems:
+            try:
+                st = sem_state(q, sem)
+            except Exception:
+                st = None
+            if st != sem_last.get(sem):
+                sem_last[sem] = st
+                if st:
+                    print("semfix: sem=0x%x idx=%d +10=0x%x +28=0x%x payload=%s"
+                          % (sem, st[0], st[1], st[2],
+                             "0x%x" % st[3] if st[3] is not None else "?"),
+                          flush=True)
         for ch in chans - seen_ch:
             d = {}
             try: