guest-diag: deterministic channel->pool->manager->pushbuffer chain + chunk-read diagnostics

semfix_watch.py now resolves the CE pushbuffer from guest truth instead of
scanning a wide raw window: it walks uvm_channel_t +0=pool, pool+0=manager,
manager+8=pushbuffer, pushbuffer+8=rm_mem, rm_mem+0x10=vas[] and prefers the
rm_mem whose GPU VA is in the 0x120000000 CE pushbuffer range. Three entry
points: uvch probe on __uvm_push_begin_acquire_on_channel_with_info, uvbp
probe on uvm_pushbuffer_begin_push, and the uvmt tracking-sem kptr (channel
scan back from sem). Misses now print structured hop counters so a failed
chunk read is distinguishable from a failed chain. guest_kprobes.sh
(re)installs uvmt/uvch/uvbp/uvpbf after every guest boot (plain, module:
and bare spellings).

1 files changed, 244 insertions, 98 deletions

diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py
index 7ae154f..7730e86 100644
--- a/tools/guest-diag/semfix_watch.py
+++ b/tools/guest-diag/semfix_watch.py
@@ -10,16 +10,26 @@
 #     `semaphore.page->memory->vas[]` with the pool page's CPU and GPU VAs;
 #     QEMU turns the CPU VA into a GPA (`gva2gpa`) which we pin.
 #  2. CE pushbuffer: the pushes live in a UVM rm_mem with GPU VA
-#     `0x120000000` whose CPU VA is vmalloc-mapped guest RAM. Its pages map
-#     linearly *descending* in GPA (page i -> base_gpa - i*0x1000). Pinning
-#     the range lets the host pump read the pushes directly and execute the
-#     PTE memsets/copies instead of only acknowledging their releases.
+#     `0x120000000` whose CPU VA is vmalloc-mapped guest RAM. Pinning the
+#     range lets the host pump read the pushes directly and execute the PTE
+#     memsets/copies instead of only acknowledging their releases.
 #
-# Sources: guest kprobe on uvm_gpu_tracking_semaphore_update_completed_value
-# (`sem=0x...`), QEMU QMP `x/..gx` reads and `gva2gpa`.
+# Pushbuffer chain (uvm 570.144 layouts):
+#     uvm_channel_t  +0x00 = pool
+#     uvm_channel_pool_t +0x00 = manager
+#     uvm_channel_manager_t +0x08 = pushbuffer
+#     uvm_pushbuffer_t +0x08 = memory (uvm_rm_mem_t)
+#     uvm_rm_mem_t +0x10 = vas[] (CPU @ +0x30, GPU @ +0x38 observed)
+# The channel is reached three ways: kprobe on
+# __uvm_push_begin_acquire_on_channel_with_info (`uvch` -> ch), kprobe on
+# uvm_pushbuffer_begin_push (`uvbp` -> pb), or the tracking-sem kptr `sem`
+# from `uvmt` (sem sits inside the channel struct at a fixed offset).
+#
+# Sources: guest kprobes, QEMU QMP `x/..gx` reads and `gva2gpa`.
 #
 # Usage: tools/guest-diag/semfix_watch.py [interval-seconds]
-# Runs until killed; prints one line per newly resolved/pinned page.
+# Runs until killed; prints one line per newly resolved/pinned page and a
+# structured diagnostic line whenever a chain resolution misses.
 
 import json
 import os
@@ -36,9 +46,11 @@ QMP_SOCK = os.environ.get("QMP_SOCK", "/tmp/qemu-qmp.sock")
 
 GPU_VA_LO = 0x100000000
 GPU_VA_HI = 0x2000000000
-PUSHBUF_VA_LO = 0x120000000
-PUSHBUF_VA_HI = 0x120400000
+PUSHBUF_VA_LO = int(os.environ.get("PUSHBUF_VA_LO", "0x120000000"), 16)
+PUSHBUF_VA_HI = int(os.environ.get("PUSHBUF_VA_HI", "0x120400000"), 16)
 PUSHBUF_MAX_PAGES = 1024
+CHAN_WINDOW_BACK = int(os.environ.get("CHAN_WINDOW_BACK", "0x4000"), 16)
+CHAN_MAX_CANDS = int(os.environ.get("CHAN_MAX_CANDS", "256"))
 
 
 class Qmp:
@@ -48,6 +60,7 @@ class Qmp:
         self.s.settimeout(15)
         self._rd()
         self.cmd("qmp_capabilities")
+        self._cache = {}
 
     def _rd(self):
         time.sleep(0.03)
@@ -67,11 +80,16 @@ class Qmp:
         return self.cmd("human-monitor-command", **{"command-line": line}).get("return", "")
 
     def read64(self, addr, n=1):
+        key = (addr, n)
+        if key in self._cache:
+            return self._cache[key]
         out = self.hmp("x/%dgx 0x%x" % (n, addr))
         m = re.search(r":(.*)$", out, re.S)
         if not m:
             return []
-        return [int(x, 16) for x in re.findall(r"0x([0-9a-fA-F]+)", m.group(1))][:n]
+        vals = [int(x, 16) for x in re.findall(r"0x([0-9a-fA-F]+)", m.group(1))][:n]
+        self._cache[key] = vals
+        return vals
 
     def gva2gpa(self, va):
         out = self.hmp("gva2gpa 0x%x" % va)
@@ -87,45 +105,200 @@ def is_gpu(v):
     return v and GPU_VA_LO <= v < GPU_VA_HI
 
 
+def is_pushbuf_gpu(v):
+    return v and PUSHBUF_VA_LO <= v < PUSHBUF_VA_HI
+
+
 def is_vmalloc(v):
     # pushbuffer rm_mems are vmalloc-mapped (the sem pools are direct-map)
     return v and 0xFFFFC90000000000 <= v < 0xFFFFE90000000000
 
 
+def diag_add(d, k, n=1):
+    d[k] = d.get(k, 0) + n
+
+
+def diag_str(d):
+    return " ".join("%s=%s" % (k, v) for k, v in sorted(d.items()))
+
+
 def guest_trace():
-    """(sem kptrs, pushbuffer kptrs) from the trace ring, then clear it.
+    """(sem kptrs, pushbuffer kptrs, channel kptrs) from the trace, then clear.
 
-    Read *before* clearing: uvm_pushbuffer_create fires once per client and
-    would be wiped by a clear that races it.
+    Read *before* clearing: the create-time probes fire once per client and
+    would be wiped by a clear that races them.
     """
-    cmd = ("sudo sh -c 'grep -E \"uvmt|uvpb\" /sys/kernel/tracing/trace | tail -200; "
-           "echo > /sys/kernel/tracing/trace'")
+    cmd = ("sudo sh -c 'grep -E \"uvmt|uvch|uvbp|uvpbf\" /sys/kernel/tracing/trace "
+           "| tail -400; echo > /sys/kernel/tracing/trace'")
     try:
         out = subprocess.run([VMEXEC, "exec", cmd], capture_output=True,
                              text=True, timeout=90).stdout
     except Exception:
-        return set(), set()
+        return set(), set(), set()
     sems = {int(m, 16) for m in re.findall(r"sem=0x([0-9a-fA-F]+)", out)}
     pbs = {int(m, 16) for m in re.findall(r"pb=0x([0-9a-fA-F]+)", out)}
-    return sems, pbs
-
-
-def resolve_pb_ptr(q, pb):
-    """pushbuffer pointer -> rm_mem -> vas[] -> (gpu_vas, cpu, gpa)."""
-    mem = q.read64(pb + 8)
-    if not mem or not is_kern(mem[0]):
-        return [], None, None
-    vas = q.read64(mem[0] + 0x10, 16)
+    chans = {int(m, 16) for m in re.findall(r"ch=0x([0-9a-fA-F]+)", out)}
+    return sems, pbs, chans
+
+
+def vas_from_mem(q, mem, d):
+    """uvm_rm_mem_t pointer -> (gpu_vas, cpu, gpa) or None."""
+    if not is_kern(mem):
+        diag_add(d, "badmem")
+        return None
+    vas = q.read64(mem + 0x10, 16)
+    if not vas:
+        diag_add(d, "empty_vas")
+        return None
     gpu_vas = [v for v in vas if is_gpu(v)]
     cpu = next((v for v in vas if is_vmalloc(v)), None)
-    if not gpu_vas or not cpu:
-        return [], None, None
+    if not gpu_vas:
+        diag_add(d, "no_gpuva")
+        return None
+    if not cpu:
+        diag_add(d, "no_vmalloc")
+        return None
     gpa = q.gva2gpa(cpu)
     if not gpa:
-        return [], None, None
+        diag_add(d, "no_gpa")
+        return None
     return gpu_vas, cpu, gpa
 
 
+def pushbuffer_of_manager(q, mgr, d):
+    pb = q.read64(mgr + 8)
+    if not pb or not is_kern(pb[0]):
+        diag_add(d, "stop_manager")
+        return None
+    mem = q.read64(pb[0] + 8)
+    if not mem or not is_kern(mem[0]):
+        diag_add(d, "stop_pushbuffer")
+        return None
+    return vas_from_mem(q, mem[0], d)
+
+
+def pushbuffer_of_pool(q, pool, d):
+    mgr = q.read64(pool)
+    if not mgr or not is_kern(mgr[0]):
+        diag_add(d, "stop_pool")
+        return None
+    return pushbuffer_of_manager(q, mgr[0], d)
+
+
+def best_candidate(found, d):
+    """Prefer a chain whose rm_mem has a GPU VA in the CE pushbuffer range."""
+    if not found:
+        return None
+    for res in found:
+        if any(is_pushbuf_gpu(v) for v in res[0]):
+            return res
+    gpu_vas = found[0][0]
+    diag_add(d, "range_miss")
+    d["_near"] = "0x%x" % gpu_vas[0]
+    return found[0]
+
+
+def chains_from_window(q, words, d):
+    """Interpret kernel pointers in the scanned window as channel/pool/
+    manager/pushbuffer pointers and walk the chain."""
+    cands = list(dict.fromkeys(w for w in words if is_kern(w)))
+    d["cands"] = len(cands)
+    found = []
+    for w in cands[:CHAN_MAX_CANDS]:
+        res = pushbuffer_of_pool(q, w, d)
+        if res:
+            found.append(res)
+            continue
+        ch = q.read64(w)
+        if ch and is_kern(ch[0]):
+            res = pushbuffer_of_pool(q, ch[0], d)
+            if res:
+                found.append(res)
+                continue
+        res = pushbuffer_of_manager(q, w, d)
+        if res:
+            found.append(res)
+            continue
+        pb = q.read64(w + 8)
+        if pb and is_kern(pb[0]):
+            res = vas_from_mem(q, pb[0], d)
+            if res:
+                found.append(res)
+    return best_candidate(found, d)
+
+
+def resolve_from_channel(q, ch, d):
+    """uvm_channel_t * -> pool -> manager -> pushbuffer -> rm_mem."""
+    d["ch"] = 1
+    pool = q.read64(ch)
+    if not pool or not is_kern(pool[0]):
+        diag_add(d, "stop_channel")
+        return None
+    res = pushbuffer_of_pool(q, pool[0], d)
+    return best_candidate([res] if res else [], d)
+
+
+def resolve_from_pushbuffer(q, pb, d):
+    d["pb"] = 1
+    mem = q.read64(pb + 8)
+    if not mem or not is_kern(mem[0]):
+        diag_add(d, "stop_pushbuffer")
+        return None
+    res = vas_from_mem(q, mem[0], d)
+    return best_candidate([res] if res else [], d)
+
+
+def resolve_from_sem(q, sem, d):
+    """Scan the pages around the channel's tracking_sem for the pool pointer
+    (channel+0 = pool) and walk pool -> manager -> pushbuffer -> rm_mem."""
+    d["sem"] = 1
+    page = sem & ~0xFFF
+    words = []
+    nback = max(1, CHAN_WINDOW_BACK // 0x1000)
+    for i in range(nback, -2, -1):
+        chunk = q.read64(page - i * 0x1000, 512)
+        if not chunk:
+            diag_add(d, "empty_chunks")
+            continue
+        words += chunk
+        diag_add(d, "chunks")
+    d["words"] = len(words)
+    if not words:
+        return None
+    return chains_from_window(q, words, d)
+
+
+def resolve_pushbuffer_wide(q, sem, d):
+    """Legacy 256 KB window scan (kept as a last resort)."""
+    base = sem & ~0xFFF
+    words = []
+    for i in range(64):
+        chunk = q.read64(base - 0x40000 + i * 0x1000, 512)
+        if not chunk:
+            diag_add(d, "empty_wide_chunks")
+            continue
+        words += chunk
+    d["wide_words"] = len(words)
+    d["wide_empty"] = d.get("empty_wide_chunks", 0)
+    if not words:
+        return None
+    cands = list(dict.fromkeys(w for w in words
+                               if is_kern(w) and (w >> 32) == (sem >> 32)))
+    d["wide_cands"] = len(cands)
+    found = []
+    for M in cands[:400]:
+        res = pushbuffer_of_manager(q, M, d)
+        if res:
+            found.append(res)
+            continue
+        pb = q.read64(M + 8)
+        if pb and is_kern(pb[0]):
+            res = vas_from_mem(q, pb[0], d)
+            if res:
+                found.append(res)
+    return best_candidate(found, d)
+
+
 def pin(va, pa):
     cmd = "printf '0x%x,0x%x' > %s/sema_map_add" % (va, pa, PARAM)
     for _ in range(40):
@@ -155,62 +328,6 @@ def resolve_sem_pool(q, sem):
     return [(gv, gpa) for gv in gpu_vas]
 
 
-def resolve_pushbuffer(q, sem):
-    """Find the pushbuffer rm_mem near the channel and return (gpu_vas, cpu, gpa)."""
-    base = sem & ~0xFFF
-    words = []
-    for i in range(64):
-        chunk = q.read64(base - 0x40000 + i * 0x1000, 512)
-        if not chunk:
-            continue
-        words += chunk
-    # The manager/pushbuffer live in the same slab region as the channel.
-    prefix = sem >> 32
-    cands = list(dict.fromkeys(w for w in words
-                               if is_kern(w) and (w >> 32) == prefix))
-    # Chain 1: value is the channel manager; +8 pushbuffer, +8 rm_mem.
-    deadline = time.time() + 45
-    for M in cands[:1200]:
-        if time.time() > deadline:
-            break
-        pb = q.read64(M + 8)
-        if not pb or not is_kern(pb[0]):
-            continue
-        mem = q.read64(pb[0] + 8)
-        if not mem or not is_kern(mem[0]):
-            continue
-        vas = q.read64(mem[0] + 0x10, 16)
-        gpu_vas = [v for v in vas if is_gpu(v)]
-        if not gpu_vas:
-            continue
-        cpu = next((v for v in vas if is_vmalloc(v)), None)
-        if not cpu:
-            continue
-        gpa = q.gva2gpa(cpu)
-        if gpa:
-            return gpu_vas, cpu, gpa
-    # Chain 2: value is the pushbuffer itself; +8 = rm_mem.
-    # Two QMP reads per candidate, so cap the scan (bounded in time).
-    deadline = time.time() + 30
-    for V in cands[:400]:
-        if time.time() > deadline:
-            break
-        mem = q.read64(V + 8)
-        if not mem or not is_kern(mem[0]):
-            continue
-        g5 = q.read64(mem[0] + 0x38)
-        if not g5 or not is_gpu(g5[0]):
-            continue
-        g4 = q.read64(mem[0] + 0x30)
-        cpu = g4[0] if g4 and is_vmalloc(g4[0]) else None
-        if not cpu:
-            continue
-        gpa = q.gva2gpa(cpu)
-        if gpa:
-            return [g5[0]], cpu, gpa
-    return [], None, None
-
-
 def pin_pushbuffer(q, gpu_vas, cpu, gpa):
     """Pin the whole pushbuffer with *per-page* translations (the layout is
     not linear across the allocation: page 32 jumped on a real client)."""
@@ -241,6 +358,7 @@ def main():
     q = Qmp(QMP_SOCK)
     seen = set()
     seen_pb = set()
+    seen_ch = set()
     pinned_ranges = set()
 
     def pin_range_once(gpu_vas, cpu, gpa):
@@ -252,19 +370,35 @@ def main():
 
     print("semfix: watching (interval %.1fs)" % interval, flush=True)
     while True:
-        sems, pbs = guest_trace()
+        sems, pbs, chans = guest_trace()
+        for ch in chans - seen_ch:
+            d = {}
+            try:
+                res = resolve_from_channel(q, ch, d)
+            except Exception as e:
+                res = None
+                d["exception"] = str(e)[:40]
+            if res:
+                seen_ch.add(ch)
+                pin_range_once(*res)
+                print("semfix: ch=0x%x -> %s" % (ch, diag_str(d)), flush=True)
+            else:
+                print("semfix: ch=0x%x pushbuf miss [%s]" % (ch, diag_str(d)), flush=True)
         for pb in pbs - seen_pb:
-            seen_pb.add(pb)
+            d = {}
             try:
-                gpu_vas, cpu, gpa = resolve_pb_ptr(q, pb)
+                res = resolve_from_pushbuffer(q, pb, d)
             except Exception as e:
-                print("semfix: pb=0x%x resolve failed: %s" % (pb, e), flush=True)
-                continue
-            if gpu_vas:
-                pin_range_once(gpu_vas, cpu, gpa)
+                res = None
+                d["exception"] = str(e)[:40]
+            if res:
+                seen_pb.add(pb)
+                pin_range_once(*res)
+                print("semfix: pb=0x%x -> %s" % (pb, diag_str(d)), flush=True)
             else:
-                print("semfix: pb=0x%x no pushbuffer rm_mem (retry)" % pb, flush=True)
+                print("semfix: pb=0x%x pushbuf miss [%s]" % (pb, diag_str(d)), flush=True)
         for sem in sems - seen:
+            d = {}
             try:
                 pairs = resolve_sem_pool(q, sem)
             except Exception as e:
@@ -278,15 +412,27 @@ def main():
                 ok = pin(gv, gpa)
                 print("semfix: sem=0x%x va=0x%x -> gpa=0x%x pin=%s"
                       % (sem, gv, gpa, "ok" if ok else "FAILED"), flush=True)
+            d = {}
             try:
-                gpu_vas, cpu, gpa = resolve_pushbuffer(q, sem)
+                res = resolve_from_sem(q, sem, d)
             except Exception as e:
-                print("semfix: pushbuf resolve failed: %s" % e, flush=True)
-                gpu_vas = []
-            if gpu_vas:
-                pin_range_once(gpu_vas, cpu, gpa)
+                res = None
+                d["exception"] = str(e)[:40]
+            if not res:
+                for k in ("chunks", "words", "cands", "stop_pool", "no_gpuva",
+                          "no_vmalloc", "empty_vas"):
+                    d.pop(k, None)
+                try:
+                    res = resolve_pushbuffer_wide(q, sem, d)
+                except Exception as e:
+                    res = None
+                    d["exception"] = str(e)[:40]
+            if res:
+                pin_range_once(*res)
+                print("semfix: sem=0x%x pushbuf -> %s" % (sem, diag_str(d)), flush=True)
             else:
-                print("semfix: pushbuf not found near sem=0x%x (retry)" % sem, flush=True)
+                print("semfix: sem=0x%x pushbuf not found [%s]"
+                      % (sem, diag_str(d)), flush=True)
         time.sleep(interval)