guest-diag: accept both GPFIFO ring windows and validate channel_info for #put (ISSUE-0002 diag)

CE channels register gpfifo=0x121010000+0x30000*rank, outside the
GR-only window [0x200200000,0x200400000) the #put resolver searched, so
no #put line ever fired for the CE channels the second session stalls
on. Resolve channel_info from both windows with layout validation, keep
re-resolving the channel after the pool mem has resolved, reset the
#put map on client switch, and add bounded resolver diagnostics to the
guest log.

1 files changed, 153 insertions, 66 deletions

diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index 844d512..14cd526 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -178,6 +178,37 @@ def is_gpu_va(v):
     return v and 0x120000000 <= v < 0x200000000
 
 
+def _is_gpu_va_wide(v):
+    """GPU VAs above 0x200000000 are the GR share window; accept them too."""
+    return v and (is_gpu_va(v) or 0x200000000 <= v < 0x208000000)
+
+
+def read_u32(kc, va):
+    d = kc.read(va, 4) if is_kern_ptr(va) else None
+    return struct.unpack("<I", d)[0] if d else 0xFFFFFFFF
+
+
+DBG_BUDGET = int(os.environ.get("STREAM_DBG_BUDGET", "600"))
+_dbg_count = [0]
+
+
+def dbg(msg):
+    """Bounded resolver diagnostics to the guest log.
+
+    The single-QGA guest agent is often wedged exactly when the second
+    client stalls, so the scanner's own log is the reliable diagnostic
+    channel; it is captured from the guest after the run.
+    """
+    if _dbg_count[0] >= DBG_BUDGET:
+        return
+    _dbg_count[0] += 1
+    try:
+        with open('/tmp/guest_pin_stream.log', 'a') as f:
+            f.write('%s DBG %s\n' % (time.strftime('%T'), msg))
+    except OSError:
+        pass
+
+
 def _direct_map_pa(kc, va):
     """Direct-map aliases translate linearly: pa = va - page_offset_base."""
     if kc.page_offset_base <= va < kc.page_offset_base + (1 << 40):
@@ -288,44 +319,86 @@ def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out):
     return emitted
 
 
-def resolve_channel_from_sem(kc, sem, out):
-    """Tracking sem (embedded in uvm_channel_t) -> ring/GPPUT pins.
+def _is_ring_va(v):
+    """True for a page-aligned GPFIFO ring GPU VA in any channel window."""
+    if not v or (v & 0xFFF):
+        return False
+    for lo, hi in RING_WINDOWS:
+        if lo <= v < hi:
+            return True
+    return False
+
+
+def _chan_info_at(kc, pos):
+    """Validate the UvmGpuChannelInfo implied by a gpFifoGpuVa at pos.
 
-    uvch events do not fire on this driver build, but the channel's tracking
-    semaphore is a uvm_channel_t member updated on every push. Locate
-    channel_info by scanning around the sem for its gpFifoGpuVa value (a
-    page-aligned GPU VA in the GR ring window), then read the ring/GPPUT
-    CPU pointers (UvmGpuChannelInfo offsets: gpPut +0x08, gpFifoEntries
-    +0x10, numGpFifoEntries +0x18, gpFifoGpuVa +0x58, gpPutGpuVa +0x60).
+    pos is the address of the +0x58 gpFifoGpuVa field; the surrounding
+    fields must read as a real channel_info (sane entry count, kernel CPU
+    pointers) before the anchor is trusted. This lets the candidate search
+    accept both ring windows without latching onto an unrelated page-aligned
+    GPU VA that happens to sit in the same allocation (the pool page VA,
+    0x121000000, is exactly such a value).
     """
-    if not is_kern_ptr(sem):
-        return 0
-    lo = sem - 0x2000
-    data = kc.read(lo, 0x4000)
+    entries_cpu = kc.read64(pos - 0x48)
+    entries = kc.read64(pos - 0x40)
+    if entries:
+        entries &= 0xFFFFFFFF
+    gp_put = kc.read64(pos - 0x50)
+    if not entries or entries > 0x100000:
+        return None
+    if not is_kern_ptr(entries_cpu) or not is_kern_ptr(gp_put):
+        return None
+    return (entries_cpu, entries, gp_put, kc.read64(pos + 0x08))
+
+
+def _find_chan_anchor(kc, lo, span):
+    """First validated chan anchor in [lo, lo+span).
+
+    Returns (pos, ring_va, entries_cpu, entries, gp_put, gp_put_va) or None.
+    """
+    data = kc.read(lo, span)
     if not data:
-        return 0
+        return None
     for off in range(0, len(data) - 8, 8):
         v = struct.unpack_from("<Q", data, off)[0]
-        if not (GR_RING_LO <= v < GR_RING_HI) or (v & 0xFFF):
+        if not _is_ring_va(v):
             continue
         pos = lo + off
-        entries_cpu = kc.read64(pos - 0x48)
-        entries = kc.read64(pos - 0x40)
-        if entries:
-            entries &= 0xFFFFFFFF
-        emitted = _emit_ring_pins(kc, v, entries_cpu, entries, out)
-        gp_put = kc.read64(pos - 0x50)
-        gp_put_va = kc.read64(pos + 0x08)
-        chan_put_note(v, gp_put)
-        if is_kern_ptr(gp_put) and gp_put_va and \
-                GR_RING_LO <= gp_put_va < 0x201000000:
-            pa = kc.va_to_pa(gp_put)
-            if pa is not None:
-                out.append("0x%x,0x%x,1\n" % (gp_put_va & ~0xFFF,
-                                               pa & ~0xFFF))
-                emitted += 1
-        return emitted
-    return 0
+        got = _chan_info_at(kc, pos)
+        if got:
+            return (pos, v) + got
+    return None
+
+
+def resolve_channel_from_sem(kc, sem, out):
+    """Tracking sem (embedded in uvm_channel_t) -> ring/GPPUT pins.
+
+    The channel's tracking semaphore is a uvm_channel_t member updated on
+    every push, and channel_info is a later union member of the same
+    allocation, so scan forward from the sem (plus a little back) and accept
+    a validated gpFifoGpuVa from any ring window. The original GR-only
+    window [0x200200000,0x200400000) rejected the CE channels' rings
+    (0x121010000 + 0x30000*rank), which is why no #put note ever fired for
+    the CE channels the second session stalls on.
+    """
+    if not is_kern_ptr(sem):
+        return 0
+    r = _find_chan_anchor(kc, sem - 0x2000, 0xA000)
+    if r is None:
+        dbg("chan-nofind sem=0x%x" % sem)
+        return 0
+    pos, v, entries_cpu, entries, gp_put, gp_put_va = r
+    dbg("chan-find sem=0x%x ring=0x%x put=%u" %
+        (sem, v, read_u32(kc, gp_put)))
+    emitted = _emit_ring_pins(kc, v, entries_cpu, entries, out)
+    chan_put_note(v, gp_put)
+    if is_kern_ptr(gp_put) and gp_put_va and _is_gpu_va_wide(gp_put_va):
+        pa = kc.va_to_pa(gp_put)
+        if pa is not None:
+            out.append("0x%x,0x%x,1\n" % (gp_put_va & ~0xFFF,
+                                           pa & ~0xFFF))
+            emitted += 1
+    return emitted
 
 
 def resolve_sem_pins(kc, sem, out):
@@ -335,12 +408,7 @@ def resolve_sem_pins(kc, sem, out):
     mem = kc.read64(vals[0])
     if not mem or not is_kern_ptr(mem):
         return 0
-    n = resolve_mem_pins(kc, mem, out, default_pages=16)
-    try:
-        n += resolve_channel_from_sem(kc, sem, out)
-    except Exception:
-        pass
-    return n
+    return resolve_mem_pins(kc, mem, out, default_pages=16)
 
 
 def resolve_pb_pins(kc, pb, out):
@@ -355,9 +423,18 @@ def resolve_pb_pins(kc, pb, out):
 #   +0x58 gpFifoGpuVa, +0x60 gpPutGpuVa, +0x68 gpGetGpuVa.
 # The offset of channel_info inside uvm_channel_t is config-dependent, so the
 # struct is located by searching the channel for its gpFifoGpuVa value (a GPU
-# VA in the GR ring window, page aligned).
+# VA in a ring window, page aligned).
 GR_RING_LO = 0x200200000
 GR_RING_HI = 0x200400000
+# ISSUE-0002: the CE channels register gpfifo=0x121010000 + 0x30000*rank in
+# the service's `channel ... registered gpfifo=...` lines, i.e. the CE
+# GPFIFO rings live just above the semaphore pool, NOT in the GR window.
+# The GR-only window rejected them, so the #put note never fired for the
+# channels the second session stalls on. Both windows are accepted; every
+# candidate is validated against the channel_info layout first.
+CE_GPFIFO_LO = 0x121000000
+CE_GPFIFO_HI = 0x121400000
+RING_WINDOWS = ((GR_RING_LO, GR_RING_HI), (CE_GPFIFO_LO, CE_GPFIFO_HI))
 CHAN_INFO_GPFIFO_VA = 0x58
 
 # ISSUE-0002: channel CPU-side GPUT pointers, keyed by ring GPU VA. The main
@@ -384,26 +461,15 @@ def resolve_channel_pins(kc, ch, out):
     """
     if not is_kern_ptr(ch):
         return 0
-    base = None
-    gpu_va = None
-    data = kc.read(ch, 0x800)
-    if not data:
-        return 0
-    for off in range(0, len(data) - 8, 8):
-        v = struct.unpack_from("<Q", data, off)[0]
-        if GR_RING_LO <= v < GR_RING_HI and (v & 0xFFF) == 0:
-            base = ch + off - CHAN_INFO_GPFIFO_VA
-            gpu_va = v
-            break
-    if base is None:
-        return 0
-    entries = kc.read64(base + 0x18)
-    if entries:
-        entries &= 0xFFFFFFFF
-    if not entries or entries > 0x100000:
+    r = _find_chan_anchor(kc, ch, 0x4000)
+    if r is None:
+        dbg("ch-nofind ch=0x%x" % ch)
         return 0
+    pos, gpu_va, entries_cpu, entries, gp_put, gp_put_va = r
+    dbg("ch-find ch=0x%x ring=0x%x put=%u entries=%u" %
+        (ch, gpu_va, read_u32(kc, gp_put), entries))
+    base = pos - CHAN_INFO_GPFIFO_VA
     emitted = 0
-    entries_cpu = kc.read64(base + 0x10)
     if is_kern_ptr(entries_cpu):
         nbytes = entries * 8
         npages = min((nbytes + 0xFFF) // 0x1000 + 2, 8)
@@ -427,19 +493,15 @@ def resolve_channel_pins(kc, ch, out):
         for k, pa, n in runs:
             out.append("0x%x,0x%x,%d\n" % (gpu_va + k * 0x1000, pa, n))
             emitted += 1
-    gp_put = kc.read64(base + 0x08)
-    gp_put_va = kc.read64(base + 0x60)
     chan_put_note(gpu_va, gp_put)
-    if is_kern_ptr(gp_put) and gp_put_va and \
-            GR_RING_LO <= gp_put_va < 0x201000000:
+    if is_kern_ptr(gp_put) and gp_put_va and _is_gpu_va_wide(gp_put_va):
         pa = kc.va_to_pa(gp_put)
         if pa is not None:
             out.append("0x%x,0x%x,1\n" % (gp_put_va & ~0xFFF, pa & ~0xFFF))
             emitted += 1
     gp_get = kc.read64(base + 0x00)
     gp_get_va = kc.read64(base + 0x68)
-    if is_kern_ptr(gp_get) and gp_get_va and \
-            GR_RING_LO <= gp_get_va < 0x201000000:
+    if is_kern_ptr(gp_get) and gp_get_va and _is_gpu_va_wide(gp_get_va):
         pa = kc.va_to_pa(gp_get)
         if pa is not None:
             out.append("0x%x,0x%x,1\n" % (gp_get_va & ~0xFFF, pa & ~0xFFF))
@@ -718,6 +780,7 @@ def main():
     seen_pbs = set()
     seen_chs = set()
     resolved_sems = set()
+    resolved_sem_chan = set()
     resolved_pbs = set()
     resolved_chs = set()
     retry_at = {}
@@ -751,9 +814,15 @@ def main():
             seen_pbs = set()
             seen_chs = set()
             resolved_sems = set()
+            resolved_sem_chan = set()
             resolved_pbs = set()
             resolved_chs = set()
             retry_at = {}
+            # Client switch: the old client's channel/put pointers are freed
+            # memory. Clearing the #put map stops stale reads from being
+            # emitted as if they were the new client's counters.
+            CHAN_PUTS.clear()
+            PUT_LAST.clear()
             clear_trace()
             if sock is not None:
                 try:
@@ -804,19 +873,37 @@ def main():
             now_r = time.time()
             added = 0
             for sem in list(seen_sems):
-                if sem in resolved_sems or now_r < retry_at.get(sem, 0):
+                if sem not in resolved_sems and now_r >= retry_at.get(sem, 0):
+                    got = []
+                    try:
+                        resolve_sem_pins(kc, sem, got)
+                    except Exception:
+                        got = []
+                    if got:
+                        lines.extend(got)
+                        resolved_sems.add(sem)
+                        added += len(got)
+                    else:
+                        retry_at[sem] = now_r + 1.0
+                # ISSUE-0002: the pool pin (mem) and the channel ring are
+                # resolved independently. The sem is marked resolved as soon
+                # as the pool page pins, but the channel_info anchor may only
+                # appear (or only become findable) later, and the #put probe
+                # must not be skipped for an already-resolved sem.
+                if sem in resolved_sem_chan or \
+                        now_r < retry_at.get(("cy", sem), 0):
                     continue
                 got = []
                 try:
-                    resolve_sem_pins(kc, sem, got)
+                    resolve_channel_from_sem(kc, sem, got)
                 except Exception:
                     got = []
                 if got:
                     lines.extend(got)
-                    resolved_sems.add(sem)
+                    resolved_sem_chan.add(sem)
                     added += len(got)
                 else:
-                    retry_at[sem] = now_r + 1.0
+                    retry_at[("cy", sem)] = now_r + 2.0
             for pb in list(seen_pbs):
                 if pb in resolved_pbs or now_r < retry_at.get(pb, 0):
                     continue