vgsp+guest-diag: kcore-truth pin resolver (pool/pushbuffer/channel ring+userd) + GR ring userd-derived candidates

guest_pin_stream.py now resolves translations entirely from guest truth via
/proc/kcore + a manual page-table walk (no QMP races): the CE semaphore pool
and pushbuffer memory from the UVM rm_mem structures, and each channel's
GPFIFO ring + GPPUT from the tracking semaphore's embedding uvm_channel_t
(channel_info located by its gpFifoGpuVa value). This removes the watcher
pipeline's flakiness on the cudaMalloc stall class (missing pool/pushbuffer
pins).

gsp_sema_service.c: svc_gr_ring_autopin tries the guest-registered USERD
memdesc base as a ring candidate (the fixed FB convention only matches when
the guest's VRAM allocator happens to reproduce the old layout), plus an
optional (gsp_sema_dbg) one-shot FB scan + raw-word diagnostics when every
candidate fails.

tools/cuda-ab/p2: self-checking arbitrary-kernel acceptance suite (10
programs: sequential launches, shapes, params, smem/GEMM, streams, malloc
churn, printf+error, FMA benchmark, mixed, managed probe) with host oracle
expectations and a guest runner.

1 files changed, 469 insertions, 5 deletions

diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index f0f07d8..581dc9d 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -13,6 +13,7 @@
 import os
 import re
 import socket
+import struct
 import subprocess
 import sys
 import threading
@@ -66,6 +67,371 @@ TRACE_PIPE = ("/sys/kernel/tracing/trace_pipe"
 SEM_EVERY = int(os.environ.get("STREAM_SEM_EVERY", "4"))
 
 
+class Kcore:
+    """Kernel-truth reader: /proc/kcore for contents + manual page-table walk.
+
+    The host's QMP-based struct walk intermittently reads garbage while the
+    client spins in user mode (PTI hides the kernel half from the sampled
+    vCPU), which left the CE push-buffer / semaphore-pool pages unpinned on
+    ~half of all boots (the deterministic cudaMalloc stall). The guest can
+    read its own kernel memory reliably: /proc/kcore has an ELF segment for
+    every kernel mapping, and the direct-map segment lets us walk the page
+    tables (init_top_pgt -> ... -> PTE) to turn the vmalloc CPU alias of a
+    UVM rm_mem into the guest physical addresses the host needs.
+    """
+
+    def __init__(self):
+        self.ok = False
+        self.fd = None
+        self.segs = []
+        self.phys_base = 0
+        self.page_offset_base = 0
+        self.pgd_pa = 0
+        self._pa_cache = {}
+
+    def load(self):
+        try:
+            self.fd = open("/proc/kcore", "rb")
+            hdr = self.fd.read(64)
+            if hdr[:4] != b"\x7fELF":
+                return False
+            e_phoff = struct.unpack_from("<Q", hdr, 32)[0]
+            e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54)
+            segs = []
+            for i in range(e_phnum):
+                self.fd.seek(e_phoff + i * e_phentsize)
+                ph = self.fd.read(56)
+                p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from(
+                    "<IIQQQQ", ph, 0)
+                if p_type == 1:
+                    segs.append((p_va, p_va + p_sz, p_off))
+            self.segs = segs
+            syms = {}
+            for line in open("/proc/kallsyms"):
+                parts = line.split()
+                if len(parts) == 3:
+                    syms.setdefault(parts[2], int(parts[0], 16))
+            need = ("phys_base", "page_offset_base", "init_top_pgt")
+            if any(s not in syms or not syms[s] for s in need):
+                return False
+            self.phys_base = self.read64(syms["phys_base"])
+            self.page_offset_base = self.read64(syms["page_offset_base"])
+            if not self.phys_base or not self.page_offset_base:
+                return False
+            self.pgd_pa = (syms["init_top_pgt"] - 0xFFFFFFFF80000000 +
+                           self.phys_base)
+            self.ok = True
+            return True
+        except OSError:
+            return False
+
+    def read(self, va, n):
+        for lo, hi, off in self.segs:
+            if lo <= va and va + n <= hi:
+                self.fd.seek(off + (va - lo))
+                d = self.fd.read(n)
+                return d if d and len(d) == n else None
+        return None
+
+    def read64(self, va):
+        d = self.read(va, 8)
+        return struct.unpack("<Q", d)[0] if d else None
+
+    def phys64(self, pa):
+        return self.read64(self.page_offset_base + (pa & 0xFFFFFFFFF))
+
+    def va_to_pa(self, va):
+        page = va & ~0xFFF
+        cached = self._pa_cache.get(page)
+        if cached is not None:
+            return cached or None
+        pa = self.pgd_pa
+        for level, shift in enumerate((39, 30, 21, 12)):
+            e = self.phys64(pa + ((va >> shift) & 0x1FF) * 8)
+            if e is None or not (e & 1):
+                self._pa_cache[page] = 0
+                return None
+            if level == 3:
+                pa = e & 0x000FFFFFFFF000
+                break
+            if e & 0x80:
+                if level == 1:
+                    pa = (e & 0x000FFFFFC0000000) + (va & ((1 << 30) - 1))
+                    break
+                if level == 2:
+                    pa = (e & 0x000FFFFFFFE00000) + (va & ((1 << 21) - 1))
+                    break
+                self._pa_cache[page] = 0
+                return None
+            pa = e & 0x000FFFFFFFF000
+        self._pa_cache[page] = pa
+        return pa
+
+
+def is_kern_ptr(v):
+    return v and (v >> 48) == 0xFFFF
+
+
+def is_gpu_va(v):
+    return v and 0x120000000 <= v < 0x200000000
+
+
+def _direct_map_pa(kc, va):
+    """Direct-map aliases translate linearly: pa = va - page_offset_base."""
+    if kc.page_offset_base <= va < kc.page_offset_base + (1 << 40):
+        return va - kc.page_offset_base
+    return None
+
+
+def _vmalloc_pages_for(kc, va):
+    """Page count of the vmap/vmalloc area containing va (0 if unknown)."""
+    now = time.time()
+    if not hasattr(_vmalloc_pages_for, "areas") or \
+            now - _vmalloc_pages_for.ts > 5.0:
+        areas = []
+        try:
+            for line in open("/proc/vmallocinfo"):
+                p = line.split()
+                if len(p) < 3:
+                    continue
+                try:
+                    lo, hi = p[0].split("-")
+                    areas.append((int(lo, 16), int(hi, 16)))
+                except ValueError:
+                    continue
+        except OSError:
+            pass
+        _vmalloc_pages_for.areas = areas
+        _vmalloc_pages_for.ts = now
+    base = va & ~0xFFF
+    for lo, hi in _vmalloc_pages_for.areas:
+        if lo <= base < hi:
+            return (hi - base) >> 12
+    return 0
+
+
+def resolve_mem_pins(kc, mem, out, default_pages=16):
+    """uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk)."""
+    if not is_kern_ptr(mem):
+        return 0
+    vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)]
+    cpu = next((v for v in vas if is_kern_ptr(v)), None)
+    gpus = [v for v in vas if is_gpu_va(v)]
+    if not cpu or not gpus:
+        return 0
+    size = kc.read64(mem + 0x60) or 0
+    dm = _direct_map_pa(kc, cpu)
+    emitted = 0
+    if dm is not None:
+        npages = max(1, min(size >> 12, 4098)) if size else default_pages
+        for gv in gpus:
+            out.append("0x%x,0x%x,%d\n" % (gv, dm, npages))
+            emitted += 1
+        return emitted
+    npages = (max(1, min(size >> 12, 4098)) if size
+              else _vmalloc_pages_for(kc, cpu))
+    if not npages:
+        npages = 1
+    runs = []
+    cur = None
+    for k in range(npages):
+        pa = kc.va_to_pa(cpu + k * 0x1000)
+        if pa is None:
+            if cur:
+                runs.append(cur)
+                cur = None
+            continue
+        if cur and pa == cur[1] + cur[2] * 0x1000:
+            cur[2] += 1
+        else:
+            if cur:
+                runs.append(cur)
+            cur = [k, pa, 1]
+    if cur:
+        runs.append(cur)
+    for gv in gpus:
+        for k, pa, n in runs:
+            out.append("0x%x,0x%x,%d\n" % (gv + k * 0x1000, pa, n))
+            emitted += 1
+    return emitted
+
+
+def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out):
+    """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT
+    buffer usually sits) from the ring's CPU mapping."""
+    emitted = 0
+    if not is_kern_ptr(entries_cpu) or not (32 <= entries <= 0x100000):
+        return 0
+    npages = min(((entries * 8) + 0xFFF) // 0x1000 + 2, 8)
+    runs = []
+    cur = None
+    for k in range(npages):
+        pa = kc.va_to_pa(entries_cpu + k * 0x1000)
+        if pa is None:
+            if cur:
+                runs.append(cur)
+                cur = None
+            continue
+        if cur and pa == cur[1] + cur[2] * 0x1000:
+            cur[2] += 1
+        else:
+            if cur:
+                runs.append(cur)
+            cur = [k, pa, 1]
+    if cur:
+        runs.append(cur)
+    for k, pa, n in runs:
+        out.append("0x%x,0x%x,%d\n" % (gpu_va + k * 0x1000, pa, n))
+        emitted += 1
+    return emitted
+
+
+def resolve_channel_from_sem(kc, sem, out):
+    """Tracking sem (embedded in uvm_channel_t) -> ring/GPPUT pins.
+
+    uvch events do not fire on this driver build, but the channel's tracking
+    semaphore is a uvm_channel_t member updated on every push. Locate
+    channel_info by scanning around the sem for its gpFifoGpuVa value (a
+    page-aligned GPU VA in the GR ring window), then read the ring/GPPUT
+    CPU pointers (UvmGpuChannelInfo offsets: gpPut +0x08, gpFifoEntries
+    +0x10, numGpFifoEntries +0x18, gpFifoGpuVa +0x58, gpPutGpuVa +0x60).
+    """
+    if not is_kern_ptr(sem):
+        return 0
+    lo = sem - 0x2000
+    data = kc.read(lo, 0x4000)
+    if not data:
+        return 0
+    for off in range(0, len(data) - 8, 8):
+        v = struct.unpack_from("<Q", data, off)[0]
+        if not (GR_RING_LO <= v < GR_RING_HI) or (v & 0xFFF):
+            continue
+        pos = lo + off
+        entries_cpu = kc.read64(pos - 0x48)
+        entries = kc.read64(pos - 0x40)
+        if entries:
+            entries &= 0xFFFFFFFF
+        emitted = _emit_ring_pins(kc, v, entries_cpu, entries, out)
+        gp_put = kc.read64(pos - 0x50)
+        gp_put_va = kc.read64(pos + 0x08)
+        if is_kern_ptr(gp_put) and gp_put_va and \
+                GR_RING_LO <= gp_put_va < 0x201000000:
+            pa = kc.va_to_pa(gp_put)
+            if pa is not None:
+                out.append("0x%x,0x%x,1\n" % (gp_put_va & ~0xFFF,
+                                               pa & ~0xFFF))
+                emitted += 1
+        return emitted
+    return 0
+
+
+def resolve_sem_pins(kc, sem, out):
+    vals = [kc.read64(sem + i * 8) for i in range(12)]
+    if not vals[0] or not is_kern_ptr(vals[0]):
+        return 0
+    mem = kc.read64(vals[0])
+    if not mem or not is_kern_ptr(mem):
+        return 0
+    n = resolve_mem_pins(kc, mem, out, default_pages=16)
+    try:
+        n += resolve_channel_from_sem(kc, sem, out)
+    except Exception:
+        pass
+    return n
+
+
+def resolve_pb_pins(kc, pb, out):
+    mem = kc.read64(pb + 8)
+    if not mem:
+        return 0
+    return resolve_mem_pins(kc, mem, out, default_pages=64)
+
+
+# UvmGpuChannelInfo field offsets (nv_uvm_types.h, x86_64):
+#   +0x00 gpGet, +0x08 gpPut, +0x10 gpFifoEntries, +0x18 numGpFifoEntries,
+#   +0x58 gpFifoGpuVa, +0x60 gpPutGpuVa, +0x68 gpGetGpuVa.
+# The offset of channel_info inside uvm_channel_t is config-dependent, so the
+# struct is located by searching the channel for its gpFifoGpuVa value (a GPU
+# VA in the GR ring window, page aligned).
+GR_RING_LO = 0x200200000
+GR_RING_HI = 0x200400000
+CHAN_INFO_GPFIFO_VA = 0x58
+
+
+def resolve_channel_pins(kc, ch, out):
+    """uvm_channel_t kptr -> pin the GPFIFO ring + GPPUT buffers.
+
+    The ring and GP_PUT buffers are CPU-mapped in the guest kernel (UVM
+    writes pushes and the put counter directly), so their guest-physical
+    backing is reachable with the page-table walk. Without these pins the
+    host cannot read GR channel entries (the stall class: RM event loop
+    waits for a completion whose ring entry was never processed).
+    """
+    if not is_kern_ptr(ch):
+        return 0
+    base = None
+    gpu_va = None
+    data = kc.read(ch, 0x800)
+    if not data:
+        return 0
+    for off in range(0, len(data) - 8, 8):
+        v = struct.unpack_from("<Q", data, off)[0]
+        if GR_RING_LO <= v < GR_RING_HI and (v & 0xFFF) == 0:
+            base = ch + off - CHAN_INFO_GPFIFO_VA
+            gpu_va = v
+            break
+    if base is None:
+        return 0
+    entries = kc.read64(base + 0x18)
+    if entries:
+        entries &= 0xFFFFFFFF
+    if not entries or entries > 0x100000:
+        return 0
+    emitted = 0
+    entries_cpu = kc.read64(base + 0x10)
+    if is_kern_ptr(entries_cpu):
+        nbytes = entries * 8
+        npages = min((nbytes + 0xFFF) // 0x1000 + 2, 8)
+        runs = []
+        cur = None
+        for k in range(npages):
+            pa = kc.va_to_pa(entries_cpu + k * 0x1000)
+            if pa is None:
+                if cur:
+                    runs.append(cur)
+                    cur = None
+                continue
+            if cur and pa == cur[1] + cur[2] * 0x1000:
+                cur[2] += 1
+            else:
+                if cur:
+                    runs.append(cur)
+                cur = [k, pa, 1]
+        if cur:
+            runs.append(cur)
+        for k, pa, n in runs:
+            out.append("0x%x,0x%x,%d\n" % (gpu_va + k * 0x1000, pa, n))
+            emitted += 1
+    gp_put = kc.read64(base + 0x08)
+    gp_put_va = kc.read64(base + 0x60)
+    if is_kern_ptr(gp_put) and gp_put_va and \
+            GR_RING_LO <= gp_put_va < 0x201000000:
+        pa = kc.va_to_pa(gp_put)
+        if pa is not None:
+            out.append("0x%x,0x%x,1\n" % (gp_put_va & ~0xFFF, pa & ~0xFFF))
+            emitted += 1
+    gp_get = kc.read64(base + 0x00)
+    gp_get_va = kc.read64(base + 0x68)
+    if is_kern_ptr(gp_get) and gp_get_va and \
+            GR_RING_LO <= gp_get_va < 0x201000000:
+        pa = kc.va_to_pa(gp_get)
+        if pa is not None:
+            out.append("0x%x,0x%x,1\n" % (gp_get_va & ~0xFFF, pa & ~0xFFF))
+            emitted += 1
+    return emitted
+
+
+
 def clear_trace():
     """Drop stale ring events so the fallback tail reader cannot re-learn a
     previous client's freed sems after a client switch. The new client's
@@ -133,6 +499,11 @@ class TraceReader:
                     if ("p", v) not in self.seen:
                         self.seen.add(("p", v))
                         self.lines.append("#pb 0x%x\n" % v)
+                for m in re.finditer(rb"ch=0x([0-9a-fA-F]+)", data):
+                    v = int(m.group(1), 16)
+                    if ("c", v) not in self.seen:
+                        self.seen.add(("c", v))
+                        self.lines.append("#ch 0x%x\n" % v)
 
     def drain(self):
         with self.lock:
@@ -163,17 +534,27 @@ def new_sems(seen):
 def cdt_pid():
     prefixes = tuple(p for p in os.environ.get("CLIENT_PREFIXES",
                                                "cdt,va,victory").split(",") if p)
+    best = None
     for d in os.listdir('/proc'):
         if not d.isdigit():
             continue
         try:
             with open('/proc/%s/comm' % d) as f:
                 name = f.read().strip()
-                if any(name.startswith(p) for p in prefixes):
-                    return int(d)
+            if not any(name.startswith(p) for p in prefixes):
+                continue
+            # Skip zombies: a defunct previous client keeps its comm and
+            # would otherwise shadow the live client (pagemap reads then fail
+            # and no pins are ever streamed).
+            with open('/proc/%s/stat' % d) as f:
+                st = f.read()
+            rp = st.rfind(')')
+            if rp >= 0 and st[rp + 2:rp + 3] == 'Z':
+                continue
+            best = int(d)
         except OSError:
             pass
-    return None
+    return best
 
 
 def scan_window(fd, lo, hi, span, out):
@@ -318,7 +699,21 @@ def main():
     sock = None
     seen = {}
     seen_sems = set()
+    seen_pbs = set()
+    seen_chs = set()
+    resolved_sems = set()
+    resolved_pbs = set()
+    resolved_chs = set()
+    retry_at = {}
     tracer = TraceReader()
+    kc = Kcore()
+    kc_ok = kc.load()
+    try:
+        with open('/tmp/guest_pin_stream.log', 'a') as f:
+            f.write('%s resolver: kcore=%s (sem->pool/pushbuffer pins)\n'
+                    % (time.strftime('%T'), 'ON' if kc_ok else 'OFF'))
+    except OSError:
+        pass
     last_pid = None
     last_log = 0.0
     loop = 0
@@ -329,12 +724,20 @@ def main():
             last_pid = None
             seen = {}
             seen_sems = set()
+            seen_pbs = set()
+            seen_chs = set()
             time.sleep(INTERVAL)
             continue
         if pid != last_pid:
             last_pid = pid
             seen = {}
             seen_sems = set()
+            seen_pbs = set()
+            seen_chs = set()
+            resolved_sems = set()
+            resolved_pbs = set()
+            resolved_chs = set()
+            retry_at = {}
             clear_trace()
             if sock is not None:
                 try:
@@ -373,9 +776,70 @@ def main():
         pending_cur = cur
         lines = []
         for ctl in tracer.drain():
-            if ctl.startswith("#sem"):
-                seen_sems.add(int(ctl.split()[1], 16))
+            parts = ctl.split()
+            if ctl.startswith("#sem") and len(parts) >= 2:
+                seen_sems.add(int(parts[1], 16))
+            elif ctl.startswith("#pb") and len(parts) >= 2:
+                seen_pbs.add(int(parts[1], 16))
+            elif ctl.startswith("#ch") and len(parts) >= 2:
+                seen_chs.add(int(parts[1], 16))
             lines.append(ctl)
+        if kc_ok:
+            now_r = time.time()
+            added = 0
+            for sem in list(seen_sems):
+                if sem in resolved_sems or now_r < retry_at.get(sem, 0):
+                    continue
+                got = []
+                try:
+                    resolve_sem_pins(kc, sem, got)
+                except Exception:
+                    got = []
+                if got:
+                    lines.extend(got)
+                    resolved_sems.add(sem)
+                    added += len(got)
+                else:
+                    retry_at[sem] = now_r + 1.0
+            for pb in list(seen_pbs):
+                if pb in resolved_pbs or now_r < retry_at.get(pb, 0):
+                    continue
+                got = []
+                try:
+                    resolve_pb_pins(kc, pb, got)
+                except Exception:
+                    got = []
+                if got:
+                    lines.extend(got)
+                    resolved_pbs.add(pb)
+                    added += len(got)
+                else:
+                    retry_at[pb] = now_r + 1.0
+            for ch in list(seen_chs):
+                if ch in resolved_chs or now_r < retry_at.get(ch, 0):
+                    continue
+                got = []
+                try:
+                    resolve_channel_pins(kc, ch, got)
+                except Exception:
+                    got = []
+                if got:
+                    lines.extend(got)
+                    resolved_chs.add(ch)
+                    added += len(got)
+                else:
+                    retry_at[ch] = now_r + 1.0
+            if added:
+                try:
+                    with open('/tmp/guest_pin_stream.log', 'a') as f:
+                        f.write('%s resolver pins: +%d run(s) '
+                                '(sems=%d/%d pbs=%d/%d chs=%d/%d)\n'
+                                % (time.strftime('%T'), added,
+                                   len(resolved_sems), len(seen_sems),
+                                   len(resolved_pbs), len(seen_pbs),
+                                   len(resolved_chs), len(seen_chs)))
+                except OSError:
+                    pass
         now = time.time()
         if now - last_log > 1.0:
             last_log = now