diff options
Diffstat (limited to 'tools/guest-diag/guest_pin_stream.py')
| -rw-r--r-- | tools/guest-diag/guest_pin_stream.py | 474 |
1 files changed, 469 insertions, 5 deletions
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index f0f07d8..581dc9d 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -13,6 +13,7 @@ import os import re import socket +import struct import subprocess import sys import threading @@ -66,6 +67,371 @@ TRACE_PIPE = ("/sys/kernel/tracing/trace_pipe" SEM_EVERY = int(os.environ.get("STREAM_SEM_EVERY", "4")) +class Kcore: + """Kernel-truth reader: /proc/kcore for contents + manual page-table walk. + + The host's QMP-based struct walk intermittently reads garbage while the + client spins in user mode (PTI hides the kernel half from the sampled + vCPU), which left the CE push-buffer / semaphore-pool pages unpinned on + ~half of all boots (the deterministic cudaMalloc stall). The guest can + read its own kernel memory reliably: /proc/kcore has an ELF segment for + every kernel mapping, and the direct-map segment lets us walk the page + tables (init_top_pgt -> ... -> PTE) to turn the vmalloc CPU alias of a + UVM rm_mem into the guest physical addresses the host needs. + """ + + def __init__(self): + self.ok = False + self.fd = None + self.segs = [] + self.phys_base = 0 + self.page_offset_base = 0 + self.pgd_pa = 0 + self._pa_cache = {} + + def load(self): + try: + self.fd = open("/proc/kcore", "rb") + hdr = self.fd.read(64) + if hdr[:4] != b"\x7fELF": + return False + e_phoff = struct.unpack_from("<Q", hdr, 32)[0] + e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54) + segs = [] + for i in range(e_phnum): + self.fd.seek(e_phoff + i * e_phentsize) + ph = self.fd.read(56) + p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from( + "<IIQQQQ", ph, 0) + if p_type == 1: + segs.append((p_va, p_va + p_sz, p_off)) + self.segs = segs + syms = {} + for line in open("/proc/kallsyms"): + parts = line.split() + if len(parts) == 3: + syms.setdefault(parts[2], int(parts[0], 16)) + need = ("phys_base", "page_offset_base", "init_top_pgt") + if any(s not in syms or not syms[s] for s in need): + return False + self.phys_base = self.read64(syms["phys_base"]) + self.page_offset_base = self.read64(syms["page_offset_base"]) + if not self.phys_base or not self.page_offset_base: + return False + self.pgd_pa = (syms["init_top_pgt"] - 0xFFFFFFFF80000000 + + self.phys_base) + self.ok = True + return True + except OSError: + return False + + def read(self, va, n): + for lo, hi, off in self.segs: + if lo <= va and va + n <= hi: + self.fd.seek(off + (va - lo)) + d = self.fd.read(n) + return d if d and len(d) == n else None + return None + + def read64(self, va): + d = self.read(va, 8) + return struct.unpack("<Q", d)[0] if d else None + + def phys64(self, pa): + return self.read64(self.page_offset_base + (pa & 0xFFFFFFFFF)) + + def va_to_pa(self, va): + page = va & ~0xFFF + cached = self._pa_cache.get(page) + if cached is not None: + return cached or None + pa = self.pgd_pa + for level, shift in enumerate((39, 30, 21, 12)): + e = self.phys64(pa + ((va >> shift) & 0x1FF) * 8) + if e is None or not (e & 1): + self._pa_cache[page] = 0 + return None + if level == 3: + pa = e & 0x000FFFFFFFF000 + break + if e & 0x80: + if level == 1: + pa = (e & 0x000FFFFFC0000000) + (va & ((1 << 30) - 1)) + break + if level == 2: + pa = (e & 0x000FFFFFFFE00000) + (va & ((1 << 21) - 1)) + break + self._pa_cache[page] = 0 + return None + pa = e & 0x000FFFFFFFF000 + self._pa_cache[page] = pa + return pa + + +def is_kern_ptr(v): + return v and (v >> 48) == 0xFFFF + + +def is_gpu_va(v): + return v and 0x120000000 <= v < 0x200000000 + + +def _direct_map_pa(kc, va): + """Direct-map aliases translate linearly: pa = va - page_offset_base.""" + if kc.page_offset_base <= va < kc.page_offset_base + (1 << 40): + return va - kc.page_offset_base + return None + + +def _vmalloc_pages_for(kc, va): + """Page count of the vmap/vmalloc area containing va (0 if unknown).""" + now = time.time() + if not hasattr(_vmalloc_pages_for, "areas") or \ + now - _vmalloc_pages_for.ts > 5.0: + areas = [] + try: + for line in open("/proc/vmallocinfo"): + p = line.split() + if len(p) < 3: + continue + try: + lo, hi = p[0].split("-") + areas.append((int(lo, 16), int(hi, 16))) + except ValueError: + continue + except OSError: + pass + _vmalloc_pages_for.areas = areas + _vmalloc_pages_for.ts = now + base = va & ~0xFFF + for lo, hi in _vmalloc_pages_for.areas: + if lo <= base < hi: + return (hi - base) >> 12 + return 0 + + +def resolve_mem_pins(kc, mem, out, default_pages=16): + """uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk).""" + if not is_kern_ptr(mem): + return 0 + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpus = [v for v in vas if is_gpu_va(v)] + if not cpu or not gpus: + return 0 + size = kc.read64(mem + 0x60) or 0 + dm = _direct_map_pa(kc, cpu) + emitted = 0 + if dm is not None: + npages = max(1, min(size >> 12, 4098)) if size else default_pages + for gv in gpus: + out.append("0x%x,0x%x,%d\n" % (gv, dm, npages)) + emitted += 1 + return emitted + npages = (max(1, min(size >> 12, 4098)) if size + else _vmalloc_pages_for(kc, cpu)) + if not npages: + npages = 1 + runs = [] + cur = None + for k in range(npages): + pa = kc.va_to_pa(cpu + k * 0x1000) + if pa is None: + if cur: + runs.append(cur) + cur = None + continue + if cur and pa == cur[1] + cur[2] * 0x1000: + cur[2] += 1 + else: + if cur: + runs.append(cur) + cur = [k, pa, 1] + if cur: + runs.append(cur) + for gv in gpus: + for k, pa, n in runs: + out.append("0x%x,0x%x,%d\n" % (gv + k * 0x1000, pa, n)) + emitted += 1 + return emitted + + +def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out): + """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT + buffer usually sits) from the ring's CPU mapping.""" + emitted = 0 + if not is_kern_ptr(entries_cpu) or not (32 <= entries <= 0x100000): + return 0 + npages = min(((entries * 8) + 0xFFF) // 0x1000 + 2, 8) + runs = [] + cur = None + for k in range(npages): + pa = kc.va_to_pa(entries_cpu + k * 0x1000) + if pa is None: + if cur: + runs.append(cur) + cur = None + continue + if cur and pa == cur[1] + cur[2] * 0x1000: + cur[2] += 1 + else: + if cur: + runs.append(cur) + cur = [k, pa, 1] + if cur: + runs.append(cur) + for k, pa, n in runs: + out.append("0x%x,0x%x,%d\n" % (gpu_va + k * 0x1000, pa, n)) + emitted += 1 + return emitted + + +def resolve_channel_from_sem(kc, sem, out): + """Tracking sem (embedded in uvm_channel_t) -> ring/GPPUT pins. + + uvch events do not fire on this driver build, but the channel's tracking + semaphore is a uvm_channel_t member updated on every push. Locate + channel_info by scanning around the sem for its gpFifoGpuVa value (a + page-aligned GPU VA in the GR ring window), then read the ring/GPPUT + CPU pointers (UvmGpuChannelInfo offsets: gpPut +0x08, gpFifoEntries + +0x10, numGpFifoEntries +0x18, gpFifoGpuVa +0x58, gpPutGpuVa +0x60). + """ + if not is_kern_ptr(sem): + return 0 + lo = sem - 0x2000 + data = kc.read(lo, 0x4000) + if not data: + return 0 + for off in range(0, len(data) - 8, 8): + v = struct.unpack_from("<Q", data, off)[0] + if not (GR_RING_LO <= v < GR_RING_HI) or (v & 0xFFF): + continue + pos = lo + off + entries_cpu = kc.read64(pos - 0x48) + entries = kc.read64(pos - 0x40) + if entries: + entries &= 0xFFFFFFFF + emitted = _emit_ring_pins(kc, v, entries_cpu, entries, out) + gp_put = kc.read64(pos - 0x50) + gp_put_va = kc.read64(pos + 0x08) + if is_kern_ptr(gp_put) and gp_put_va and \ + GR_RING_LO <= gp_put_va < 0x201000000: + pa = kc.va_to_pa(gp_put) + if pa is not None: + out.append("0x%x,0x%x,1\n" % (gp_put_va & ~0xFFF, + pa & ~0xFFF)) + emitted += 1 + return emitted + return 0 + + +def resolve_sem_pins(kc, sem, out): + vals = [kc.read64(sem + i * 8) for i in range(12)] + if not vals[0] or not is_kern_ptr(vals[0]): + return 0 + mem = kc.read64(vals[0]) + if not mem or not is_kern_ptr(mem): + return 0 + n = resolve_mem_pins(kc, mem, out, default_pages=16) + try: + n += resolve_channel_from_sem(kc, sem, out) + except Exception: + pass + return n + + +def resolve_pb_pins(kc, pb, out): + mem = kc.read64(pb + 8) + if not mem: + return 0 + return resolve_mem_pins(kc, mem, out, default_pages=64) + + +# UvmGpuChannelInfo field offsets (nv_uvm_types.h, x86_64): +# +0x00 gpGet, +0x08 gpPut, +0x10 gpFifoEntries, +0x18 numGpFifoEntries, +# +0x58 gpFifoGpuVa, +0x60 gpPutGpuVa, +0x68 gpGetGpuVa. +# The offset of channel_info inside uvm_channel_t is config-dependent, so the +# struct is located by searching the channel for its gpFifoGpuVa value (a GPU +# VA in the GR ring window, page aligned). +GR_RING_LO = 0x200200000 +GR_RING_HI = 0x200400000 +CHAN_INFO_GPFIFO_VA = 0x58 + + +def resolve_channel_pins(kc, ch, out): + """uvm_channel_t kptr -> pin the GPFIFO ring + GPPUT buffers. + + The ring and GP_PUT buffers are CPU-mapped in the guest kernel (UVM + writes pushes and the put counter directly), so their guest-physical + backing is reachable with the page-table walk. Without these pins the + host cannot read GR channel entries (the stall class: RM event loop + waits for a completion whose ring entry was never processed). + """ + if not is_kern_ptr(ch): + return 0 + base = None + gpu_va = None + data = kc.read(ch, 0x800) + if not data: + return 0 + for off in range(0, len(data) - 8, 8): + v = struct.unpack_from("<Q", data, off)[0] + if GR_RING_LO <= v < GR_RING_HI and (v & 0xFFF) == 0: + base = ch + off - CHAN_INFO_GPFIFO_VA + gpu_va = v + break + if base is None: + return 0 + entries = kc.read64(base + 0x18) + if entries: + entries &= 0xFFFFFFFF + if not entries or entries > 0x100000: + return 0 + emitted = 0 + entries_cpu = kc.read64(base + 0x10) + if is_kern_ptr(entries_cpu): + nbytes = entries * 8 + npages = min((nbytes + 0xFFF) // 0x1000 + 2, 8) + runs = [] + cur = None + for k in range(npages): + pa = kc.va_to_pa(entries_cpu + k * 0x1000) + if pa is None: + if cur: + runs.append(cur) + cur = None + continue + if cur and pa == cur[1] + cur[2] * 0x1000: + cur[2] += 1 + else: + if cur: + runs.append(cur) + cur = [k, pa, 1] + if cur: + runs.append(cur) + for k, pa, n in runs: + out.append("0x%x,0x%x,%d\n" % (gpu_va + k * 0x1000, pa, n)) + emitted += 1 + gp_put = kc.read64(base + 0x08) + gp_put_va = kc.read64(base + 0x60) + if is_kern_ptr(gp_put) and gp_put_va and \ + GR_RING_LO <= gp_put_va < 0x201000000: + pa = kc.va_to_pa(gp_put) + if pa is not None: + out.append("0x%x,0x%x,1\n" % (gp_put_va & ~0xFFF, pa & ~0xFFF)) + emitted += 1 + gp_get = kc.read64(base + 0x00) + gp_get_va = kc.read64(base + 0x68) + if is_kern_ptr(gp_get) and gp_get_va and \ + GR_RING_LO <= gp_get_va < 0x201000000: + pa = kc.va_to_pa(gp_get) + if pa is not None: + out.append("0x%x,0x%x,1\n" % (gp_get_va & ~0xFFF, pa & ~0xFFF)) + emitted += 1 + return emitted + + + def clear_trace(): """Drop stale ring events so the fallback tail reader cannot re-learn a previous client's freed sems after a client switch. The new client's @@ -133,6 +499,11 @@ class TraceReader: if ("p", v) not in self.seen: self.seen.add(("p", v)) self.lines.append("#pb 0x%x\n" % v) + for m in re.finditer(rb"ch=0x([0-9a-fA-F]+)", data): + v = int(m.group(1), 16) + if ("c", v) not in self.seen: + self.seen.add(("c", v)) + self.lines.append("#ch 0x%x\n" % v) def drain(self): with self.lock: @@ -163,17 +534,27 @@ def new_sems(seen): def cdt_pid(): prefixes = tuple(p for p in os.environ.get("CLIENT_PREFIXES", "cdt,va,victory").split(",") if p) + best = None for d in os.listdir('/proc'): if not d.isdigit(): continue try: with open('/proc/%s/comm' % d) as f: name = f.read().strip() - if any(name.startswith(p) for p in prefixes): - return int(d) + if not any(name.startswith(p) for p in prefixes): + continue + # Skip zombies: a defunct previous client keeps its comm and + # would otherwise shadow the live client (pagemap reads then fail + # and no pins are ever streamed). + with open('/proc/%s/stat' % d) as f: + st = f.read() + rp = st.rfind(')') + if rp >= 0 and st[rp + 2:rp + 3] == 'Z': + continue + best = int(d) except OSError: pass - return None + return best def scan_window(fd, lo, hi, span, out): @@ -318,7 +699,21 @@ def main(): sock = None seen = {} seen_sems = set() + seen_pbs = set() + seen_chs = set() + resolved_sems = set() + resolved_pbs = set() + resolved_chs = set() + retry_at = {} tracer = TraceReader() + kc = Kcore() + kc_ok = kc.load() + try: + with open('/tmp/guest_pin_stream.log', 'a') as f: + f.write('%s resolver: kcore=%s (sem->pool/pushbuffer pins)\n' + % (time.strftime('%T'), 'ON' if kc_ok else 'OFF')) + except OSError: + pass last_pid = None last_log = 0.0 loop = 0 @@ -329,12 +724,20 @@ def main(): last_pid = None seen = {} seen_sems = set() + seen_pbs = set() + seen_chs = set() time.sleep(INTERVAL) continue if pid != last_pid: last_pid = pid seen = {} seen_sems = set() + seen_pbs = set() + seen_chs = set() + resolved_sems = set() + resolved_pbs = set() + resolved_chs = set() + retry_at = {} clear_trace() if sock is not None: try: @@ -373,9 +776,70 @@ def main(): pending_cur = cur lines = [] for ctl in tracer.drain(): - if ctl.startswith("#sem"): - seen_sems.add(int(ctl.split()[1], 16)) + parts = ctl.split() + if ctl.startswith("#sem") and len(parts) >= 2: + seen_sems.add(int(parts[1], 16)) + elif ctl.startswith("#pb") and len(parts) >= 2: + seen_pbs.add(int(parts[1], 16)) + elif ctl.startswith("#ch") and len(parts) >= 2: + seen_chs.add(int(parts[1], 16)) lines.append(ctl) + if kc_ok: + now_r = time.time() + added = 0 + for sem in list(seen_sems): + if sem in resolved_sems or now_r < retry_at.get(sem, 0): + continue + got = [] + try: + resolve_sem_pins(kc, sem, got) + except Exception: + got = [] + if got: + lines.extend(got) + resolved_sems.add(sem) + added += len(got) + else: + retry_at[sem] = now_r + 1.0 + for pb in list(seen_pbs): + if pb in resolved_pbs or now_r < retry_at.get(pb, 0): + continue + got = [] + try: + resolve_pb_pins(kc, pb, got) + except Exception: + got = [] + if got: + lines.extend(got) + resolved_pbs.add(pb) + added += len(got) + else: + retry_at[pb] = now_r + 1.0 + for ch in list(seen_chs): + if ch in resolved_chs or now_r < retry_at.get(ch, 0): + continue + got = [] + try: + resolve_channel_pins(kc, ch, got) + except Exception: + got = [] + if got: + lines.extend(got) + resolved_chs.add(ch) + added += len(got) + else: + retry_at[ch] = now_r + 1.0 + if added: + try: + with open('/tmp/guest_pin_stream.log', 'a') as f: + f.write('%s resolver pins: +%d run(s) ' + '(sems=%d/%d pbs=%d/%d chs=%d/%d)\n' + % (time.strftime('%T'), added, + len(resolved_sems), len(seen_sems), + len(resolved_pbs), len(seen_pbs), + len(resolved_chs), len(seen_chs))) + except OSError: + pass now = time.time() if now - last_log > 1.0: last_log = now |