diff options
Diffstat (limited to 'tools/guest-diag')
| -rw-r--r-- | tools/guest-diag/guest_pin_stream.py | 288 | ||||
| -rw-r--r-- | tools/guest-diag/pin_gr_stream.py | 23 |
2 files changed, 298 insertions, 13 deletions
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 97c68f2..10a9aba 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -400,6 +400,77 @@ def mem_page_off(kc, mem): return 0 +def _ce_mem_pages(kc, mem): + """CE mem walk parameters: (gpus, cpu, off, npages) or None. + + Mirrors resolve_mem_pins' derivation so the delta sweeper emits the same + page set the one-shot resolver used: size at +0x128 when readable, else + the vmalloc area length; all CE-window GPU VAs share the CPU mapping. + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpus = [v for v in vas if v and PB_CE_LO <= v < PB_CE_HI] + if not cpu or not gpus: + return None + if _direct_map_pa(kc, cpu) is not None: + return None # never a vmap: not the push-buffer shape + off = mem_page_off(kc, mem) + area = _vmalloc_pages_for(kc, cpu) + size = _mem_size(kc, mem) + npages = (max(1, min(size >> 12, 4098)) if size + else area or 64) + if area and off + npages > area: + npages = max(1, area - off) + return (gpus, cpu, off, npages) + + +def _emit_ce_mem_delta(kc, st, out, fresh_map=False): + """Emit the CE mem's live {va,pa} pairs as content-validated runs. + + The one-shot emission this replaces pinned the whole 16 MB window + including pages whose content was zero at the time and pages whose vmap + pairing had drifted; the host then read zeros for the other CE ranks' + pushes (the second-client stall class). The sweep now skips zero pages + (a zero page cannot carry a push), re-derives every PA against the live + page tables (optionally bursting the Kcore cache first), and emits only + pairs that are new or changed since the last sweep - so a page the + client writes later, or a mem whose vmap pages moved, is re-emitted + within KC_REEMIT_S and the host's last-store-wins pin sees the truth. + Never drops a previously emitted pair (a stale pin is corrected by the + changed pair, and a failed burst never clears the whole map). + """ + gpus, cpu, off, npages = st["gpus"], st["cpu"], st["page_off"], st["npages"] + last = st["last"] + if fresh_map: + for k in range(npages): + kc._pa_cache.pop(cpu + (k + off) * 0x1000, None) + changed = [] + for k in range(npages): + d = kc.read(cpu + (k + off) * 0x1000, 32) + if not d or not any(d): + continue # zero/unreadable: nothing to pin yet + pa = kc.va_to_pa(cpu + (k + off) * 0x1000) + if pa is None: + continue + for gv in gpus: + if last.get(gv + k * 0x1000) != pa: + changed.append((gv + k * 0x1000, pa)) + changed.sort() + merged = [] + for va, pa in changed: + if merged and va == merged[-1][0] + merged[-1][2] * 0x1000 and \ + pa == merged[-1][1] + merged[-1][2] * 0x1000: + merged[-1][2] += 1 + else: + merged.append([va, pa, 1]) + for va, pa, n in merged: + out.append("0x%x,0x%x,%d\n" % (va, pa, n)) + for va, pa in changed: + last[va] = pa + st["emit_total"] = st.get("emit_total", 0) + len(merged) + return len(merged) + + def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out): """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT buffer usually sits) from the ring's CPU mapping.""" @@ -559,8 +630,10 @@ KC_CHAIN_BACK = 0x2000 KC_CHAIN_SPAN = 0xA000 KC_CHAIN_MAX = 400 RESOLVED_KC_PB = set() -KC_PB_STATE = {} # sem kptr -> resolved-mem fingerprint (freshness) +KC_PB_STATE = {} # sem kptr -> resolved-mem state (fingerprint + pins) KC_REFRESH_S = 2.0 # re-validate a resolved CE mem at most this often +KC_REEMIT_S = 2.0 # delta-sweep the CE mem's live pages this often +KC_REMAP_S = 30.0 # full re-translation sweep (catches remaps) def _chain_mems_at(kc, w): @@ -639,6 +712,16 @@ def _kc_pb_fresh(kc, sem): return False if not st.get("page_va"): return True + if not st.get("page_pa"): + # Unknown PA at store time: the manual page-table walk could not + # translate the vmap page (it may have been read before the kernel + # page tables were walkable). Treat the record as fresh instead of + # failing every check - a permanently-failing freshness test made + # the scanner drop the latch and re-resolve in a tight loop, and + # each failed re-resolution left the host with no pins at all + # (observed r131: "kc-chain stale: re-resolving" every second for + # five sems while smiley sat in cudaMalloc). + return True pa = kc.va_to_pa(st["page_va"]) if pa is None or pa != st.get("page_pa"): return False @@ -669,6 +752,42 @@ def resolve_pb_chain(kc, sem, out): return False words = struct.unpack_from("<%dQ" % (len(data) // 8), data, 0) cands = list(dict.fromkeys(w for w in words if is_kern_ptr(w))) + # Channel-anchored expansion (r134 live dump): the tracking sem is a + # member of a uvm_channel_t table; the channel records sit in the sem's + # own window and carry their CE ring GPU VA ([0x121010000, 0x121400000)) + # plus back-to-back 32-bit object handles. The pool/manager/pushbuffer + # pointers hang off those records' targets, which the bare sem-window + # scan never reaches - the second-client stall is exactly this: the only + # live uvmt event is the spinning sem, and no mem candidate is found + # around it. For each ring-VA word, scan the surrounding record for + # kptrs, take those targets, and try the chain shape on both. Additive + # and bounded; content scoring still decides, so a wrong candidate + # cannot shadow the live pages. + extra = [] + for i, w in enumerate(words): + if not w or not (0x121010000 <= w < 0x121400000): + continue + lo = max(0, i - 0x60) + hi = min(len(words), i + 0x60) + for c in words[lo:hi]: + if not is_kern_ptr(c): + continue + extra.append(c) + tgt = kc.read(c, 0x100) + if tgt: + tw = struct.unpack_from("<%dQ" % (len(tgt) // 8), + tgt, 0) + for w2 in tw: + if is_kern_ptr(w2): + extra.append(w2) + if len(extra) >= 512: + break + if extra: + head = [] + for w in extra: + if w not in head and w not in cands: + head.append(w) + cands = head + cands mems = [] for w in cands[:KC_CHAIN_MAX]: for mem in _chain_mems_at(kc, w): @@ -697,18 +816,23 @@ def resolve_pb_chain(kc, sem, out): if mem in RESOLVED_KC_PB: return True RESOLVED_KC_PB.add(mem) - off = mem_page_off(kc, mem) - n = resolve_mem_pins(kc, mem, out, default_pages=64, - va_lo=PB_CE_LO, va_hi=PB_CE_HI, - page_off=off) - st = {"mem": mem, "gpu": gpu, "cpu": cpu, - "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, "page_off": 0} + params = _ce_mem_pages(kc, mem) + if not params: + return found + gpus, cpu, off, npages = params + st = {"mem": mem, "gpu": gpu, "cpu": cpu, "gpus": gpus, + "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, + "page_off": off, "npages": npages, "last": {}} + # Fresh translation for the first sweep: the Kcore PA cache may hold + # pairs from before this mem's vmap area was (re)populated. + kc._pa_cache.clear() + n = _emit_ce_mem_delta(kc, st, out) KC_PB_STATE[sem] = st try: with open('/tmp/guest_pin_stream.log', 'a') as f: f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x ' - 'pin-runs=%d off=%d size=%d score=%d\n' - % (time.strftime('%T'), mem, sem, n, off, + 'pin-runs=%d off=%d npages=%d size=%d score=%d\n' + % (time.strftime('%T'), mem, sem, n, off, npages, _mem_size(kc, mem), _score)) except OSError: pass @@ -975,6 +1099,75 @@ def new_sems(seen): return found +# --------------------------------------------------------------- scrubber +# PMA scrubber channel discovery (third-client scrub freeze). The guest's +# PMA scrubber is an internal CeUtils CE channel (4096 GPFIFO entries, +# pbGpuVA 0x120000000, ring at pbGpuVA+0x64000, finish payload at +# pbGpuVA+0x6c004). It lives in a kernel vmap area, not in any user pagemap, +# and its GPU VAs collide numerically with the client CE push-buffer window +# in the host's global translation map - so the host cannot resolve it and +# must be told the finish/host-semaphore word PAs directly. Resolve them +# here (vmallocinfo + manual kernel page-table walk) and emit one +# #scrub <fin_pa>,<hsem_pa> +# line; pin_gr_stream.py registers them through sema_scrubbuf_add. +SCRUB_AREA_SIZES = (0x6e000, 0x6d000, 0x6c000, 0x6b000, 0x6f000, + 0x70000, 0x71000, 0x72000) +_scrub_sent = {} + + +def _scrub_ring_ok(kc, ring_pa): + """A 906F GPFIFO ring page: all-zero (fresh) or entries whose GET names a + push inside the channel's own 0x120000000 window (encoded as + push_va - 0x100000000, 4-byte aligned).""" + buf = kc.read(kc.page_offset_base + ring_pa, 0x1000) + if not buf: + return False + for off in range(0, 0x100, 8): + e0, _e1 = struct.unpack_from('<II', buf, off) + if e0 == 0: + continue + if not (0x20000000 <= e0 < 0x20080000) or (e0 & 3): + return False + return True + + +def scrub_scan(kc, lines): + """Emit the scrubber buffer words' PAs once (re-emitted if they move).""" + try: + f = open('/proc/vmallocinfo') + except OSError: + return + with f: + for line in f: + if 'nvidia' not in line or 'ioremap' in line or 'vmap' not in line: + continue + parts = line.split() + try: + a, b = parts[0].split('-') + lo, hi = int(a, 16), int(b, 16) + except (ValueError, IndexError): + continue + if (hi - lo) not in SCRUB_AREA_SIZES: + continue + fin_pa = kc.va_to_pa(lo + 0x6c000) + ring_pa = kc.va_to_pa(lo + 0x64000) + if fin_pa is None or ring_pa is None: + continue + if not _scrub_ring_ok(kc, ring_pa): + dbg('scrub candidate 0x%x rejected (ring shape)' % lo) + continue + fin = fin_pa + 4 + hsem = fin_pa + key = (fin, hsem) + if _scrub_sent.get('key') == key: + return + _scrub_sent['key'] = key + lines.append('#scrub 0x%x,0x%x\n' % (fin, hsem)) + dbg('scrub buffer emitted lo=0x%x fin=0x%x hsem=0x%x' + % (lo, fin, hsem)) + return + + def nvidia_fds(pid): """CUDA device fds this pid holds: 'uvm', 'gpu', or None. @@ -1206,6 +1399,8 @@ def main(): resolved_kc_pb = set() retry_at = {} kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} tracer = TraceReader() kc = Kcore() kc_ok = kc.load() @@ -1217,6 +1412,7 @@ def main(): pass last_pid = None last_log = 0.0 + scrub_next = 0.0 loop = 0 while True: pid = cdt_pid() @@ -1242,6 +1438,8 @@ def main(): resolved_kc_pb = set() retry_at = {} kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} # Client switch: the old client's channel/put pointers are freed # memory. Clearing the #put map stops stale reads from being # emitted as if they were the new client's counters. @@ -1293,6 +1491,14 @@ def main(): new = [(va, pa) for va, pa in cur.items() if seen.get(va) != pa] pending_cur = cur lines = [] + # PMA scrubber buffer discovery (throttled): resolve the finish/host- + # semaphore word PAs and stream them for host registration. + if kc_ok and time.time() >= scrub_next: + scrub_next = time.time() + 5.0 + try: + scrub_scan(kc, lines) + except Exception as e: + dbg('scrub scan failed: %s' % e) for ctl in tracer.drain(): parts = ctl.split() if ctl.startswith("#sem") and len(parts) >= 2: @@ -1333,12 +1539,68 @@ def main(): except Exception: _fresh = True if not _fresh: - _st = KC_PB_STATE.pop(sem, None) + # Re-resolve, but never lose the latch on a failed + # attempt: a dropped latch means the host keeps no + # pins for this channel at all, which is worse than + # a possibly-stale pin (the host content-checks the + # first page and now rejects all-FF continuation + # chunks, so stale pages are bounded). On failure + # keep the old state and defer the next attempt. + _st = KC_PB_STATE.get(sem) if _st and _st.get("mem"): RESOLVED_KC_PB.discard(_st["mem"]) - resolved_kc_pb.discard(sem) - retry_at.pop(("pk", sem), None) - dbg("kc-chain stale: re-resolving sem=0x%x" % sem) + got = [] + try: + _found2 = resolve_pb_chain(kc, sem, got) + except Exception: + _found2 = False + got = [] + if _found2 and got: + _new = KC_PB_STATE.get(sem) + _newmem = _new.get("mem") if _new else None + dbg("kc-chain stale: re-resolved sem=0x%x " + "new-mem=%s runs=%d" + % (sem, "yes" if _newmem != + (_st.get("mem") if _st else None) + else "no", len(got))) + lines.extend(got) + added += len(got) + resolved_kc_pb.add(sem) + else: + if _st and _st.get("mem"): + RESOLVED_KC_PB.add(_st["mem"]) + if sem not in KC_PB_STATE: + KC_PB_STATE[sem] = _st + kc_refresh_at[sem] = now_r + 5.0 + dbg("kc-chain stale: keep old latch sem=0x%x" + % sem) + else: + # CE mem delta sweep (second-client pins): the one-shot + # emission pinned the mem once; pages the client writes + # later (or whose vmap pairing moved) were never + # re-emitted, so the host kept zeros for the other CE + # ranks' pushes. Sweep every KC_REEMIT_S, emitting only + # new/changed nonzero pairs; burst the Kcore PA cache + # every KC_REMAP_S so a remap is not shadowed. + _st = KC_PB_STATE.get(sem) + if _st is not None and \ + now_r >= kc_reemit_at.get(sem, 0): + _fresh_map = now_r >= kc_remap_at.get(sem, 0) + if _fresh_map: + kc_remap_at[sem] = now_r + KC_REMAP_S + kc_reemit_at[sem] = now_r + KC_REEMIT_S + got = [] + try: + _n2 = _emit_ce_mem_delta(kc, _st, got, + fresh_map=_fresh_map) + except Exception: + _n2 = 0 + got = [] + if got: + lines.extend(got) + added += len(got) + dbg("ce-delta sem=0x%x runs=%d mem=0x%x" + % (sem, _n2, _st.get("mem", 0))) # CE push-buffer chain (second-client stall fix): resolve # pool -> manager -> pushbuffer -> rm_mem directly, so the # push-buffer pages never depend on the racy one-shot uvbp diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index b864ab5..2a5d768 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -101,6 +101,22 @@ def inject(va, pa, ln): return False +def scrub_register(fin_pa, hsem_pa): + """Register the PMA scrubber buffer words (guest-resolved PAs) with the + service so it can author the scrub progress (third-client scrub freeze).""" + cmd = ("printf '0x%x,0x%x' > %s/sema_scrubbuf_add" + % (fin_pa, hsem_pa, PARAM)) + for _ in range(60): + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True) + if r.returncode == 0: + log("scrub registered fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa)) + return True + time.sleep(0.05) + log("scrub register FAILED fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa)) + return False + + def main(): if not install_and_start(): log("WARN: could not start guest scanner (will retry via vsock wait)") @@ -158,6 +174,13 @@ def main(): log(text) _mirror(text) continue + if text.startswith("#scrub"): + m = re.search(r"0x([0-9a-fA-F]+),0x([0-9a-fA-F]+)", text) + _mirror(text) + if m: + scrub_register(int(m.group(1), 16), + int(m.group(2), 16)) + continue m = re.match(r"^(0x[0-9a-f]+),(0x[0-9a-f]+)(?:,(\d+))?$", text) if m: runs.append((int(m.group(1), 16), int(m.group(2), 16), |