From b90ced5aa52d202e678918093d1bb497ec1efd2b Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 16 Oct 2026 20:54:49 -0400 Subject: vgsp: m163 lost-H2D copy integrity (drain injected pins before CE op resolution; hold failed device-copy ops) + m164 CLASS-2 GR no-capture entry hold + m165 pool release applies at fresh-pin time + m166 timeout-ladder warm-sem fail-fast; scanner: first-class pool VA resolution + periodic re-assert + demand routing --- tools/guest-diag/guest_pin_stream.py | 99 ++++++++++++++++++++++++++++++++++-- tools/guest-diag/pin_gr_stream.py | 11 ++-- 2 files changed, 101 insertions(+), 9 deletions(-) (limited to 'tools') diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 6012dd1..b308136 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -734,6 +734,49 @@ def resolve_sem_pins(kc, sem, out): return resolve_mem_pins(kc, mem, out, default_pages=16) +def resolve_pool_va(kc, va, out, sems): + """Pool/waiter VA -> PA via tracker-sem -> page -> mem; NO content match. + + The word at POOL base + 4*idx is a slot in a UVM semaphore-pool page: a + sysmem uvm_rm_mem whose PA is only reachable through its kernel CPU + alias (the client pagemap has no present entry for it). Same kcore walk + the CE pushbuffer uses; the chain is shorter (sem+0 = page, page+0 = + mem). The page sits 16 MB above the CE push buffer base, outside + resolve_sem_pins' default_pages window, which is why the host saw + 'pending release starved' for it on every client's first seconds. + """ + page = va & ~0xFFF + if not (POOL_SEM_LO <= page < POOL_SEM_HI): + return False + for sem in list(sems): + if not is_kern_ptr(sem): + continue + try: + pg = kc.read64(sem) + if not is_kern_ptr(pg): + continue + mem = kc.read64(pg) + if not is_kern_ptr(mem): + continue + got = [] + resolve_mem_pins(kc, mem, got, default_pages=1, + va_lo=page, va_hi=page + 0x1000) + except Exception: + continue + for line in got: + try: + lva = int(line.split(',')[0], 16) + except Exception: + continue + if lva == page: + out.append(line) + dbg('pool-va: 0x%x -> %s (mem=0x%x)' + % (page, line.strip(), mem)) + return True + dbg('pool-va: 0x%x unresolved (%d known sems)' % (page, len(sems))) + return False + + def resolve_pb_pins(kc, pb, out): mem = kc.read64(pb + 8) if not mem: @@ -767,6 +810,14 @@ def resolve_pb_pins(kc, pb, out): # hold the ring-entry pushes is chosen (content anchor, _mem_content_ok). PB_CE_LO = 0x120000000 PB_CE_HI = 0x121000000 +# m165 (D-F1a): UVM semaphore-pool page range. The pool is a 4 KB sysmem +# uvm_rm_mem placed by the UVM arena right after the 16 MB CE push buffer; +# the client's tracker words (0x121000000 + 4*idx) live in its first page. +# The client pagemap has no present entry for it, so only the kcore +# tracker-sem -> page -> mem walk can translate it. +POOL_SEM_LO = 0x121000000 +POOL_SEM_HI = 0x122000000 +KC_POOL_REEMIT_S = 2.0 # re-assert the pool page pair this often too KC_CHAIN_BACK = 0x2000 KC_CHAIN_SPAN = 0xA000 KC_CHAIN_MAX = 400 @@ -1569,6 +1620,11 @@ def main(): last_log = 0.0 scrub_next = 0.0 demand_next = 0.0 + # m165 (D-F1a): pool page VAs learned from demand, re-asserted every + # KC_POOL_REEMIT_S. Boot-stable (UVM frees pool pages only at GPU + # deinit), so the set deliberately persists across client switches. + pool_pages = {} + pool_reemit_next = 0.0 _want_state = {"mtime": None, "done": set()} loop = 0 while True: @@ -1669,16 +1725,49 @@ def main(): if wv in _want_state["done"]: continue got = [] - try: - ok = resolve_va_by_content(kc, wv, got) - except Exception: - ok = False - got = [] + if POOL_SEM_LO <= (wv & ~0xFFF) < POOL_SEM_HI: + # m165 (D-F1a): pool/waiter VAs have no method + # stream content and the CE offset math does not + # apply; route them to the structural tracker-sem + # walk instead. + try: + ok = resolve_pool_va(kc, wv, got, + seen_sems) + except Exception: + ok = False + got = [] + dbg('demand-pool-va: 0x%x %s' + % (wv, 'ok' if ok else 'pending')) + if ok and got: + pool_pages.setdefault(wv & ~0xFFF, True) + else: + try: + ok = resolve_va_by_content(kc, wv, got) + except Exception: + ok = False + got = [] if ok and got: lines.extend(got) _want_state["done"].add(wv) except OSError: pass + # m165 (D-F1a): re-assert the pool page pair every KC_POOL_REEMIT_S. + # The pool page is boot-stable (UVM frees it only at GPU deinit) but + # the host pin table is wiped at every session flush, so a one-shot + # pin leaves later clients starving until the slow structural walks + # answer; the periodic re-assert heals the wipe within seconds (the + # same argument as the CE mem sweep above). + if kc_ok and pool_pages and time.time() >= pool_reemit_next: + pool_reemit_next = time.time() + KC_POOL_REEMIT_S + for pv in list(pool_pages): + got = [] + try: + ok = resolve_pool_va(kc, pv, got, seen_sems) + except Exception: + ok = False + got = [] + if ok and got: + lines.extend(got) # PMA scrubber buffer discovery (throttled): resolve the finish/host- # semaphore word PAs and stream them for host registration. if kc_ok and time.time() >= scrub_next: diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index e1f5195..dff2d2d 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -145,7 +145,7 @@ def _demand_loop(): except Exception: continue now = time.time() - for m in re.finditer(r"stalled VA #\d+: (0x[0-9a-f]+)", + for m in re.finditer(r"(?:stalled VA #\d+|pending release starved va)=(0x[0-9a-f]+)", (r.stdout or "")[-200000:]): va = int(m.group(1), 16) track.setdefault(va, [now, 0.0]) @@ -258,14 +258,17 @@ def _flush(runs, seen): """ todo = [] for va, pa, n in runs: - # The CE window (0x120000000..0x121000000) is re-asserted by the + # The CE window (0x120000000..0x121000000) and the UVM semaphore + # pool page range (0x121000000..0x122000000) are re-asserted by the # guest every couple of seconds on purpose: the host's session flush # wipes the whole pin table when the previous client's last channel # closes, and a pin batch that raced in for the new client must be # allowed to land again even when the pair is unchanged. Skipping # repeats there stranded the second client's CE pushes (the r144/ - # r145 stall: sema-probe "NO ENTRY" for the live push pages). - ce_win = 0x120000000 <= va < 0x121000000 + # r145 stall: sema-probe "NO ENTRY" for the live push pages) and the + # pool/tracker page (pending release starved, every client's first + # seconds). + ce_win = 0x120000000 <= va < 0x122000000 if not ce_win and seen.get(va) == pa and n == 1: continue todo.append((va, pa, n)) -- cgit v1.2.3