diff options
Diffstat (limited to 'tools/guest-diag')
| -rw-r--r-- | tools/guest-diag/guest_pin_stream.py | 253 | ||||
| -rw-r--r-- | tools/guest-diag/pin_gr_stream.py | 33 |
2 files changed, 276 insertions, 10 deletions
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 49974c9..7d07f55 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -400,6 +400,119 @@ def mem_page_off(kc, mem): return 0 +def _ce_mem_pages(kc, mem): + """CE mem walk parameters: (gpus, cpu, off, npages) or None. + + Walks the same page set resolve_mem_pins used: the whole vmalloc area + (minus the header offset) when known, else size at +0x128, else the + default. The area governs: the live pushes are written sequentially and + may run past the size field, so a size-derived prefix stopped one page + short of the write frontier (r144: last page 0x120030000 emitted, the + live page 0x120031000 never in any pin set). all CE-window GPU VAs share + the CPU mapping. + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpus = [v for v in vas if v and PB_CE_LO <= v < PB_CE_HI] + if not cpu or not gpus: + return None + if _direct_map_pa(kc, cpu) is not None: + return None # never a vmap: not the push-buffer shape + off = mem_page_off(kc, mem) + area = _vmalloc_pages_for(kc, cpu) + size = _mem_size(kc, mem) + size_pages = max(1, min(size >> 12, 4098)) if size else 0 + area_pages = area - off if area and area > off else 0 + npages = min(max(size_pages, area_pages) or 64, 4098) + return (gpus, cpu, off, npages) + + +def _emit_ce_mem_delta(kc, st, out, fresh_map=False): + """Emit the CE mem's live {va,pa} pairs as merged runs. + + Two emission modes share one merge step: + + * full re-assert (KC_FULLREM_S cadence, and every fresh_map pass): every + page that translates is emitted, content-independent. This is the + fix for the host's session flush: at a client switch the host wipes + its whole sema map when the previous client's last channel closes + ("session flush (all channels closed)"), and a pin batch that raced + in for the new client is lost. The old content-gated delta never + re-emitted those unchanged pairs, so the new client's CE pushes were + permanently unreadable ("NO ENTRY" in sema-probe) and every GPFIFO + entry was skipped after the 8 s grace (r144/r145 second-client stall, + both mail and matmul). Re-asserting the full pair set every couple of + seconds heals any number of flush wipes; merged contiguous runs keep + the traffic to a few lines per pass. Zero-content pages are pinned + deliberately: the pin carries the translation, the host reads the + live content, and the client writes it later. + + * delta (between full passes): content-gated, emits pages whose PA is + new or changed, so a remap or a freshly-faulted page is picked up + without waiting for the next full pass. Never drops a previously + emitted pair (the host's last-store-wins pin sees the truth, and a + failed burst never clears the whole map). + """ + gpus, cpu, off, npages = st["gpus"], st["cpu"], st["page_off"], st["npages"] + last = st["last"] + nowm = time.monotonic() + full = nowm >= st.get("full_next", 0.0) + if full: + st["full_next"] = nowm + KC_FULLREM_S + if fresh_map: + for k in range(npages): + kc._pa_cache.pop(cpu + (k + off) * 0x1000, None) + changed = [] + if full: + # Re-walk pages that did not translate on an earlier pass: the + # vmap's leaf sizes are huge (2 MB/1 GB), so most pages translate + # immediately; a straggler (the vmap populated lazily) is retried + # with its negative PA-cache entry popped, bounded per pass so the + # sweep stays cheap. + miss = st.setdefault("miss", set()) + retry = 0 + for k in range(npages): + page = cpu + (k + off) * 0x1000 + pa = kc.va_to_pa(page) + if pa is None: + if k in miss and retry < 512: + kc._pa_cache.pop(page, None) + retry += 1 + else: + miss.add(k) + continue + miss.discard(k) + for gv in gpus: + changed.append((gv + k * 0x1000, pa)) + else: + for k in range(npages): + d = kc.read(cpu + (k + off) * 0x1000, 32) + if not d or not any(d): + continue # zero/unreadable: nothing to pin yet + pa = kc.va_to_pa(cpu + (k + off) * 0x1000) + if pa is None: + continue + for gv in gpus: + if last.get(gv + k * 0x1000) != pa: + changed.append((gv + k * 0x1000, pa)) + changed.sort() + merged = [] + for va, pa in changed: + if merged and va == merged[-1][0] + merged[-1][2] * 0x1000 and \ + pa == merged[-1][1] + merged[-1][2] * 0x1000: + merged[-1][2] += 1 + else: + merged.append([va, pa, 1]) + for va, pa, n in merged: + out.append("0x%x,0x%x,%d\n" % (va, pa, n)) + for va, pa in changed: + last[va] = pa + st["emit_total"] = st.get("emit_total", 0) + len(merged) + if full: + st["full_total"] = st.get("full_total", 0) + 1 + return len(merged) + + def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out): """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT buffer usually sits) from the ring's CPU mapping.""" @@ -559,8 +672,16 @@ KC_CHAIN_BACK = 0x2000 KC_CHAIN_SPAN = 0xA000 KC_CHAIN_MAX = 400 RESOLVED_KC_PB = set() -KC_PB_STATE = {} # sem kptr -> resolved-mem fingerprint (freshness) +KC_PB_STATE = {} # sem kptr -> resolved-mem state (fingerprint + pins) KC_REFRESH_S = 2.0 # re-validate a resolved CE mem at most this often +KC_REEMIT_S = 2.0 # sweep the CE mem's live pages this often +KC_REMAP_S = 30.0 # full re-translation sweep (catches remaps) +# Full re-assert cadence: every pass at least this far apart re-emits the +# whole translatable pair set regardless of the delta, healing host-side +# session-flush wipes (see _emit_ce_mem_delta). Must stay well under the +# host's gsp_sema_ce_grace_ms (8 s) so a wiped-and-needed page is restored +# inside the hold window. +KC_FULLREM_S = 1.5 def _chain_mems_at(kc, w): @@ -743,18 +864,23 @@ def resolve_pb_chain(kc, sem, out): if mem in RESOLVED_KC_PB: return True RESOLVED_KC_PB.add(mem) - off = mem_page_off(kc, mem) - n = resolve_mem_pins(kc, mem, out, default_pages=64, - va_lo=PB_CE_LO, va_hi=PB_CE_HI, - page_off=off) - st = {"mem": mem, "gpu": gpu, "cpu": cpu, - "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, "page_off": 0} + params = _ce_mem_pages(kc, mem) + if not params: + return found + gpus, cpu, off, npages = params + st = {"mem": mem, "gpu": gpu, "cpu": cpu, "gpus": gpus, + "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, + "page_off": off, "npages": npages, "last": {}} + # Fresh translation for the first sweep: the Kcore PA cache may hold + # pairs from before this mem's vmap area was (re)populated. + kc._pa_cache.clear() + n = _emit_ce_mem_delta(kc, st, out) KC_PB_STATE[sem] = st try: with open('/tmp/guest_pin_stream.log', 'a') as f: f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x ' - 'pin-runs=%d off=%d size=%d score=%d\n' - % (time.strftime('%T'), mem, sem, n, off, + 'pin-runs=%d off=%d npages=%d size=%d score=%d\n' + % (time.strftime('%T'), mem, sem, n, off, npages, _mem_size(kc, mem), _score)) except OSError: pass @@ -1021,6 +1147,75 @@ def new_sems(seen): return found +# --------------------------------------------------------------- scrubber +# PMA scrubber channel discovery (third-client scrub freeze). The guest's +# PMA scrubber is an internal CeUtils CE channel (4096 GPFIFO entries, +# pbGpuVA 0x120000000, ring at pbGpuVA+0x64000, finish payload at +# pbGpuVA+0x6c004). It lives in a kernel vmap area, not in any user pagemap, +# and its GPU VAs collide numerically with the client CE push-buffer window +# in the host's global translation map - so the host cannot resolve it and +# must be told the finish/host-semaphore word PAs directly. Resolve them +# here (vmallocinfo + manual kernel page-table walk) and emit one +# #scrub <fin_pa>,<hsem_pa> +# line; pin_gr_stream.py registers them through sema_scrubbuf_add. +SCRUB_AREA_SIZES = (0x6e000, 0x6d000, 0x6c000, 0x6b000, 0x6f000, + 0x70000, 0x71000, 0x72000) +_scrub_sent = {} + + +def _scrub_ring_ok(kc, ring_pa): + """A 906F GPFIFO ring page: all-zero (fresh) or entries whose GET names a + push inside the channel's own 0x120000000 window (encoded as + push_va - 0x100000000, 4-byte aligned).""" + buf = kc.read(kc.page_offset_base + ring_pa, 0x1000) + if not buf: + return False + for off in range(0, 0x100, 8): + e0, _e1 = struct.unpack_from('<II', buf, off) + if e0 == 0: + continue + if not (0x20000000 <= e0 < 0x20080000) or (e0 & 3): + return False + return True + + +def scrub_scan(kc, lines): + """Emit the scrubber buffer words' PAs once (re-emitted if they move).""" + try: + f = open('/proc/vmallocinfo') + except OSError: + return + with f: + for line in f: + if 'nvidia' not in line or 'ioremap' in line or 'vmap' not in line: + continue + parts = line.split() + try: + a, b = parts[0].split('-') + lo, hi = int(a, 16), int(b, 16) + except (ValueError, IndexError): + continue + if (hi - lo) not in SCRUB_AREA_SIZES: + continue + fin_pa = kc.va_to_pa(lo + 0x6c000) + ring_pa = kc.va_to_pa(lo + 0x64000) + if fin_pa is None or ring_pa is None: + continue + if not _scrub_ring_ok(kc, ring_pa): + dbg('scrub candidate 0x%x rejected (ring shape)' % lo) + continue + fin = fin_pa + 4 + hsem = fin_pa + key = (fin, hsem) + if _scrub_sent.get('key') == key: + return + _scrub_sent['key'] = key + lines.append('#scrub 0x%x,0x%x\n' % (fin, hsem)) + dbg('scrub buffer emitted lo=0x%x fin=0x%x hsem=0x%x' + % (lo, fin, hsem)) + return + + def nvidia_fds(pid): """CUDA device fds this pid holds: 'uvm', 'gpu', or None. @@ -1252,6 +1447,8 @@ def main(): resolved_kc_pb = set() retry_at = {} kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} tracer = TraceReader() kc = Kcore() kc_ok = kc.load() @@ -1263,6 +1460,7 @@ def main(): pass last_pid = None last_log = 0.0 + scrub_next = 0.0 loop = 0 while True: pid = cdt_pid() @@ -1288,6 +1486,8 @@ def main(): resolved_kc_pb = set() retry_at = {} kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} # Client switch: the old client's channel/put pointers are freed # memory. Clearing the #put map stops stale reads from being # emitted as if they were the new client's counters. @@ -1339,6 +1539,14 @@ def main(): new = [(va, pa) for va, pa in cur.items() if seen.get(va) != pa] pending_cur = cur lines = [] + # PMA scrubber buffer discovery (throttled): resolve the finish/host- + # semaphore word PAs and stream them for host registration. + if kc_ok and time.time() >= scrub_next: + scrub_next = time.time() + 5.0 + try: + scrub_scan(kc, lines) + except Exception as e: + dbg('scrub scan failed: %s' % e) for ctl in tracer.drain(): parts = ctl.split() if ctl.startswith("#sem") and len(parts) >= 2: @@ -1414,6 +1622,33 @@ def main(): kc_refresh_at[sem] = now_r + 5.0 dbg("kc-chain stale: keep old latch sem=0x%x" % sem) + else: + # CE mem delta sweep (second-client pins): the one-shot + # emission pinned the mem once; pages the client writes + # later (or whose vmap pairing moved) were never + # re-emitted, so the host kept zeros for the other CE + # ranks' pushes. Sweep every KC_REEMIT_S, emitting only + # new/changed nonzero pairs; burst the Kcore PA cache + # every KC_REMAP_S so a remap is not shadowed. + _st = KC_PB_STATE.get(sem) + if _st is not None and \ + now_r >= kc_reemit_at.get(sem, 0): + _fresh_map = now_r >= kc_remap_at.get(sem, 0) + if _fresh_map: + kc_remap_at[sem] = now_r + KC_REMAP_S + kc_reemit_at[sem] = now_r + KC_REEMIT_S + got = [] + try: + _n2 = _emit_ce_mem_delta(kc, _st, got, + fresh_map=_fresh_map) + except Exception: + _n2 = 0 + got = [] + if got: + lines.extend(got) + added += len(got) + dbg("ce-delta sem=0x%x runs=%d mem=0x%x" + % (sem, _n2, _st.get("mem", 0))) # CE push-buffer chain (second-client stall fix): resolve # pool -> manager -> pushbuffer -> rm_mem directly, so the # push-buffer pages never depend on the racy one-shot uvbp diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index b864ab5..c0a8c9d 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -101,6 +101,22 @@ def inject(va, pa, ln): return False +def scrub_register(fin_pa, hsem_pa): + """Register the PMA scrubber buffer words (guest-resolved PAs) with the + service so it can author the scrub progress (third-client scrub freeze).""" + cmd = ("printf '0x%x,0x%x' > %s/sema_scrubbuf_add" + % (fin_pa, hsem_pa, PARAM)) + for _ in range(60): + r = subprocess.run(["sudo", "-n", "sh", "-c", cmd], + capture_output=True, text=True) + if r.returncode == 0: + log("scrub registered fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa)) + return True + time.sleep(0.05) + log("scrub register FAILED fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa)) + return False + + def main(): if not install_and_start(): log("WARN: could not start guest scanner (will retry via vsock wait)") @@ -158,6 +174,13 @@ def main(): log(text) _mirror(text) continue + if text.startswith("#scrub"): + m = re.search(r"0x([0-9a-fA-F]+),0x([0-9a-fA-F]+)", text) + _mirror(text) + if m: + scrub_register(int(m.group(1), 16), + int(m.group(2), 16)) + continue m = re.match(r"^(0x[0-9a-f]+),(0x[0-9a-f]+)(?:,(\d+))?$", text) if m: runs.append((int(m.group(1), 16), int(m.group(2), 16), @@ -182,7 +205,15 @@ def _flush(runs, seen): """ todo = [] for va, pa, n in runs: - if seen.get(va) == pa and n == 1: + # The CE window (0x120000000..0x121000000) is re-asserted by the + # guest every couple of seconds on purpose: the host's session flush + # wipes the whole pin table when the previous client's last channel + # closes, and a pin batch that raced in for the new client must be + # allowed to land again even when the pair is unchanged. Skipping + # repeats there stranded the second client's CE pushes (the r144/ + # r145 stall: sema-probe "NO ENTRY" for the live push pages). + ce_win = 0x120000000 <= va < 0x121000000 + if not ce_win and seen.get(va) == pa and n == 1: continue todo.append((va, pa, n)) if not todo: |