From 02de599cd6150099141fd876ac378418855672d5 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Wed, 14 Oct 2026 23:41:59 -0400 Subject: guest-diag: heal session-flush pin wipes with periodic full CE re-assert The host wipes its whole sema map when the previous client's last channel closes ('session flush (all channels closed)'). A pin batch that raced in for the new client (observed: the 1-page 0x120000000 pin emitted in the same second as the flush) was lost, and the scanner's content-gated delta never re-emitted the unchanged pairs - so the second client's CE pushes were permanently unreadable (sema-probe 'NO ENTRY for page 0x120000000'), every GPFIFO entry was grace-skipped (r144/r145 matmul and mail stalls), and the guest spun in uvm_tracker_wait_for_entry. Fix (scanner, guest side): - _ce_mem_pages: walk the whole vmalloc area (minus header off), not the size-derived prefix that stopped one page short of the write frontier. - _emit_ce_mem_delta: full re-assert mode every KC_FULLREM_S (1.5 s) plus every fresh_map pass; emits every translatable pair content-independent, merged into contiguous runs. Zero-content pages are pinned on purpose: the pin carries the translation, the host reads live content. - missing pages retried with the negative PA cache popped (bounded 512/pass). Fix (host tool): - pin_gr_stream._flush: never dedupe single-page repeats of the CE window (0x120000000..0x121000000) so the re-asserts land. Live-validated in r145 (same-boot, no rebuild): mail green as 4th client, matmul green as 2nd client (bad=0, 64-changed acceptance of 857fe5f9), both with zero 'stalled VA'/'skipping stale entry' lines; pre-fix the same sequences stalled. Headless smiley all-wrong (bad=131072) reproduced independently and is the next target (engine-side constant class). --- tools/guest-diag/guest_pin_stream.py | 108 +++++++++++++++++++++++++---------- tools/guest-diag/pin_gr_stream.py | 10 +++- 2 files changed, 87 insertions(+), 31 deletions(-) (limited to 'tools/guest-diag') diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 10a9aba..7d07f55 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -403,9 +403,13 @@ def mem_page_off(kc, mem): def _ce_mem_pages(kc, mem): """CE mem walk parameters: (gpus, cpu, off, npages) or None. - Mirrors resolve_mem_pins' derivation so the delta sweeper emits the same - page set the one-shot resolver used: size at +0x128 when readable, else - the vmalloc area length; all CE-window GPU VAs share the CPU mapping. + Walks the same page set resolve_mem_pins used: the whole vmalloc area + (minus the header offset) when known, else size at +0x128, else the + default. The area governs: the live pushes are written sequentially and + may run past the size field, so a size-derived prefix stopped one page + short of the write frontier (r144: last page 0x120030000 emitted, the + live page 0x120031000 never in any pin set). all CE-window GPU VAs share + the CPU mapping. """ vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] cpu = next((v for v in vas if is_kern_ptr(v)), None) @@ -417,44 +421,80 @@ def _ce_mem_pages(kc, mem): off = mem_page_off(kc, mem) area = _vmalloc_pages_for(kc, cpu) size = _mem_size(kc, mem) - npages = (max(1, min(size >> 12, 4098)) if size - else area or 64) - if area and off + npages > area: - npages = max(1, area - off) + size_pages = max(1, min(size >> 12, 4098)) if size else 0 + area_pages = area - off if area and area > off else 0 + npages = min(max(size_pages, area_pages) or 64, 4098) return (gpus, cpu, off, npages) def _emit_ce_mem_delta(kc, st, out, fresh_map=False): - """Emit the CE mem's live {va,pa} pairs as content-validated runs. - - The one-shot emission this replaces pinned the whole 16 MB window - including pages whose content was zero at the time and pages whose vmap - pairing had drifted; the host then read zeros for the other CE ranks' - pushes (the second-client stall class). The sweep now skips zero pages - (a zero page cannot carry a push), re-derives every PA against the live - page tables (optionally bursting the Kcore cache first), and emits only - pairs that are new or changed since the last sweep - so a page the - client writes later, or a mem whose vmap pages moved, is re-emitted - within KC_REEMIT_S and the host's last-store-wins pin sees the truth. - Never drops a previously emitted pair (a stale pin is corrected by the - changed pair, and a failed burst never clears the whole map). + """Emit the CE mem's live {va,pa} pairs as merged runs. + + Two emission modes share one merge step: + + * full re-assert (KC_FULLREM_S cadence, and every fresh_map pass): every + page that translates is emitted, content-independent. This is the + fix for the host's session flush: at a client switch the host wipes + its whole sema map when the previous client's last channel closes + ("session flush (all channels closed)"), and a pin batch that raced + in for the new client is lost. The old content-gated delta never + re-emitted those unchanged pairs, so the new client's CE pushes were + permanently unreadable ("NO ENTRY" in sema-probe) and every GPFIFO + entry was skipped after the 8 s grace (r144/r145 second-client stall, + both mail and matmul). Re-asserting the full pair set every couple of + seconds heals any number of flush wipes; merged contiguous runs keep + the traffic to a few lines per pass. Zero-content pages are pinned + deliberately: the pin carries the translation, the host reads the + live content, and the client writes it later. + + * delta (between full passes): content-gated, emits pages whose PA is + new or changed, so a remap or a freshly-faulted page is picked up + without waiting for the next full pass. Never drops a previously + emitted pair (the host's last-store-wins pin sees the truth, and a + failed burst never clears the whole map). """ gpus, cpu, off, npages = st["gpus"], st["cpu"], st["page_off"], st["npages"] last = st["last"] + nowm = time.monotonic() + full = nowm >= st.get("full_next", 0.0) + if full: + st["full_next"] = nowm + KC_FULLREM_S if fresh_map: for k in range(npages): kc._pa_cache.pop(cpu + (k + off) * 0x1000, None) changed = [] - for k in range(npages): - d = kc.read(cpu + (k + off) * 0x1000, 32) - if not d or not any(d): - continue # zero/unreadable: nothing to pin yet - pa = kc.va_to_pa(cpu + (k + off) * 0x1000) - if pa is None: - continue - for gv in gpus: - if last.get(gv + k * 0x1000) != pa: + if full: + # Re-walk pages that did not translate on an earlier pass: the + # vmap's leaf sizes are huge (2 MB/1 GB), so most pages translate + # immediately; a straggler (the vmap populated lazily) is retried + # with its negative PA-cache entry popped, bounded per pass so the + # sweep stays cheap. + miss = st.setdefault("miss", set()) + retry = 0 + for k in range(npages): + page = cpu + (k + off) * 0x1000 + pa = kc.va_to_pa(page) + if pa is None: + if k in miss and retry < 512: + kc._pa_cache.pop(page, None) + retry += 1 + else: + miss.add(k) + continue + miss.discard(k) + for gv in gpus: changed.append((gv + k * 0x1000, pa)) + else: + for k in range(npages): + d = kc.read(cpu + (k + off) * 0x1000, 32) + if not d or not any(d): + continue # zero/unreadable: nothing to pin yet + pa = kc.va_to_pa(cpu + (k + off) * 0x1000) + if pa is None: + continue + for gv in gpus: + if last.get(gv + k * 0x1000) != pa: + changed.append((gv + k * 0x1000, pa)) changed.sort() merged = [] for va, pa in changed: @@ -468,6 +508,8 @@ def _emit_ce_mem_delta(kc, st, out, fresh_map=False): for va, pa in changed: last[va] = pa st["emit_total"] = st.get("emit_total", 0) + len(merged) + if full: + st["full_total"] = st.get("full_total", 0) + 1 return len(merged) @@ -632,8 +674,14 @@ KC_CHAIN_MAX = 400 RESOLVED_KC_PB = set() KC_PB_STATE = {} # sem kptr -> resolved-mem state (fingerprint + pins) KC_REFRESH_S = 2.0 # re-validate a resolved CE mem at most this often -KC_REEMIT_S = 2.0 # delta-sweep the CE mem's live pages this often +KC_REEMIT_S = 2.0 # sweep the CE mem's live pages this often KC_REMAP_S = 30.0 # full re-translation sweep (catches remaps) +# Full re-assert cadence: every pass at least this far apart re-emits the +# whole translatable pair set regardless of the delta, healing host-side +# session-flush wipes (see _emit_ce_mem_delta). Must stay well under the +# host's gsp_sema_ce_grace_ms (8 s) so a wiped-and-needed page is restored +# inside the hold window. +KC_FULLREM_S = 1.5 def _chain_mems_at(kc, w): diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index 2a5d768..c0a8c9d 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -205,7 +205,15 @@ def _flush(runs, seen): """ todo = [] for va, pa, n in runs: - if seen.get(va) == pa and n == 1: + # The CE window (0x120000000..0x121000000) is re-asserted by the + # guest every couple of seconds on purpose: the host's session flush + # wipes the whole pin table when the previous client's last channel + # closes, and a pin batch that raced in for the new client must be + # allowed to land again even when the pair is unchanged. Skipping + # repeats there stranded the second client's CE pushes (the r144/ + # r145 stall: sema-probe "NO ENTRY" for the live push pages). + ce_win = 0x120000000 <= va < 0x121000000 + if not ce_win and seen.get(va) == pa and n == 1: continue todo.append((va, pa, n)) if not todo: -- cgit v1.2.3