summaryrefslogtreecommitdiffstats
path: root/tools/guest-diag
diff options
context:
space:
mode:
Diffstat (limited to 'tools/guest-diag')
-rw-r--r--tools/guest-diag/guest_pin_stream.py606
-rw-r--r--tools/guest-diag/pin_gr_stream.py101
-rw-r--r--tools/guest-diag/semfix_watch.py38
3 files changed, 718 insertions, 27 deletions
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index c0cf485..4bfaaf8 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -266,6 +266,103 @@ def _vmalloc_pages_for(kc, va):
return 0
+def _nv_vmap_areas(kc):
+ """nv_vm_map_pages vmalloc areas: [(lo, hi, pages)] (5 s cache).
+
+ Every CE push-buffer rm_mem is a vmap created by nv_vm_map_pages; the
+ demand pinner walks these areas to find the page holding a stalled
+ push VA by content, without needing any kprobe event or object chain.
+ """
+ now = time.time()
+ if getattr(_nv_vmap_areas, "ts", 0) > now - 5.0:
+ return _nv_vmap_areas.areas
+ areas = []
+ try:
+ for line in open("/proc/vmallocinfo"):
+ if "nv_vm_map_pages" not in line:
+ continue
+ p = line.split()
+ if len(p) < 2:
+ continue
+ try:
+ lo, hi = p[0].split("-")
+ lo = int(lo, 16)
+ hi = int(hi, 16)
+ except ValueError:
+ continue
+ pages = (hi - lo) >> 12
+ if pages > 0:
+ areas.append((lo, hi, pages))
+ except OSError:
+ pass
+ _nv_vmap_areas.areas = areas
+ _nv_vmap_areas.ts = now
+ return areas
+
+
+# Learned page-offset per nv_vm_map_pages area for the demand content match
+# (2026-10-16b): the first successful resolution records which page index the
+# VA maps to, and later pushes in the same allocation retry that offset first
+# instead of walking the whole +/-16 neighbourhood.
+_area_off = {}
+
+
+def resolve_va_by_content(kc, va, out):
+ """Demand pin: find the page carrying GPU VA 'va' by content match.
+
+ The host reports the CE push-buffer VAs whose GPFIFO entries it cannot
+ read (dmesg 'stalled VA #N: 0x...'). The guest walks every
+ nv_vm_map_pages vmap area and tests the candidate leaf page at the VA's
+ in-window offset (plus a neighbourhood, for the per-mem header-page
+ offset ambiguity) against the host's method-stream prefix check. On a
+ match it emits the exact pair, so the host can translate and consume the
+ entry - no kprobe event, no object-chain discovery, no resolution
+ lottery. Only the requested VA is pinned, so a wrong candidate cannot
+ poison other VAs.
+
+ 2026-10-16b: the offset search window was +/-1 page. Pushes of a client
+ whose pushbuffer vmap carries a larger per-allocation header (observed
+ with multi-MB CE staging: pushes at 0x120000f58+ never resolved and the
+ demand loop re-sent the same VA for minutes while the client crawled)
+ fall outside it. Widen to +/-16 pages and LEARN the winning offset per
+ vmap area, because consecutive pushes advance sequentially inside the
+ same allocation and share the offset.
+ """
+ if not (PB_CE_LO <= va < PB_CE_HI):
+ return False
+ off = (va - PB_CE_LO) & 0xFFFFFF
+ k = off >> 12
+ areas = _nv_vmap_areas(kc)
+ for (lo, _hi, pages) in areas:
+ learned = _area_off.get(lo)
+ if learned is not None:
+ adjs = [learned]
+ else:
+ adjs = [0, 1, -1]
+ adjs += [d for m in (2, 4, 8, 16) for d in (m, -m)]
+ for adj in adjs:
+ idx = k + adj
+ if idx < 0 or idx >= pages:
+ continue
+ cpu = lo + idx * 0x1000
+ d = kc.read(cpu, 64)
+ if not d or not any(d[:32]):
+ continue
+ if not _push_prefix_ok(d):
+ continue
+ pa = kc.va_to_pa(cpu)
+ if pa is None:
+ continue
+ _area_off[lo] = adj
+ out.append("0x%x,0x%x,1\n" % (va & ~0xFFF, pa))
+ dbg("demand-va: 0x%x -> pa=0x%x (area off=%d adj=%+d)"
+ % (va & ~0xFFF, pa, idx, adj))
+ return True
+ dbg("demand-va: 0x%x not found by content (%d areas)"
+ % (va & ~0xFFF, len(areas)))
+ return False
+
+
def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0,
page_off=0):
"""uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk).
@@ -400,6 +497,121 @@ def mem_page_off(kc, mem):
return 0
+def _ce_mem_pages(kc, mem):
+ """CE mem walk parameters: (gpus, cpu, off, npages) or None.
+
+ Walks the same page set resolve_mem_pins used: the whole vmalloc area
+ (minus the header offset) when known, else size at +0x128, else the
+ default. The area governs: the live pushes are written sequentially and
+ may run past the size field, so a size-derived prefix stopped one page
+ short of the write frontier (r144: last page 0x120030000 emitted, the
+ live page 0x120031000 never in any pin set). all CE-window GPU VAs share
+ the CPU mapping.
+ """
+ vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)]
+ cpu = next((v for v in vas if is_kern_ptr(v)), None)
+ gpus = [v for v in vas if v and PB_CE_LO <= v < PB_CE_HI]
+ if not cpu or not gpus:
+ return None
+ if _direct_map_pa(kc, cpu) is not None:
+ return None # never a vmap: not the push-buffer shape
+ off = mem_page_off(kc, mem)
+ area = _vmalloc_pages_for(kc, cpu)
+ size = _mem_size(kc, mem)
+ size_pages = max(1, min(size >> 12, 4098)) if size else 0
+ area_pages = area - off if area and area > off else 0
+ npages = min(max(size_pages, area_pages) or 64, 4098)
+ return (gpus, cpu, off, npages)
+
+
+def _emit_ce_mem_delta(kc, st, out, fresh_map=False):
+ """Emit the CE mem's live {va,pa} pairs as merged runs.
+
+ Two emission modes share one merge step:
+
+ * full re-assert (KC_FULLREM_S cadence, and every fresh_map pass): every
+ page that translates is emitted, content-independent. This is the
+ fix for the host's session flush: at a client switch the host wipes
+ its whole sema map when the previous client's last channel closes
+ ("session flush (all channels closed)"), and a pin batch that raced
+ in for the new client is lost. The old content-gated delta never
+ re-emitted those unchanged pairs, so the new client's CE pushes were
+ permanently unreadable ("NO ENTRY" in sema-probe) and every GPFIFO
+ entry was skipped after the 8 s grace (r144/r145 second-client stall,
+ both mail and matmul). Re-asserting the full pair set every couple of
+ seconds heals any number of flush wipes; merged contiguous runs keep
+ the traffic to a few lines per pass. Zero-content pages are pinned
+ deliberately: the pin carries the translation, the host reads the
+ live content, and the client writes it later.
+
+ * delta (between full passes): content-gated, emits pages whose PA is
+ new or changed, so a remap or a freshly-faulted page is picked up
+ without waiting for the next full pass. Never drops a previously
+ emitted pair (the host's last-store-wins pin sees the truth, and a
+ failed burst never clears the whole map).
+ """
+ gpus, cpu, off, npages = st["gpus"], st["cpu"], st["page_off"], st["npages"]
+ last = st["last"]
+ nowm = time.monotonic()
+ full = nowm >= st.get("full_next", 0.0)
+ if full:
+ age = nowm - st.get("resolved_at", nowm)
+ st["full_next"] = nowm + (KC_FULLREM_S if age < KC_FULLREM_FAST_S
+ else KC_FULLREM_SLOW_S)
+ if fresh_map:
+ for k in range(npages):
+ kc._pa_cache.pop(cpu + (k + off) * 0x1000, None)
+ changed = []
+ if full:
+ # Re-walk pages that did not translate on an earlier pass: the
+ # vmap's leaf sizes are huge (2 MB/1 GB), so most pages translate
+ # immediately; a straggler (the vmap populated lazily) is retried
+ # with its negative PA-cache entry popped, bounded per pass so the
+ # sweep stays cheap.
+ miss = st.setdefault("miss", set())
+ retry = 0
+ for k in range(npages):
+ page = cpu + (k + off) * 0x1000
+ pa = kc.va_to_pa(page)
+ if pa is None:
+ if k in miss and retry < 512:
+ kc._pa_cache.pop(page, None)
+ retry += 1
+ else:
+ miss.add(k)
+ continue
+ miss.discard(k)
+ for gv in gpus:
+ changed.append((gv + k * 0x1000, pa))
+ else:
+ for k in range(npages):
+ d = kc.read(cpu + (k + off) * 0x1000, 32)
+ if not d or not any(d):
+ continue # zero/unreadable: nothing to pin yet
+ pa = kc.va_to_pa(cpu + (k + off) * 0x1000)
+ if pa is None:
+ continue
+ for gv in gpus:
+ if last.get(gv + k * 0x1000) != pa:
+ changed.append((gv + k * 0x1000, pa))
+ changed.sort()
+ merged = []
+ for va, pa in changed:
+ if merged and va == merged[-1][0] + merged[-1][2] * 0x1000 and \
+ pa == merged[-1][1] + merged[-1][2] * 0x1000:
+ merged[-1][2] += 1
+ else:
+ merged.append([va, pa, 1])
+ for va, pa, n in merged:
+ out.append("0x%x,0x%x,%d\n" % (va, pa, n))
+ for va, pa in changed:
+ last[va] = pa
+ st["emit_total"] = st.get("emit_total", 0) + len(merged)
+ if full:
+ st["full_total"] = st.get("full_total", 0) + 1
+ return len(merged)
+
+
def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out):
"""Pin the GPFIFO ring pages (and the pages after it, where the GPPUT
buffer usually sits) from the ring's CPU mapping."""
@@ -522,6 +734,49 @@ def resolve_sem_pins(kc, sem, out):
return resolve_mem_pins(kc, mem, out, default_pages=16)
+def resolve_pool_va(kc, va, out, sems):
+ """Pool/waiter VA -> PA via tracker-sem -> page -> mem; NO content match.
+
+ The word at POOL base + 4*idx is a slot in a UVM semaphore-pool page: a
+ sysmem uvm_rm_mem whose PA is only reachable through its kernel CPU
+ alias (the client pagemap has no present entry for it). Same kcore walk
+ the CE pushbuffer uses; the chain is shorter (sem+0 = page, page+0 =
+ mem). The page sits 16 MB above the CE push buffer base, outside
+ resolve_sem_pins' default_pages window, which is why the host saw
+ 'pending release starved' for it on every client's first seconds.
+ """
+ page = va & ~0xFFF
+ if not (POOL_SEM_LO <= page < POOL_SEM_HI):
+ return False
+ for sem in list(sems):
+ if not is_kern_ptr(sem):
+ continue
+ try:
+ pg = kc.read64(sem)
+ if not is_kern_ptr(pg):
+ continue
+ mem = kc.read64(pg)
+ if not is_kern_ptr(mem):
+ continue
+ got = []
+ resolve_mem_pins(kc, mem, got, default_pages=1,
+ va_lo=page, va_hi=page + 0x1000)
+ except Exception:
+ continue
+ for line in got:
+ try:
+ lva = int(line.split(',')[0], 16)
+ except Exception:
+ continue
+ if lva == page:
+ out.append(line)
+ dbg('pool-va: 0x%x -> %s (mem=0x%x)'
+ % (page, line.strip(), mem))
+ return True
+ dbg('pool-va: 0x%x unresolved (%d known sems)' % (page, len(sems)))
+ return False
+
+
def resolve_pb_pins(kc, pb, out):
mem = kc.read64(pb + 8)
if not mem:
@@ -555,12 +810,35 @@ def resolve_pb_pins(kc, pb, out):
# hold the ring-entry pushes is chosen (content anchor, _mem_content_ok).
PB_CE_LO = 0x120000000
PB_CE_HI = 0x121000000
+# m165 (D-F1a): UVM semaphore-pool page range. The pool is a 4 KB sysmem
+# uvm_rm_mem placed by the UVM arena right after the 16 MB CE push buffer;
+# the client's tracker words (0x121000000 + 4*idx) live in its first page.
+# The client pagemap has no present entry for it, so only the kcore
+# tracker-sem -> page -> mem walk can translate it.
+POOL_SEM_LO = 0x121000000
+POOL_SEM_HI = 0x122000000
+KC_POOL_REEMIT_S = 2.0 # re-assert the pool page pair this often too
KC_CHAIN_BACK = 0x2000
KC_CHAIN_SPAN = 0xA000
KC_CHAIN_MAX = 400
RESOLVED_KC_PB = set()
-KC_PB_STATE = {} # sem kptr -> resolved-mem fingerprint (freshness)
+KC_PB_STATE = {} # sem kptr -> resolved-mem state (fingerprint + pins)
KC_REFRESH_S = 2.0 # re-validate a resolved CE mem at most this often
+KC_REEMIT_S = 2.0 # sweep the CE mem's live pages this often
+KC_REMAP_S = 30.0 # full re-translation sweep (catches remaps)
+# Full re-assert cadence: every pass at least this far apart re-emits the
+# whole translatable pair set regardless of the delta, healing host-side
+# session-flush wipes (see _emit_ce_mem_delta). Must stay well under the
+# host's gsp_sema_ce_grace_ms (8 s) so a wiped-and-needed page is restored
+# inside the hold window. The fast cadence applies to a mem's first
+# KC_FULLREM_FAST_S seconds (the flush race lives at the client switch);
+# after that the re-assert backs off to KC_FULLREM_SLOW_S - a fully
+# fragmented pair set (descending PAs) emits one page per line, and 4 k
+# lines every 2 s saturated a guest under multi-client load (r148: load 19,
+# three scanners from an install race, clients wedged).
+KC_FULLREM_S = float(os.environ.get("STREAM_FULLREM_S", "1.5"))
+KC_FULLREM_FAST_S = float(os.environ.get("STREAM_FULLREM_FAST_S", "45"))
+KC_FULLREM_SLOW_S = float(os.environ.get("STREAM_FULLREM_SLOW_S", "10.0"))
def _chain_mems_at(kc, w):
@@ -639,6 +917,16 @@ def _kc_pb_fresh(kc, sem):
return False
if not st.get("page_va"):
return True
+ if not st.get("page_pa"):
+ # Unknown PA at store time: the manual page-table walk could not
+ # translate the vmap page (it may have been read before the kernel
+ # page tables were walkable). Treat the record as fresh instead of
+ # failing every check - a permanently-failing freshness test made
+ # the scanner drop the latch and re-resolve in a tight loop, and
+ # each failed re-resolution left the host with no pins at all
+ # (observed r131: "kc-chain stale: re-resolving" every second for
+ # five sems while smiley sat in cudaMalloc).
+ return True
pa = kc.va_to_pa(st["page_va"])
if pa is None or pa != st.get("page_pa"):
return False
@@ -669,6 +957,42 @@ def resolve_pb_chain(kc, sem, out):
return False
words = struct.unpack_from("<%dQ" % (len(data) // 8), data, 0)
cands = list(dict.fromkeys(w for w in words if is_kern_ptr(w)))
+ # Channel-anchored expansion (r134 live dump): the tracking sem is a
+ # member of a uvm_channel_t table; the channel records sit in the sem's
+ # own window and carry their CE ring GPU VA ([0x121010000, 0x121400000))
+ # plus back-to-back 32-bit object handles. The pool/manager/pushbuffer
+ # pointers hang off those records' targets, which the bare sem-window
+ # scan never reaches - the second-client stall is exactly this: the only
+ # live uvmt event is the spinning sem, and no mem candidate is found
+ # around it. For each ring-VA word, scan the surrounding record for
+ # kptrs, take those targets, and try the chain shape on both. Additive
+ # and bounded; content scoring still decides, so a wrong candidate
+ # cannot shadow the live pages.
+ extra = []
+ for i, w in enumerate(words):
+ if not w or not (0x121010000 <= w < 0x121400000):
+ continue
+ lo = max(0, i - 0x60)
+ hi = min(len(words), i + 0x60)
+ for c in words[lo:hi]:
+ if not is_kern_ptr(c):
+ continue
+ extra.append(c)
+ tgt = kc.read(c, 0x100)
+ if tgt:
+ tw = struct.unpack_from("<%dQ" % (len(tgt) // 8),
+ tgt, 0)
+ for w2 in tw:
+ if is_kern_ptr(w2):
+ extra.append(w2)
+ if len(extra) >= 512:
+ break
+ if extra:
+ head = []
+ for w in extra:
+ if w not in head and w not in cands:
+ head.append(w)
+ cands = head + cands
mems = []
for w in cands[:KC_CHAIN_MAX]:
for mem in _chain_mems_at(kc, w):
@@ -697,18 +1021,24 @@ def resolve_pb_chain(kc, sem, out):
if mem in RESOLVED_KC_PB:
return True
RESOLVED_KC_PB.add(mem)
- off = mem_page_off(kc, mem)
- n = resolve_mem_pins(kc, mem, out, default_pages=64,
- va_lo=PB_CE_LO, va_hi=PB_CE_HI,
- page_off=off)
- st = {"mem": mem, "gpu": gpu, "cpu": cpu,
- "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, "page_off": 0}
+ params = _ce_mem_pages(kc, mem)
+ if not params:
+ return found
+ gpus, cpu, off, npages = params
+ st = {"mem": mem, "gpu": gpu, "cpu": cpu, "gpus": gpus,
+ "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0,
+ "page_off": off, "npages": npages, "last": {},
+ "resolved_at": time.monotonic()}
+ # Fresh translation for the first sweep: the Kcore PA cache may hold
+ # pairs from before this mem's vmap area was (re)populated.
+ kc._pa_cache.clear()
+ n = _emit_ce_mem_delta(kc, st, out)
KC_PB_STATE[sem] = st
try:
with open('/tmp/guest_pin_stream.log', 'a') as f:
f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x '
- 'pin-runs=%d off=%d size=%d score=%d\n'
- % (time.strftime('%T'), mem, sem, n, off,
+ 'pin-runs=%d off=%d npages=%d size=%d score=%d\n'
+ % (time.strftime('%T'), mem, sem, n, off, npages,
_mem_size(kc, mem), _score))
except OSError:
pass
@@ -807,6 +1137,38 @@ def resolve_channel_pins(kc, ch, out):
+def _read_trace_snapshot(max_bytes=1 << 20, budget=0.5):
+ """Best-effort non-blocking read of the trace ring file.
+
+ Spawning `tail` here left uninterruptible D-state processes when the
+ seq_file read blocked under load (observed r127: dozens of stuck tails,
+ load 26); a direct os.read with O_NONBLOCK returns whatever is ready and
+ never accumulates processes.
+ """
+ try:
+ fd = os.open(TRACE, os.O_RDONLY | os.O_NONBLOCK)
+ except OSError:
+ return b""
+ chunks = []
+ got = 0
+ t0 = time.time()
+ try:
+ while got < (8 << 20) and time.time() - t0 < budget:
+ try:
+ d = os.read(fd, 262144)
+ except BlockingIOError:
+ break
+ except OSError:
+ break
+ if not d:
+ break
+ chunks.append(d)
+ got += len(d)
+ finally:
+ os.close(fd)
+ return b"".join(chunks)[-max_bytes:]
+
+
def seed_events_from_trace(seen_sems, seen_pbs, seen_chs):
"""One-shot tail scan of the trace ring after a pid switch / start.
@@ -821,8 +1183,7 @@ def seed_events_from_trace(seen_sems, seen_pbs, seen_chs):
(the mem candidates are scored and deferred).
"""
try:
- out = subprocess.run(["tail", "-c", "1048576", TRACE],
- capture_output=True, timeout=10).stdout
+ out = _read_trace_snapshot()
except Exception:
return 0
n = 0
@@ -944,6 +1305,75 @@ def new_sems(seen):
return found
+# --------------------------------------------------------------- scrubber
+# PMA scrubber channel discovery (third-client scrub freeze). The guest's
+# PMA scrubber is an internal CeUtils CE channel (4096 GPFIFO entries,
+# pbGpuVA 0x120000000, ring at pbGpuVA+0x64000, finish payload at
+# pbGpuVA+0x6c004). It lives in a kernel vmap area, not in any user pagemap,
+# and its GPU VAs collide numerically with the client CE push-buffer window
+# in the host's global translation map - so the host cannot resolve it and
+# must be told the finish/host-semaphore word PAs directly. Resolve them
+# here (vmallocinfo + manual kernel page-table walk) and emit one
+# #scrub <fin_pa>,<hsem_pa>
+# line; pin_gr_stream.py registers them through sema_scrubbuf_add.
+SCRUB_AREA_SIZES = (0x6e000, 0x6d000, 0x6c000, 0x6b000, 0x6f000,
+ 0x70000, 0x71000, 0x72000)
+_scrub_sent = {}
+
+
+def _scrub_ring_ok(kc, ring_pa):
+ """A 906F GPFIFO ring page: all-zero (fresh) or entries whose GET names a
+ push inside the channel's own 0x120000000 window (encoded as
+ push_va - 0x100000000, 4-byte aligned)."""
+ buf = kc.read(kc.page_offset_base + ring_pa, 0x1000)
+ if not buf:
+ return False
+ for off in range(0, 0x100, 8):
+ e0, _e1 = struct.unpack_from('<II', buf, off)
+ if e0 == 0:
+ continue
+ if not (0x20000000 <= e0 < 0x20080000) or (e0 & 3):
+ return False
+ return True
+
+
+def scrub_scan(kc, lines):
+ """Emit the scrubber buffer words' PAs once (re-emitted if they move)."""
+ try:
+ f = open('/proc/vmallocinfo')
+ except OSError:
+ return
+ with f:
+ for line in f:
+ if 'nvidia' not in line or 'ioremap' in line or 'vmap' not in line:
+ continue
+ parts = line.split()
+ try:
+ a, b = parts[0].split('-')
+ lo, hi = int(a, 16), int(b, 16)
+ except (ValueError, IndexError):
+ continue
+ if (hi - lo) not in SCRUB_AREA_SIZES:
+ continue
+ fin_pa = kc.va_to_pa(lo + 0x6c000)
+ ring_pa = kc.va_to_pa(lo + 0x64000)
+ if fin_pa is None or ring_pa is None:
+ continue
+ if not _scrub_ring_ok(kc, ring_pa):
+ dbg('scrub candidate 0x%x rejected (ring shape)' % lo)
+ continue
+ fin = fin_pa + 4
+ hsem = fin_pa
+ key = (fin, hsem)
+ if _scrub_sent.get('key') == key:
+ return
+ _scrub_sent['key'] = key
+ lines.append('#scrub 0x%x,0x%x\n' % (fin, hsem))
+ dbg('scrub buffer emitted lo=0x%x fin=0x%x hsem=0x%x'
+ % (lo, fin, hsem))
+ return
+
+
def nvidia_fds(pid):
"""CUDA device fds this pid holds: 'uvm', 'gpu', or None.
@@ -1175,6 +1605,8 @@ def main():
resolved_kc_pb = set()
retry_at = {}
kc_refresh_at = {}
+ kc_reemit_at = {}
+ kc_remap_at = {}
tracer = TraceReader()
kc = Kcore()
kc_ok = kc.load()
@@ -1186,6 +1618,14 @@ def main():
pass
last_pid = None
last_log = 0.0
+ scrub_next = 0.0
+ demand_next = 0.0
+ # m165 (D-F1a): pool page VAs learned from demand, re-asserted every
+ # KC_POOL_REEMIT_S. Boot-stable (UVM frees pool pages only at GPU
+ # deinit), so the set deliberately persists across client switches.
+ pool_pages = {}
+ pool_reemit_next = 0.0
+ _want_state = {"mtime": None, "done": set()}
loop = 0
while True:
pid = cdt_pid()
@@ -1211,6 +1651,8 @@ def main():
resolved_kc_pb = set()
retry_at = {}
kc_refresh_at = {}
+ kc_reemit_at = {}
+ kc_remap_at = {}
# Client switch: the old client's channel/put pointers are freed
# memory. Clearing the #put map stops stale reads from being
# emitted as if they were the new client's counters.
@@ -1262,6 +1704,84 @@ def main():
new = [(va, pa) for va, pa in cur.items() if seen.get(va) != pa]
pending_cur = cur
lines = []
+ # Demand pins (host -> this scanner): the host appends the CE
+ # push-buffer VAs of GPFIFO entries it cannot read to
+ # /tmp/guest_want_vas.txt (written via the guest agent by
+ # pin_gr_stream); resolve each by content against the
+ # nv_vm_map_pages areas and emit the pair. This is the
+ # event-independent hole filler for the stall class where the
+ # one-shot #sem kprobe event was lost and the object-chain
+ # resolver never learns the client.
+ if kc_ok and time.time() >= demand_next:
+ demand_next = time.time() + 1.0
+ try:
+ st_d = os.stat("/tmp/guest_want_vas.txt")
+ if st_d.st_mtime != _want_state.get("mtime"):
+ with open("/tmp/guest_want_vas.txt") as fw:
+ wvas = [int(t, 16) for t in fw.read().split()
+ if t.startswith("0x")]
+ _want_state["mtime"] = st_d.st_mtime
+ for wv in wvas[-64:]:
+ if wv in _want_state["done"]:
+ continue
+ got = []
+ if POOL_SEM_LO <= (wv & ~0xFFF) < POOL_SEM_HI:
+ # m165 (D-F1a): pool/waiter VAs have no method
+ # stream content and the CE offset math does not
+ # apply; route them to the structural tracker-sem
+ # walk instead. m168: the tracker sems are
+ # long-lived objects (allocated once) whose page
+ # pointer follows the pool across clients, so try
+ # every sem we have ever resolved a chain for -
+ # seen_sems alone can miss the live generation.
+ _psems = set(seen_sems) | set(KC_PB_STATE.keys()) \
+ | set(resolved_kc_pb)
+ try:
+ ok = resolve_pool_va(kc, wv, got,
+ _psems)
+ except Exception:
+ ok = False
+ got = []
+ dbg('demand-pool-va: 0x%x %s'
+ % (wv, 'ok' if ok else 'pending'))
+ if ok and got:
+ pool_pages.setdefault(wv & ~0xFFF, True)
+ else:
+ try:
+ ok = resolve_va_by_content(kc, wv, got)
+ except Exception:
+ ok = False
+ got = []
+ if ok and got:
+ lines.extend(got)
+ _want_state["done"].add(wv)
+ except OSError:
+ pass
+ # m165 (D-F1a): re-assert the pool page pair every KC_POOL_REEMIT_S.
+ # The pool page is boot-stable (UVM frees it only at GPU deinit) but
+ # the host pin table is wiped at every session flush, so a one-shot
+ # pin leaves later clients starving until the slow structural walks
+ # answer; the periodic re-assert heals the wipe within seconds (the
+ # same argument as the CE mem sweep above).
+ if kc_ok and pool_pages and time.time() >= pool_reemit_next:
+ pool_reemit_next = time.time() + KC_POOL_REEMIT_S
+ for pv in list(pool_pages):
+ got = []
+ try:
+ ok = resolve_pool_va(kc, pv, got, seen_sems)
+ except Exception:
+ ok = False
+ got = []
+ if ok and got:
+ lines.extend(got)
+ # PMA scrubber buffer discovery (throttled): resolve the finish/host-
+ # semaphore word PAs and stream them for host registration.
+ if kc_ok and time.time() >= scrub_next:
+ scrub_next = time.time() + 5.0
+ try:
+ scrub_scan(kc, lines)
+ except Exception as e:
+ dbg('scrub scan failed: %s' % e)
for ctl in tracer.drain():
parts = ctl.split()
if ctl.startswith("#sem") and len(parts) >= 2:
@@ -1302,12 +1822,68 @@ def main():
except Exception:
_fresh = True
if not _fresh:
- _st = KC_PB_STATE.pop(sem, None)
+ # Re-resolve, but never lose the latch on a failed
+ # attempt: a dropped latch means the host keeps no
+ # pins for this channel at all, which is worse than
+ # a possibly-stale pin (the host content-checks the
+ # first page and now rejects all-FF continuation
+ # chunks, so stale pages are bounded). On failure
+ # keep the old state and defer the next attempt.
+ _st = KC_PB_STATE.get(sem)
if _st and _st.get("mem"):
RESOLVED_KC_PB.discard(_st["mem"])
- resolved_kc_pb.discard(sem)
- retry_at.pop(("pk", sem), None)
- dbg("kc-chain stale: re-resolving sem=0x%x" % sem)
+ got = []
+ try:
+ _found2 = resolve_pb_chain(kc, sem, got)
+ except Exception:
+ _found2 = False
+ got = []
+ if _found2 and got:
+ _new = KC_PB_STATE.get(sem)
+ _newmem = _new.get("mem") if _new else None
+ dbg("kc-chain stale: re-resolved sem=0x%x "
+ "new-mem=%s runs=%d"
+ % (sem, "yes" if _newmem !=
+ (_st.get("mem") if _st else None)
+ else "no", len(got)))
+ lines.extend(got)
+ added += len(got)
+ resolved_kc_pb.add(sem)
+ else:
+ if _st and _st.get("mem"):
+ RESOLVED_KC_PB.add(_st["mem"])
+ if sem not in KC_PB_STATE:
+ KC_PB_STATE[sem] = _st
+ kc_refresh_at[sem] = now_r + 5.0
+ dbg("kc-chain stale: keep old latch sem=0x%x"
+ % sem)
+ else:
+ # CE mem delta sweep (second-client pins): the one-shot
+ # emission pinned the mem once; pages the client writes
+ # later (or whose vmap pairing moved) were never
+ # re-emitted, so the host kept zeros for the other CE
+ # ranks' pushes. Sweep every KC_REEMIT_S, emitting only
+ # new/changed nonzero pairs; burst the Kcore PA cache
+ # every KC_REMAP_S so a remap is not shadowed.
+ _st = KC_PB_STATE.get(sem)
+ if _st is not None and \
+ now_r >= kc_reemit_at.get(sem, 0):
+ _fresh_map = now_r >= kc_remap_at.get(sem, 0)
+ if _fresh_map:
+ kc_remap_at[sem] = now_r + KC_REMAP_S
+ kc_reemit_at[sem] = now_r + KC_REEMIT_S
+ got = []
+ try:
+ _n2 = _emit_ce_mem_delta(kc, _st, got,
+ fresh_map=_fresh_map)
+ except Exception:
+ _n2 = 0
+ got = []
+ if got:
+ lines.extend(got)
+ added += len(got)
+ dbg("ce-delta sem=0x%x runs=%d mem=0x%x"
+ % (sem, _n2, _st.get("mem", 0)))
# CE push-buffer chain (second-client stall fix): resolve
# pool -> manager -> pushbuffer -> rm_mem directly, so the
# push-buffer pages never depend on the racy one-shot uvbp
diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index b864ab5..dff2d2d 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -16,6 +16,7 @@ import re
import socket
import subprocess
import sys
+import threading
import time
HERE = os.path.dirname(os.path.abspath(__file__))
@@ -62,12 +63,19 @@ def _start_cmd():
capture_output=True, text=True).stdout
prefixes = os.environ.get("CLIENT_PREFIXES", "cdt,va,victory")
vma_every = os.environ.get("STREAM_VMA_EVERY", "1")
- return ("echo '%s' | base64 -d > /tmp/guest_pin_stream.py; "
- "if [ -f /tmp/guest_pin_stream.pid ]; then "
- "p=$(cat /tmp/guest_pin_stream.pid 2>/dev/null); "
- "[ -n \"$p\" ] && grep -q guest_pin_stream "
- "/proc/$p/cmdline 2>/dev/null && sudo kill $p 2>/dev/null; "
- "fi; "
+ # Kill EVERY existing scanner, not just the pid-file one: a retry after
+ # a partially-failed first attempt (pid file not written yet) used to
+ # leave the first scanner alive, and multiple scanners each re-assert
+ # the whole CE pair set - r148 ran three and drove the guest to load 19
+ # (clients wedged in userspace spins). The pattern is anchored
+ # (^python3) and assembled at runtime via printf: an unanchored pattern
+ # or a literal one would also match this command's own shell (its
+ # cmdline contains the script path), and pkill would kill the shell
+ # before it could start the new scanner - every start attempt then
+ # failed silently (r149).
+ return ("P=$(printf '^python3 /tmp/guest_pin_%%s.py' stream); "
+ "sudo pkill -f \"$P\" 2>/dev/null; sleep 1; "
+ "echo '%s' | base64 -d > /tmp/guest_pin_stream.py; "
"sudo rm -f /tmp/guest_pin_stream.pid; "
"sudo setsid nohup env CLIENT_PREFIXES='%s' "
"STREAM_VMA_EVERY='%s' "
@@ -101,9 +109,70 @@ def inject(va, pa, ln):
return False
+def scrub_register(fin_pa, hsem_pa):
+ """Register the PMA scrubber buffer words (guest-resolved PAs) with the
+ service so it can author the scrub progress (third-client scrub freeze)."""
+ cmd = ("printf '0x%x,0x%x' > %s/sema_scrubbuf_add"
+ % (fin_pa, hsem_pa, PARAM))
+ for _ in range(60):
+ r = subprocess.run(["sudo", "-n", "sh", "-c", cmd],
+ capture_output=True, text=True)
+ if r.returncode == 0:
+ log("scrub registered fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa))
+ return True
+ time.sleep(0.05)
+ log("scrub register FAILED fin=0x%x hsem=0x%x" % (fin_pa, hsem_pa))
+ return False
+
+
+def _demand_loop():
+ """Push the host's unreadable CE push VAs to the guest scanner.
+
+ The service logs each distinct stalled VA once per boot ('stalled VA
+ #N: 0x...'). Each sighting is tracked and re-sent every 20 s for five
+ minutes: the guest resolves the VA by content against the
+ nv_vm_map_pages areas (event-independent, no object chain), so a page
+ that was not yet written when first asked is picked up on a retry.
+ This closes the stall class where the one-shot #sem kprobe event was
+ lost and the chain resolver never learns the client.
+ """
+ track = {} # va -> [first_seen, last_sent]
+ while True:
+ time.sleep(3)
+ try:
+ r = subprocess.run(["sudo", "-n", "dmesg"],
+ capture_output=True, text=True, timeout=20)
+ except Exception:
+ continue
+ now = time.time()
+ for m in re.finditer(r"(?:stalled VA #\d+|pending release starved va)=(0x[0-9a-f]+)",
+ (r.stdout or "")[-200000:]):
+ va = int(m.group(1), 16)
+ track.setdefault(va, [now, 0.0])
+ for va in list(track):
+ first, last = track[va]
+ if now - first > 300.0:
+ del track[va]
+ todo = [va for va, (f, l) in track.items() if now - l > 20.0]
+ if not todo:
+ continue
+ lines = "".join("0x%x\n" % va for va in todo)
+ try:
+ vexec("printf '%%s' '%s' >> /tmp/guest_want_vas.txt" % lines, 30)
+ except Exception:
+ continue
+ for va in todo:
+ track[va][1] = now
+ log("demand: sent %d stalled VA(s) to guest" % len(todo))
+
+
def main():
if not install_and_start():
log("WARN: could not start guest scanner (will retry via vsock wait)")
+ try:
+ threading.Thread(target=_demand_loop, daemon=True).start()
+ except Exception as e:
+ log("WARN: demand thread failed to start: %s" % e)
srv = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
srv.bind((socket.VMADDR_CID_ANY, PORT))
@@ -158,6 +227,13 @@ def main():
log(text)
_mirror(text)
continue
+ if text.startswith("#scrub"):
+ m = re.search(r"0x([0-9a-fA-F]+),0x([0-9a-fA-F]+)", text)
+ _mirror(text)
+ if m:
+ scrub_register(int(m.group(1), 16),
+ int(m.group(2), 16))
+ continue
m = re.match(r"^(0x[0-9a-f]+),(0x[0-9a-f]+)(?:,(\d+))?$", text)
if m:
runs.append((int(m.group(1), 16), int(m.group(2), 16),
@@ -182,7 +258,18 @@ def _flush(runs, seen):
"""
todo = []
for va, pa, n in runs:
- if seen.get(va) == pa and n == 1:
+ # The CE window (0x120000000..0x121000000) and the UVM semaphore
+ # pool page range (0x121000000..0x122000000) are re-asserted by the
+ # guest every couple of seconds on purpose: the host's session flush
+ # wipes the whole pin table when the previous client's last channel
+ # closes, and a pin batch that raced in for the new client must be
+ # allowed to land again even when the pair is unchanged. Skipping
+ # repeats there stranded the second client's CE pushes (the r144/
+ # r145 stall: sema-probe "NO ENTRY" for the live push pages) and the
+ # pool/tracker page (pending release starved, every client's first
+ # seconds).
+ ce_win = 0x120000000 <= va < 0x122000000
+ if not ce_win and seen.get(va) == pa and n == 1:
continue
todo.append((va, pa, n))
if not todo:
diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py
index 5124805..a4b2fc0 100644
--- a/tools/guest-diag/semfix_watch.py
+++ b/tools/guest-diag/semfix_watch.py
@@ -189,18 +189,46 @@ def guest_trace():
Read *before* clearing: the create-time probes fire once per client and
would be wiped by a clear that races them.
+
+ 2026-10-16 robustness fix (the cascade root): this used to run a bare
+ `tail` over the guest agent. A tracefs read that blocks under a busy
+ guest left *uninterruptible* tail processes behind (observed dozens in
+ D state), the guest-agent exec then blocked for its full 90 s timeout
+ every iteration, and the pool/tracker pins this tool supplies stopped
+ flowing - after ~5 clients every new client froze at DEVICE waiting on
+ its UVM tracker with `pending release starved ... pool/waiter page`.
+ Now the read is bounded remotely (`timeout -s KILL 12`) so no tail can
+ outlive its call, and SSH (which stays healthy when the guest agent
+ wedges) is preferred with the agent as fallback.
"""
- inner = ("tail -c 131072 /sys/kernel/tracing/trace "
+ inner = ("timeout -s KILL 12 tail -c 131072 /sys/kernel/tracing/trace "
"| grep -E \"uvmt|uvch|uvbp|uvpbf\" "
"| tail -400")
if not os.environ.get("KEEP_TRACE"):
inner += "; echo > /sys/kernel/tracing/trace"
- cmd = "sudo sh -c '%s'" % inner
+ outer = "sudo sh -c '%s'" % inner
+ out = ""
+ ssh = os.environ.get("SEMFIX_SSH_PORT", "2222")
+ ssh_cmd = [
+ "sshpass", "-p", os.environ.get("VGPU_GUEST_PASS", "ubuntu"),
+ "ssh", "-o", "StrictHostKeyChecking=no",
+ "-o", "UserKnownHostsFile=/dev/null",
+ "-o", "PubkeyAuthentication=no", "-o", "ConnectTimeout=6",
+ "-p", ssh, "ubuntu@127.0.0.1", outer,
+ ]
try:
- out = subprocess.run([VMEXEC, "exec", cmd], capture_output=True,
- text=True, timeout=90).stdout
+ r = subprocess.run(ssh_cmd, capture_output=True, text=True,
+ timeout=25)
+ out = r.stdout or ""
except Exception:
- return set(), set(), set()
+ out = ""
+ if not out:
+ try:
+ out = subprocess.run([VMEXEC, "exec", outer],
+ capture_output=True, text=True,
+ timeout=25).stdout or ""
+ except Exception:
+ return set(), set(), set()
sems = {int(m, 16) for m in re.findall(r"sem=0x([0-9a-fA-F]+)", out)}
pbs = {int(m, 16) for m in re.findall(r"pb=0x([0-9a-fA-F]+)", out)}
chans = {int(m, 16) for m in re.findall(r"ch=0x([0-9a-fA-F]+)", out)}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-17 11:20:34 +0000