diff options
Diffstat (limited to 'tools/guest-diag/guest_pin_stream.py')
| -rw-r--r-- | tools/guest-diag/guest_pin_stream.py | 1045 |
1 files changed, 1007 insertions, 38 deletions
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 3f4183c..a645a86 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -211,7 +211,11 @@ def read_u32(kc, va): return struct.unpack("<I", d)[0] if d else 0xFFFFFFFF -DBG_BUDGET = int(os.environ.get("STREAM_DBG_BUDGET", "600")) +DBG_BUDGET = int(os.environ.get("STREAM_DBG_BUDGET", "2400")) +# SESSION 153-CONTINUED-65: minimum seconds between full-offset sweeps of +# one strict anchor (the demand file re-sends every ~2 s; the sweep can cost +# tens of seconds - see the throttle comment at the strict processing loop). +ANCHOR_RETRY_S = float(os.environ.get("STREAM_ANCHOR_RETRY_S", "20")) _dbg_count = [0] @@ -300,50 +304,702 @@ def _nv_vmap_areas(kc): return areas +# Learned page-offset per nv_vm_map_pages area for the demand content match +# (2026-10-16b): the first successful resolution records which page index the +# VA maps to, and later pushes in the same allocation retry that offset first +# instead of walking the whole +/-16 neighbourhood. +_area_off = {} + +# SESSION 153-CONTINUED-55: per-GL-family learned {family_base: (area_lo, +# adj)}. Once one page of a GL push-buffer family is content-verified the +# whole family resolves through the learned area/offset (continuation and +# zero pages included); cleared when the area disappears (context rebuild). +_gl_fam = {} + + def resolve_va_by_content(kc, va, out): """Demand pin: find the page carrying GPU VA 'va' by content match. The host reports the CE push-buffer VAs whose GPFIFO entries it cannot read (dmesg 'stalled VA #N: 0x...'). The guest walks every nv_vm_map_pages vmap area and tests the candidate leaf page at the VA's - in-window offset (plus one neighbour on either side, for the per-mem - header-page offset ambiguity) against the host's method-stream prefix - check. On a match it emits the exact pair, so the host can translate - and consume the entry - no kprobe event, no object-chain discovery, no - resolution lottery. Only the requested VA is pinned, so a wrong - candidate cannot poison other VAs. + in-window offset (plus a neighbourhood, for the per-mem header-page + offset ambiguity) against the host's method-stream prefix check. On a + match it emits the exact pair, so the host can translate and consume the + entry - no kprobe event, no object-chain discovery, no resolution + lottery. Only the requested VA is pinned, so a wrong candidate cannot + poison other VAs. + + SESSION 153-CONTINUED-55: the same rule now covers the GL push-buffer + families (PB_GL_FAMILIES). The setup-push holds (va=0x4001xxx) and the + draw pushes (0x402xxxx) are nv_vm_map_pages allocations exactly like the + CE window; without this they were rejected here silently and the held + setup push was never pinned, so its 4K pages read zeros at the host. + + 2026-10-16b: the offset search window was +/-1 page. Pushes of a client + whose pushbuffer vmap carries a larger per-allocation header (observed + with multi-MB CE staging: pushes at 0x120000f58+ never resolved and the + demand loop re-sent the same VA for minutes while the client crawled) + fall outside it. Widen to +/-16 pages and LEARN the winning offset per + vmap area, because consecutive pushes advance sequentially inside the + same allocation and share the offset. """ - if not (PB_CE_LO <= va < PB_CE_HI): - return False - # 2026-10-16 fix: the CE push-buffer window is 16 MB (PB_CE_LO..PB_CE_HI) - # and each rank is a 1 MB slice of ONE nv_vm_map_pages vmap. The old - # `& 0xFFFFF` folded every 1 MB boundary to offset 0, so ranks 0x120700000, - # 0x120800000 and 0x120a00000 all resolved to page 0 of the first area - - # the same wrong PA - and the host read garbage for every rank but the - # first. Keep the full 16 MB offset: the page index is the true offset - # into the mem, and the per-area pages bound rejects areas too small. - off = (va - PB_CE_LO) & 0xFFFFFF - k = off >> 12 - for (lo, _hi, pages) in _nv_vmap_areas(kc): - for adj in (0, 1, -1): + if PB_CE_LO <= va < PB_CE_HI: + k = ((va - PB_CE_LO) & 0xFFFFFF) >> 12 + gl_base = None + else: + gl_base = next((lo for lo, hi in PB_GL_FAMILIES if lo <= va < hi), + None) + if gl_base is None: + return False + k = (va - gl_base) >> 12 + areas = _nv_vmap_areas(kc) + # SESSION 153-CONTINUED-55: a GL push-buffer family is one allocation + # whose vmap offset is IDENTITY with the family offset (verified: the + # setup push's bytes at va 0x4001004 read back byte-exact at the vmap + # page's +0x1004). Once one page of the family resolves, every page of + # the family is known: continuation pages of a stream do NOT start with + # a method header (the 60216-byte setup push crosses page boundaries + # mid-run, so pages 2/3/5/7 fail the prefix test) and unwritten pages + # read all zero (pages 8..15 of the same push) yet are real translations + # the host read still needs. Use the learned (area, adj) for the whole + # family; only the FIRST learn pays the content test. + learned_fam = _gl_fam.get(gl_base) if gl_base else None + if learned_fam is not None: + lo, adj = learned_fam + if any(a_lo == lo for (a_lo, _h, _p) in areas): + idx = k + adj + pages = next(p for (a_lo, _h, p) in areas if a_lo == lo) + if 0 <= idx < pages: + cpu = lo + idx * 0x1000 + pa = kc.va_to_pa(cpu) + d = kc.read(cpu, 64) if pa is not None else None + if pa is not None and d is not None: + if gl_base is not None or (_push_prefix_ok(d) and + any(d[:32])): + out.append("0x%x,0x%x,1\n" % (va & ~0xFFF, pa)) + dbg("demand-va: 0x%x -> pa=0x%x (gl family learned" + " area=0x%x adj=%+d)" + % (va & ~0xFFF, pa, lo, adj)) + return True + # The learned area is gone (context rebuilt): fall through to a + # fresh content match and re-learn. + _gl_fam.pop(gl_base, None) + for (lo, _hi, pages) in areas: + learned = _area_off.get(lo) + if learned is not None: + adjs = [learned] + else: + adjs = [0, 1, -1] + adjs += [d for m in (2, 4, 8, 16) for d in (m, -m)] + for adj in adjs: idx = k + adj if idx < 0 or idx >= pages: continue cpu = lo + idx * 0x1000 - d = kc.read(cpu, 64) - if not d or not any(d[:32]): - continue - if not _push_prefix_ok(d): - continue pa = kc.va_to_pa(cpu) if pa is None: continue + # 0x200 (not 64): a GL family's page 0 can carry its first + # method header at offset ~0x68 behind a zero-filled prefix + # (observed on the 60216 B setup push), invisible to a 64-byte + # read. + d = kc.read(cpu, 0x200) + if not d or not any(d): + continue + if not _push_prefix_ok(d): + continue + _area_off[lo] = adj + if gl_base is not None: + # First content-verified page of this GL family: the + # identity mapping (area, adj) now serves every page of + # the family, including continuation and zero pages. + _gl_fam[gl_base] = (lo, adj) out.append("0x%x,0x%x,1\n" % (va & ~0xFFF, pa)) - dbg("demand-va: 0x%x -> pa=0x%x (area off=%d adj=%+d)" - % (va & ~0xFFF, pa, idx, adj)) + dbg("demand-va: 0x%x -> pa=0x%x (area off=%d adj=%+d%s)" + % (va & ~0xFFF, pa, idx, adj, + " gl-learn" if gl_base is not None else "")) return True dbg("demand-va: 0x%x not found by content (%d areas)" - % (va & ~0xFFF, len(_nv_vmap_areas(kc)))) + % (va & ~0xFFF, len(areas))) + return False + + +# SESSION 153-CONTINUED-64: candidate adjacencies for the strict window scan. +_STRICT_ADJ = (0, 1, -1, 2, -2, 3, -3, 4, -4, 6, -6, 8, -8, 14, -14, 16, -16) + + +def _stream_parse_full(buf): + """Exact port of the host's svc_stream_parse (gsp_sema_service.c), plus + the MME-setup content requirements the bare parse lacks. + + Zero words are NOP padding (skipped anywhere in the stream), ops 1/3/5 + are INC/NON_INC/ONE_INC runs with a 12-bit count (1..0xfff) and count + data words, ops 0/2/4/7 are header-only, anything else fails. The buffer + parses only when at least one method was seen and the walk lands exactly + on the end (trailing zero padding allowed). + + SESSION 153-CONTINUED-64b: the bare exact-end parse accepted a GMMU + page-table window - PTEs are {pfn<<12, flags} u32 pairs whose values sit + below 0x20000000, so every word reads as op 0 (header-only) and the walk + lands exactly on the end (r1122 pinned pa=0x11dbb0000: all 32 words of + every page op 0, zero data runs, host read `sema-bigrun hdr=0x1dbb7000`). + A window is now only accepted when it carries at least one data run AND + at least one run addressing an MME-setup method (0x114..0x124 / 0x234 / + 0x550..0x568 / 0x3400..0x34fc - the same windows the host's + svc_mme_setup_run_addr classifies with). A page table can never satisfy + either; the held setup push always carries both (the r1116 capture's + first run `20020047` = INC run @0x11c). + """ + off = 0 + methods = 0 + data_runs = 0 + mme_hits = 0 + code_words = 0 + n = len(buf) + while off + 4 <= n: + hdr = struct.unpack_from('<I', buf, off)[0] + if hdr == 0: + off += 4 + continue + op = (hdr >> 29) & 7 + count = (hdr >> 16) & 0xFFF + if op in (1, 3, 5): + if not count or off + 4 + count * 4 > n: + return False + data_runs += 1 + addr = (hdr & 0x1FFF) << 2 + if (0x114 <= addr <= 0x124 or addr == 0x234 or + 0x550 <= addr <= 0x568 or 0x3400 <= addr <= 0x34FC): + mme_hits += 1 + if addr == 0x114 and count >= 1: + code_words += count + off += 4 + count * 4 + elif op in (0, 2, 4, 7): + off += 4 + else: + return False + methods += 1 + if methods > 8192: + return False + while off < n and buf[off] == 0: + off += 1 + return (methods > 0 and off == n and data_runs > 0 and mme_hits > 0 + and code_words >= 8) + + +def _walk_measure(buf): + """Method-shape walk returning (end_ok, data_runs, mme_hits, code_words, + pair_runs, parsed_bytes). 'end_ok' mirrors _stream_parse_full's + exact-end + zero padding landing; 'pair_runs' counts runs addressing the + SA-pair method 0x11c (the client push signature); 'parsed_bytes' is how + far the walk got before a shape failure (or the window end).""" + off = 0 + methods = 0 + data_runs = 0 + mme_hits = 0 + code_words = 0 + pair_runs = 0 + n = len(buf) + end_ok = False + while off + 4 <= n: + hdr = struct.unpack_from('<I', buf, off)[0] + if hdr == 0: + off += 4 + continue + op = (hdr >> 29) & 7 + count = (hdr >> 16) & 0xFFF + if op in (1, 3, 5): + if not count or off + 4 + count * 4 > n: + break + data_runs += 1 + addr = (hdr & 0x1FFF) << 2 + if (0x114 <= addr <= 0x124 or addr == 0x234 or + 0x550 <= addr <= 0x568 or 0x3400 <= addr <= 0x34FC): + mme_hits += 1 + if addr == 0x114: + code_words += count + if addr == 0x11c and count >= 2: + pair_runs += 1 + off += 4 + count * 4 + elif op in (0, 2, 4, 7): + off += 4 + else: + break + methods += 1 + if methods > 8192: + break + else: + o2 = off + while o2 < n and buf[o2] == 0: + o2 += 1 + end_ok = (methods > 0 and o2 == n) + return end_ok, data_runs, mme_hits, code_words, pair_runs, off + + +def _window_accept(buf, ln): + """Acceptance for a client-mapping candidate window. + + SESSION 153-CONTINUED-65: the full-offset sweep of the client-mem + resolver needs a false-positive gate that survives pointer/event data. + The probe of the r1125 boot found a 60216-byte window of pointer junk + that walked to an exact end with one misinterpreted 0x114 run + (mme=1): the old accept (data>0, mme>0, code>=8) would have pinned it. + Large windows therefore require the client push signature - at least + two MME-setup runs or two SA-pair (0x11c) runs - in addition to the + counts; small windows keep the historical strict rule. The relaxed + path additionally accepts a window whose tail is unmapped/unwritten + (walk parses >= 0x1000) when it carries >= 64 code words (the macro + upload signature). + + Returns (ok, mode). + """ + if ln < 0x8000: + return _stream_parse_full(buf), "strict" + end_ok, data, mme, code, pairs, parsed = _walk_measure(buf) + # The client-push signature: SA-pair (0x11c) runs or 0x114 code runs. + # The r1125 probe of the live client mappings measured 10k+ pointer/float + # windows with data/mme hits but pairs=0, code=0 - MME-setup-family hits + # alone (0x124/0x34xx) are NOT a discriminator; the pair/code runs are. + sig = (pairs >= 2 or code >= 8) + if end_ok and data >= 2 and sig: + return True, "strict" + if data >= 2 and sig and parsed >= 0x1000: + return True, "relaxed" + return False, None + + +def _prefix_walk_ok(buf, off): + """Cheap first-page pre-filter for a strict window candidate. + + Walks the method shapes from `off` to the end of `buf` WITHOUT requiring + the end to land on a boundary (runs may cross into the next page and a + run header near the page end declares data we cannot see yet). Fails on + the first header no method shape can produce; this kills arbitrary data + pages after a few words so the full parse only runs on survivors. + """ + n = len(buf) + while off + 4 <= n: + hdr = struct.unpack_from('<I', buf, off)[0] + if hdr == 0: + off += 4 + continue + op = (hdr >> 29) & 7 + count = (hdr >> 16) & 0xFFF + if op in (1, 3, 5): + if not count: + return False + off += 4 + count * 4 + elif op in (0, 2, 4, 7): + off += 4 + else: + return False + return True + + +def _read_window(kc, cpu, off, ln): + """Read [cpu+off, cpu+off+ln) page-by-page through kcore. + + A vmap area's pages are contiguous in the kernel VA but arbitrary in + physical memory, so the window is assembled one page at a time. + """ + out = b"" + a = cpu + off + left = ln + while left > 0: + n = min(left, 0x1000 - (a & 0xFFF)) + d = kc.read(a, n) + if not d: + return None + out += d + a += n + left -= n + return out + + +def _mme_run_in_page(d): + """True when a 4K page contains a run addressing an MME-setup method. + + The strict-window prefilter for client-mapping candidates (cheap: one + page read rejects the ~511/512 wrong shifts before the full parse). + """ + n = len(d) + off = 0 + while off + 4 <= n: + hdr = struct.unpack_from('<I', d, off)[0] + if hdr == 0: + off += 4 + continue + op = (hdr >> 29) & 7 + count = (hdr >> 16) & 0xFFF + if op in (1, 3, 5): + if not count: + off += 4 + continue + addr = (hdr & 0x1FFF) << 2 + if (0x114 <= addr <= 0x124 or addr == 0x234 or + 0x550 <= addr <= 0x568 or 0x3400 <= addr <= 0x34FC): + return True + off += 4 + count * 4 + else: + off += 4 + return False + + +def resolve_va_client_mem(pid, ent_va, off, ln, out): + """Strict pin for a setup push whose pages live in CLIENT sysmem. + + SESSION 153-CONTINUED-64c: when the setup push's allocation is UVM + sysmem it has NO kernel vmap (r1122: no nv_vm_map_pages area held the + `a0100045` 0x114 upload runs anywhere on the boot - the content exists + only in the client's `rw-s` /dev/nvidia mappings, and the host's + GPU-page-table walk answered an unrelated page-table buffer for the + held VA). The client's CPU mappings + /proc/<pid>/pagemap are the only + ground truth for those pages. Scan the client's readable+writable + mappings: for each candidate page shift, read the exact [va, va+len) + window via /proc/<pid>/mem, validate with the host's own full parse, + and emit pins for every page of the validated window from the + pagemap. The cheap per-page MME-run prefilter rejects wrong shifts. + """ + if not pid: + return False + try: + pfd = os.open('/proc/%d/pagemap' % pid, os.O_RDONLY) + mfd = os.open('/proc/%d/mem' % pid, os.O_RDONLY) + except OSError: + return False + try: + gl_base = next((lo for lo, hi in PB_GL_FAMILIES + if lo <= (ent_va & ~0xFFF) < hi), None) + k = ((ent_va & ~0xFFF) - gl_base) >> 12 if gl_base is not None else None + if ln <= 0 or ln > 0x40000: + return False + need = (off + ln + 0xFFF) // 0x1000 + if need > 64: + return False + try: + maps = open('/proc/%d/maps' % pid).read().splitlines() + except OSError: + return False + tried = 0 + for line in maps: + parts = line.split() + if len(parts) < 5 or 'rw' not in parts[1]: + continue + # SESSION 153-CONTINUED-64e: any rw mapping may carry the + # sysmem push (observed as pathless rw-s /dev/nvidia mappings; + # memfd-backed buffers cannot be excluded either - the strict + # parse + MME prefilter are the safety gates, not the path). + try: + lo, hi = [int(x, 16) for x in parts[0].split('-')] + except ValueError: + continue + if hi - lo < (need + 4) * 0x1000 or hi - lo > (64 << 20): + continue + pages = (hi - lo) >> 12 + if k is None or k == 0: + # Unknown family, or the FAMILY-BASE page (k == 0): neither + # has an identity page index to key on, so sweep every page + # start of the mapping (bounded). SESSION 153-CONTINUED-65: + # the 0x3200000 macro-upload push's window sits at an + # arbitrary page offset inside the client's mapping (the + # r1124 power-of-two neighbourhood never tried it; the + # window content measures code=712, exact-end, under the + # strict walk). + adjs = range(0, max(1, pages - need + 1)) + else: + # SESSION 153-CONTINUED-65: neighbourhood FIRST, then the + # FULL page sweep. The r1125 setup push (family pages k=1 + # and k=33, 60216 B) failed every power-of-two offset while + # a matching window could sit at an arbitrary mapping page; + # every candidate still passes _window_accept before any + # pin is emitted, so the widened sweep cannot pin a window + # the parse rejects. + near = [0, 1, -1, 2, -2, 4, -4, 8, -8, 16, -16, 32, -32, + 64, -64, 128, -128, 256, -256] + near_set = set(near) + rest = [a for a in range(-k, pages - need + 1 - k) + if a not in near_set] + adjs = near + rest + for adj in adjs: + idx = (k + adj) if k is not None else adj + if idx < 0 or idx + need > pages: + continue + win = lo + idx * 0x1000 + off + tried += 1 + if tried > 8192: + break + try: + first = os.pread(mfd, 0x1000 - off, win) + except OSError: + continue + if not first: + continue + # An all-zero first page is a legitimate push head (the + # method stream may start later); only a page with non-zero + # content and no MME-setup run is rejected outright. + if not _mme_run_in_page(first) and any(first): + continue + buf = b'' + left = ln + a = win + ok = True + while left > 0: + n = min(left, 0x1000 - (a & 0xFFF)) + try: + d = os.pread(mfd, n, a) + except OSError: + ok = False + break + if not d or len(d) != n: + ok = False + break + buf += d + a += n + left -= n + if not ok: + continue + acc, mode = _window_accept(buf, ln) + if not acc: + continue + pas = [] + for j in range(need): + vpg = (ent_va & ~0xFFF) + j * 0x1000 + cpg = lo + (idx + j) * 0x1000 + os.lseek(pfd, (cpg >> 12) * 8, 0) + e = int.from_bytes(os.read(pfd, 8), 'little') + if not (e >> 63) & 1: + pas = None + break + pas.append(((e & ((1 << 55) - 1)) << 12)) + if not pas: + continue + for j, pa in enumerate(pas): + out.append("0x%x,0x%x,1\n" + % ((ent_va & ~0xFFF) + j * 0x1000, pa)) + dbg("demand-va: 0x%x -> pa=0x%x (%s client-mem parse" + " %u B ok pages=%u map=0x%x idx=%d)" + % (ent_va & ~0xFFF, pas[0], mode, ln, need, lo, idx)) + return True + if tried > 8192: + break + return False + finally: + os.close(pfd) + os.close(mfd) + + +def _strict_kvm_sweep(kc, ent_va, off, ln, out): + """Kernel-vmap sweep for an anchor with no known-family page index. + + SESSION 153-CONTINUED-74: mirrors the k == 0 known-family path (the + resolver that found the r1126 0x3200000 macro window) but works for ANY + anchor outside PB_GL_FAMILIES. Every candidate window passes the host's + own full parse (`_window_accept`) before a single pin is emitted, so a + false positive is impossible by construction; the cheap per-page prefix + walk rejects arbitrary data after a few words. Bounded: at most + 8192 candidate windows and at most 64 pages of a window (the same caps + as resolve_va_client_mem). + + Areas swept: the nv_vm_map_pages vmap set (the historical resolver) plus + the nvidia os_alloc_mem vmalloc areas (`vpages`), which the r1156 probe + showed can be large (571 pages in one area) and are completely outside + _nv_vmap_areas' filter. A window is only emitted for an area whose pages + are all present in the CPU alias. + """ + import re as _re + + page = ent_va & ~0xFFF + need = (off + ln + 0xFFF) // 0x1000 + if ln <= 0 or ln > 0x40000 or need > 64: + return False + areas = list(_nv_vmap_areas(kc)) + try: + for line in open("/proc/vmallocinfo"): + if "[nvidia]" not in line or "vmalloc" not in line: + continue + if "nv_vm_map_pages" in line or "phys=" in line: + continue + if "os_map_kernel_space" in line or "ioremap" in line: + continue + m = _re.match(r'0x([0-9a-f]+)-0x([0-9a-f]+)\s+(\d+)', line) + if not m: + continue + lo = int(m.group(1), 16) + hi = int(m.group(2), 16) + pages = (hi - lo) >> 12 + if pages >= need and lo < hi: + areas.append((lo, hi, pages)) + except OSError: + pass + tried = 0 + for (lo, _hi, pages) in areas: + for adj in range(0, max(1, pages - need + 1)): + tried += 1 + if tried > 8192: + return False + cpu = lo + adj * 0x1000 + first = kc.read(cpu, 0x1000) + if not first: + continue + if any(first) and not _prefix_walk_ok(first, off): + continue + buf = _read_window(kc, cpu, off, ln) + if buf is None: + continue + acc, mode = _window_accept(buf, ln) + if not acc: + continue + pas = [] + for j in range(need): + vpa = kc.va_to_pa(cpu + j * 0x1000) + if vpa is None: + pas = None + break + pas.append(vpa) + if not pas: + continue + for j, vpa in enumerate(pas): + out.append("0x%x,0x%x,1\n" % (page + j * 0x1000, vpa)) + dbg("demand-va: 0x%x -> pa=0x%x (%s vmap parse %u B ok" + " pages=%u area=0x%x adj=%+d)" + % (page, pas[0], mode, ln, need, lo, adj)) + return True + dbg("demand-va: 0x%x strict vmap sweep failed (%u B, %u windows)" + % (page, ln, tried)) + return False + + +def resolve_va_strict(kc, ent_va, off, ln, out, pid=None): + """Demand pin validated by the host's exact full-parse of [ent_va, +ln). + + SESSION 153-CONTINUED-64: the content heuristic learned a GL family from + ONE page's weak 0x200-byte prefix test and then served every family page + from that (area, adj) with NO content check at all; r1120 pinned + 0x4001000 -> a page whose bytes (`00040000 00001000...`) failed the + host's stream parse for the whole boot (`plausible=0 unreadable=1`), so + the setup push was never captured and the prelude gate deferred every + launch. This path runs the HOST'S OWN full parse on the exact bytes the + host will read before any pin is emitted, then pins EVERY page of the + validated window and learns the family from it. A family is never + served from an unvalidated learn. + """ + page = ent_va & ~0xFFF + gl_base = next((lo for lo, hi in PB_GL_FAMILIES if lo <= page < hi), None) + if ln <= 0 or ln > 0x40000: + return False + if gl_base is None: + # SESSION 153-CONTINUED-64e: an anchor outside the known families + # (r1124: the hold moved to a 153600-byte push at 0x3200000) has no + # identity page index; only the client-mapping sweep can resolve it. + # + # SESSION 153-CONTINUED-74: the unknown-family branch skipped the + # kernel-vmap sweep entirely - the known-family path has always tried + # it, and it is the resolver that found the r1126 0x3200000 macro + # window (a KERNEL-mapped allocation no process pagemap covers). The + # r1156 held push (va=0x3e4ccccc, 123916 B) failed here after only + # the client-mem attempt ("strict unknown-family client-mem failed"), + # so a kernel-resident window could never be found for any anchor + # outside PB_GL_FAMILIES. Try the vmap sweep first (same full-parse + # gate; strictly additive - a failed sweep costs the bounded reads + # and falls through to the client-mem path unchanged). + try: + if _strict_kvm_sweep(kc, ent_va, off, ln, out): + return True + except Exception: + pass + try: + if resolve_va_client_mem(pid, ent_va, off, ln, out): + return True + except Exception: + pass + dbg("demand-va: 0x%x strict unknown-family (vmap+client-mem) failed" + % page) + return False + gl_hi = next(hi for lo, hi in PB_GL_FAMILIES if lo <= page < hi) + k = (page - gl_base) >> 12 + need = (off + ln + 0xFFF) // 0x1000 + if need > 64 or page + need * 0x1000 > gl_hi: + return False + areas = _nv_vmap_areas(kc) + cands = [] + fam = _gl_fam.get(gl_base) + if fam is not None and any(a_lo == fam[0] for (a_lo, _h, _p) in areas): + cands.append(fam) + if k == 0: + # SESSION 153-CONTINUED-65: a family-BASE anchor (the 0x3200000 + # macro-upload push) has no identity page index to key on. The + # r1126 macro window lives in a KERNEL-mapped allocation (no + # process pagemap maps it: the whole-process reverse scan found + # none), so the kcore vmap sweep is the only resolver - and the + # power-of-two neighborhood missed the window's arbitrary page + # index. Sweep every page of every nv_vm_map_pages area; the + # _window_accept parse gates each candidate. + for (lo, _hi, pages) in areas: + for adj in range(0, max(1, pages - need + 1)): + c = (lo, adj) + if c not in cands: + cands.append(c) + else: + for (lo, _hi, pages) in areas: + for adj in _STRICT_ADJ: + c = (lo, adj) + if c not in cands: + cands.append(c) + tried = 0 + for (lo, adj) in cands: + pages = next(p for (a_lo, _h, p) in areas if a_lo == lo) + idx = k + adj + if idx < 0 or idx + need > pages: + continue + tried += 1 + if tried > (8192 if k == 0 else 128): + break + cpu = lo + idx * 0x1000 + first = kc.read(cpu, 0x1000) + if not first: + continue + # SESSION 153-CONTINUED-65: an all-zero first page is a legitimate + # push head - the r1126 macro-upload window (0x3200000, 153600 B) + # starts with a fully zero page (the method stream lands later in + # the window); the old `not any(first)` skip threw exactly that + # candidate away (the r1128 sweep tried 166 kernel-vmap windows and + # rejected the true one on this check). The prefix walk trivially + # passes an all-zero page; the full-window accept is the gate. + if any(first) and not _prefix_walk_ok(first, off): + continue + buf = _read_window(kc, cpu, off, ln) + if buf is None: + continue + acc, mode = _window_accept(buf, ln) + if not acc: + continue + pas = [] + for j in range(need): + vpa = kc.va_to_pa(cpu + j * 0x1000) + if vpa is None: + pas = None + break + pas.append(vpa) + if not pas: + continue + for j, vpa in enumerate(pas): + out.append("0x%x,0x%x,1\n" % (page + j * 0x1000, vpa)) + _area_off[lo] = adj + _gl_fam[gl_base] = (lo, adj) + dbg("demand-va: 0x%x -> pa=0x%x (%s parse %u B ok pages=%u" + " area=0x%x adj=%+d)" % (page, pas[0], mode, ln, need, lo, adj)) + return True + dbg("demand-va: 0x%x strict parse failed (%u B, %u windows tried)" + % (page, ln, tried)) + # SESSION 153-CONTINUED-64c: no kernel vmap held the content - the + # allocation is UVM sysmem visible only in the client's own mappings. + try: + if resolve_va_client_mem(pid, ent_va, off, ln, out): + return True + except Exception: + pass + dbg("demand-va: 0x%x strict client-mem parse failed too" % page) return False @@ -718,6 +1374,49 @@ def resolve_sem_pins(kc, sem, out): return resolve_mem_pins(kc, mem, out, default_pages=16) +def resolve_pool_va(kc, va, out, sems): + """Pool/waiter VA -> PA via tracker-sem -> page -> mem; NO content match. + + The word at POOL base + 4*idx is a slot in a UVM semaphore-pool page: a + sysmem uvm_rm_mem whose PA is only reachable through its kernel CPU + alias (the client pagemap has no present entry for it). Same kcore walk + the CE pushbuffer uses; the chain is shorter (sem+0 = page, page+0 = + mem). The page sits 16 MB above the CE push buffer base, outside + resolve_sem_pins' default_pages window, which is why the host saw + 'pending release starved' for it on every client's first seconds. + """ + page = va & ~0xFFF + if not (POOL_SEM_LO <= page < POOL_SEM_HI): + return False + for sem in list(sems): + if not is_kern_ptr(sem): + continue + try: + pg = kc.read64(sem) + if not is_kern_ptr(pg): + continue + mem = kc.read64(pg) + if not is_kern_ptr(mem): + continue + got = [] + resolve_mem_pins(kc, mem, got, default_pages=1, + va_lo=page, va_hi=page + 0x1000) + except Exception: + continue + for line in got: + try: + lva = int(line.split(',')[0], 16) + except Exception: + continue + if lva == page: + out.append(line) + dbg('pool-va: 0x%x -> %s (mem=0x%x)' + % (page, line.strip(), mem)) + return True + dbg('pool-va: 0x%x unresolved (%d known sems)' % (page, len(sems))) + return False + + def resolve_pb_pins(kc, pb, out): mem = kc.read64(pb + 8) if not mem: @@ -751,6 +1450,27 @@ def resolve_pb_pins(kc, pb, out): # hold the ring-entry pushes is chosen (content anchor, _mem_content_ok). PB_CE_LO = 0x120000000 PB_CE_HI = 0x121000000 +# SESSION 153-CONTINUED-55: GL push-buffer VA families. The GL client's +# setup/MME-prelude push buffers (observed hold va=0x4001004 len=60216) and +# the draw pushes (observed va=0x4021004) are separate nv_vm_map_pages +# allocations in two fixed windows; the same family-relative page index +# rule applies: k = (va - base) >> 12. +# SESSION 153-CONTINUED-63: the r1117 boot's setup push lives at +# va=0x4200110 len=12288 - a THIRD window (0x4200000..0x4210000) the two +# original families did not cover, so the content resolver answered +# nothing and the read lottery could never be won for it (the +# 0x4020000-family hold re-armed for its whole 20 s window). +PB_GL_FAMILIES = ((0x3200000, 0x3300000), (0x4000000, 0x4100000), + (0x4200000, 0x4210000), + (0x42000000, 0x43000000)) +# m165 (D-F1a): UVM semaphore-pool page range. The pool is a 4 KB sysmem +# uvm_rm_mem placed by the UVM arena right after the 16 MB CE push buffer; +# the client's tracker words (0x121000000 + 4*idx) live in its first page. +# The client pagemap has no present entry for it, so only the kcore +# tracker-sem -> page -> mem walk can translate it. +POOL_SEM_LO = 0x121000000 +POOL_SEM_HI = 0x122000000 +KC_POOL_REEMIT_S = 2.0 # re-assert the pool page pair this often too KC_CHAIN_BACK = 0x2000 KC_CHAIN_SPAN = 0xA000 KC_CHAIN_MAX = 400 @@ -1331,13 +2051,49 @@ def nvidia_fds(pid): return kind +def pid_starttime(pid): + """Starttime (clock ticks, field 22) of /proc/<pid>, or None. + + Part of the client identity (pid, starttime): pids are recycled, so a + per-client reset keyed on the pid alone can mis-fire on a recycled pid. + Parsed after the last ')' so comm names with spaces/parens are safe.""" + try: + with open('/proc/%d/stat' % pid) as f: + st = f.read() + rp = st.rfind(')') + if rp < 0: + return None + toks = st[rp + 2:].split() + if len(toks) < 20: # starttime = field 22 overall = index 19 + return None + return int(toks[19]) + except (OSError, ValueError): + return None + + +def pid_line(tag, pid, start): + """'#<tag> <pid> [starttime]' - the optional starttime keeps the line + backward compatible with older hosts (they parse the pid only).""" + if start is None: + return "#%s %d\n" % (tag, pid) + return "#%s %d %d\n" % (tag, pid, start) + + def cdt_pid(): prefixes = tuple(p for p in os.environ.get("CLIENT_PREFIXES", - "cdt,va,victory").split(",") if p) + "cdt,va,victory,Xorg,X,glxgears," + "gzdoom").split(",") if p) # Processes that hold a /dev/nvidia* fd but are not CUDA clients; the # scanner must never commit to them (a transient nvidia-smi at boot was # picked once and its exited pagemap read back EOF forever). - non_clients = ("nvidia-smi", "nvidia-persistenced", "Xorg", "X") + # + # SESSION 146: Xorg/X used to be excluded here too, but the GL/GZDOOM + # path needs the X server's pages pinned - Xorg screen init fails with + # "Failed to initialize DMA / Failed to allocate push buffer" when the + # nvidia-push buffers cannot be resolved (observed r816). Xorg is a + # long-lived process (unlike the nvidia-smi transient the exclusion + # guarded against), so it is a valid, stable pin target. + non_clients = ("nvidia-smi", "nvidia-persistenced") cands = [] for d in os.listdir('/proc'): if not d.isdigit(): @@ -1449,6 +2205,18 @@ def scan_vmas(fd, pid, out): parts = line.split() if len(parts) < 5: continue + # Skip mappings with no read permission: a CUDA device buffer + # is a PROT_NONE reservation (e.g. `---p 00000000 00:00 0`), + # potentially hundreds of MB. Its pages are never present in + # the pagemap, so scanning it produces no pairs - but it + # consumed the VMA_MAX_PAGES budget and the scan never reached + # the client's present staging buffers that sit ABOVE the + # device reservation (observed: a client whose device range + # started below its staging buffer got no pins at all and its + # H2D copies failed with rc=-2 forever). + perms = parts[1] if len(parts) >= 2 else '' + if 'r' not in perms: + continue # Anonymous mappings (no path field) are the client's UVM sysmem # buffers when the driver maps them without a /dev/zero backing: # the second client's cudaMalloc'd data buffers show up exactly @@ -1550,16 +2318,35 @@ def main(): except OSError: pass last_pid = None + last_start = None last_log = 0.0 scrub_next = 0.0 demand_next = 0.0 - _want_state = {"mtime": None, "done": set()} + # m165 (D-F1a): pool page VAs learned from demand, re-asserted every + # KC_POOL_REEMIT_S. Boot-stable (UVM frees pool pages only at GPU + # deinit), so the set deliberately persists across client switches. + pool_pages = {} + pool_reemit_next = 0.0 + _want_state = {"mtime": None, "done": set(), "strict_pending": {}, + "strict_try": {}} loop = 0 while True: pid = cdt_pid() loop += 1 if pid is None: + if last_pid is not None and sock is not None: + # In-band client-death signal: the host tools purge the dead + # client's per-client state on this line, so the NEXT client + # works without a VM cycle (the kill cascade). Best-effort - + # if the socket is gone the host still has its own probe. + try: + sock.sendall(pid_line("nopid", last_pid, + last_start).encode()) + except OSError: + sock.close() + sock = None last_pid = None + last_start = None seen = {} seen_sems = set() seen_pbs = set() @@ -1568,6 +2355,7 @@ def main(): continue if pid != last_pid: last_pid = pid + last_start = pid_starttime(pid) seen = {} seen_sems = set() seen_pbs = set() @@ -1598,7 +2386,7 @@ def main(): dbg("switch seed: +%d event kptr(s)" % _seeded) if sock is not None: try: - sock.sendall(("#pid %d\n" % pid).encode()) + sock.sendall(pid_line("pid", pid, last_start).encode()) except OSError: sock.close() sock = None @@ -1646,23 +2434,203 @@ def main(): st_d = os.stat("/tmp/guest_want_vas.txt") if st_d.st_mtime != _want_state.get("mtime"): with open("/tmp/guest_want_vas.txt") as fw: - wvas = [int(t, 16) for t in fw.read().split() - if t.startswith("0x")] + # r582: tolerate malformed historical lines. The + # r581 boot's scanner died here: a transport-escaped + # append merged VAs into one token containing + # literal backslash-n ('0x20400f000\\n0x121000000\\n + # ...') and int(t, 16) raised ValueError out of + # main(), stopping ALL new pins (the observed + # pool-pin latency). Extract every 0x token instead; + # a bad token is skipped, never fatal. + text = fw.read() _want_state["mtime"] = st_d.st_mtime - for wv in wvas[-64:]: - if wv in _want_state["done"]: + # SESSION 153-CONTINUED-64: strict demand anchors first. + # A line `S 0x<entry_va> 0x<off> <len>` (written by + # pin_gr_stream from the service's anchored debugfs + # demand record) asks for the exact [entry_va, +len) + # read window to be validated with the host's own full + # stream parse before a pin is injected. Loose lines + # (plain `0x<va>`) keep the content/pagemap paths, but + # a GL family with a pending anchor is deferred so its + # learn can only come from the strict validation. + strict = [] + loose_tokens = [] + for line in text.splitlines(): + ms = re.match( + r"\s*S\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)" + r"\s+(\d+)\s*$", line) + if ms: + try: + strict.append((int(ms.group(1), 16), + int(ms.group(2), 16), + int(ms.group(3)))) + except ValueError: + pass + continue + loose_tokens += re.findall(r"0x[0-9a-fA-F]+", line) + now_d = time.time() + for epage in list(_want_state["strict_pending"]): + if now_d - _want_state["strict_pending"][epage] > 300.0: + del _want_state["strict_pending"][epage] + for (ev, eoff, eln) in reversed(strict): + epage = ev & ~0xFFF + if epage in _want_state["done"]: continue + # SESSION 153-CONTINUED-65: throttle anchor + # re-resolution. The full-offset sweep can read + # thousands of windows (tens of seconds of CPU); + # the demand file re-sends the anchor every ~2 s, + # so without this throttle the scanner starves the + # client (both measured at ~91% CPU at r1125). One + # attempt per anchor per window is enough - content + # that lands late is picked up on the next window. + lt = _want_state["strict_try"].get(epage, 0.0) + if now_d - lt < ANCHOR_RETRY_S: + continue + _want_state["strict_try"][epage] = now_d got = [] try: - ok = resolve_va_by_content(kc, wv, got) + ok = resolve_va_strict(kc, ev, eoff, eln, got, + pid) except Exception: ok = False got = [] + if not ok and eoff: + # The demand's off is the entry va's in-page + # offset; the push content can be page-aligned + # instead (r1123 vs r1125 parse behaviour). A + # second try at page alignment is cheap next to + # the completed sweep and the same pins result. + got2 = [] + try: + ok = resolve_va_strict(kc, ev & ~0xFFF, 0, + eln, got2, pid) + except Exception: + ok = False + if ok: + got = got2 + if ok and got: + lines.extend(got) + for g in got: + try: + _want_state["done"].add( + int(g.split(",", 1)[0], 16)) + except ValueError: + pass + _want_state["done"].add(epage) + _want_state["strict_pending"].pop(epage, None) + else: + _want_state["strict_pending"].setdefault( + epage, now_d) + # Loose VAs in a family whose strict anchor is still + # pending are deferred to the strict path only. + pend_bases = set() + for epage in _want_state["strict_pending"]: + for lo, hi in PB_GL_FAMILIES: + if lo <= epage < hi: + pend_bases.add((lo, hi)) + # SESSION 153-CONTINUED-55: process the NEWEST VAs first, + # not the deduped head. The file is append-only and grows + # to hundreds of lines (every stalled-VA row is appended + # each demand cycle, ~48/4 s under load), so both the + # literal tail (`wvas[-64:]`) and a first-occurrence + # dedupe sorted older pending VAs out of the window (the + # 16-page GL setup-push demand fell out between its + # appends and the next mtime change). Walk the raw tail + # backwards, newest instruction first, skipping resolved + # VAs, up to a bounded batch. + wvas = [] + for t in reversed(loose_tokens): + try: + v = int(t, 16) + except ValueError: + continue + if v not in _want_state["done"] and v not in wvas: + wvas.append(v) + if len(wvas) >= 256: + break + for wv in wvas: + if wv in _want_state["done"]: + continue + if any(lo <= (wv & ~0xFFF) < hi + for lo, hi in pend_bases): + continue + got = [] + if POOL_SEM_LO <= (wv & ~0xFFF) < POOL_SEM_HI: + # m165 (D-F1a): pool/waiter VAs have no method + # stream content and the CE offset math does not + # apply; route them to the structural tracker-sem + # walk instead. m168: the tracker sems are + # long-lived objects (allocated once) whose page + # pointer follows the pool across clients, so try + # every sem we have ever resolved a chain for - + # seen_sems alone can miss the live generation. + _psems = set(seen_sems) | set(KC_PB_STATE.keys()) \ + | set(resolved_kc_pb) + try: + ok = resolve_pool_va(kc, wv, got, + _psems) + except Exception: + ok = False + got = [] + dbg('demand-pool-va: 0x%x %s' + % (wv, 'ok' if ok else 'pending')) + if ok and got: + pool_pages.setdefault(wv & ~0xFFF, True) + else: + try: + ok = resolve_va_by_content(kc, wv, got) + except Exception: + ok = False + got = [] + # r586: a demanded VA outside the CE + # pushbuffer window (the GRFILL staging pages, + # high client VAs) is a plain client mapping + # page: resolve it through the client's own + # pagemap. Without this the demand for those + # VAs was a silent no-op (resolve_va_by_content + # rejects anything outside its windows) and the + # fills kept dropping them. + if not ok and not got and pid: + try: + pfd = os.open('/proc/%d/pagemap' % pid, + os.O_RDONLY) + try: + sub = {} + vpg = wv & ~0xFFF + scan_range(pfd, vpg, vpg + 0x1000, + sub) + finally: + os.close(pfd) + if sub: + for v, p in sub.items(): + got.append('0x%x,0x%x,1\n' + % (v, p)) + ok = True + except OSError: + pass if ok and got: lines.extend(got) _want_state["done"].add(wv) except OSError: pass + # m165 (D-F1a): re-assert the pool page pair every KC_POOL_REEMIT_S. + # The pool page is boot-stable (UVM frees it only at GPU deinit) but + # the host pin table is wiped at every session flush, so a one-shot + # pin leaves later clients starving until the slow structural walks + # answer; the periodic re-assert heals the wipe within seconds (the + # same argument as the CE mem sweep above). + if kc_ok and pool_pages and time.time() >= pool_reemit_next: + pool_reemit_next = time.time() + KC_POOL_REEMIT_S + for pv in list(pool_pages): + got = [] + try: + ok = resolve_pool_va(kc, pv, got, seen_sems) + except Exception: + ok = False + got = [] + if ok and got: + lines.extend(got) # PMA scrubber buffer discovery (throttled): resolve the finish/host- # semaphore word PAs and stream them for host registration. if kc_ok and time.time() >= scrub_next: @@ -1914,7 +2882,8 @@ def main(): sock.connect((CID, PORT)) sock.settimeout(None) if last_pid is not None: - sock.sendall(("#pid %d\n" % last_pid).encode()) + sock.sendall(pid_line("pid", last_pid, + last_start).encode()) except OSError: if sock: sock.close() |