From 50e293e39fc322fce3221c01a2a7219d49f618d1 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 25 Oct 2026 17:41:03 -0400 Subject: wall2 session 115: decompose the first-fence wall - the sema service cannot decode the submit's GR pushes because the client VAS page directory is undiscoverable (walk_probe: discovered pdb=0x0; the COPY_SERVER_PDES candidate is garbage; the r496f vidmem discovery never runs) and the pushbuffer is noncontiguous; the RAM ring scan gives up after 12 slices before the pushes exist; ring pages ARE content-locatable by their GPFIFO entries and injectable via sema_map_add (fence_assist.py) but the ring pair alone does not unlock the push decode; next = extract the pdb from the guest kernel's RM objects (PDB injection) or scan vidmem with the ring pair as verifier --- tools/guest-diag/wall2/README-session77.md | 48 +++++++++++++ tools/guest-diag/wall2/fence_assist.py | 112 +++++++++++++++++++++++++++++ tools/guest-diag/wall2/find_push.py | 59 +++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 tools/guest-diag/wall2/fence_assist.py create mode 100644 tools/guest-diag/wall2/find_push.py (limited to 'tools/guest-diag') diff --git a/tools/guest-diag/wall2/README-session77.md b/tools/guest-diag/wall2/README-session77.md index 1834c29..d830bef 100644 --- a/tools/guest-diag/wall2/README-session77.md +++ b/tools/guest-diag/wall2/README-session77.md @@ -1145,3 +1145,51 @@ complete the fence when it consumes the submit's GPFIFO entries); (2) then the first VKFRAME seq/DONE + PPM; (3) fold the COMPR flip into a host-side answer (see D); (4) the mission series (Xorg/GZDOOM, KVMFR, CUDA 15/15 + soak, test series, video, Windows). + +**40. Session 115: THE FENCE WALL DECOMPOSED - THE CLIENT VAS PAGE DIRECTORY IS UNDISCOVERABLE AND THE PUSHES ARE NONCONTIGUOUS; RING PAGES ARE CONTENT-LOCATABLE AND INJECTABLE, BUT THE RING PAIR ALONE DOES NOT UNLOCK THE PUSH DECODE.** +(1) The fence timeout is upstream of everything else: the sema service cannot +decode the submit's GR pushes (`sema-entry: unread ... va=0x4ca0000 +len=1432` -> `skipping stale entry ... after 185-194 failed rescues`; +`work_gr submitted=0` - no GR forward ever starts). (2) The pump's push +reads resolve ONLY through the client VAS page directory (or pins). The +service has NO pdb for this client: `walk_probe` of the ring and push VAs +prints `discovered pdb=0x0 found=0` and no registered VASes; the r496f +vidmem pdb discovery never ran (it is only called from the RAM ring-scan +latch), and the r496j COPY_SERVER_PDES candidate (0x109b9c000) is garbage +(its content is not a page table). (3) THE RAM RING SCAN MISSES BY +CONSTRUCTION for this client: it sweeps [0, 6 GB) in 12 slices and gives +up (`found no candidate in 12 slice(s)`), but the sweep runs BEFORE the +client's pushes exist and never re-runs - while the ring page content is +provably in RAM (content-located below). (4) CONTENT LOCATION WORKS: the +GR ring page (ring VA 0x4cc0000, first entries {va=0x4ca0000 len=1432, +va=0x4ca0598 len=4}) was found in the guest RAM file by its first two GPFIFO +entries (at GPA 0x12C600000 then 0x12898c000 on later runs - per-run +allocations), and `sema_map_add` injection of the TRUE ring pair is +accepted (`INJECT va=0x4d20000 pa=0x13619d000 : ok`). BUT the pushes do not +become readable from the ring pair alone: the pushbuffer is NONCONTIGUOUS +(the window guess `ring_pa - 0x20000` reads zeros for both the GR and the +CE channels), the pushes are not CPU-mapped in the client process (the +ring GPA is absent from /proc/PID/pagemap of every rw-s nvidia mapping), +and a bottom-up search for the level-3 table (a 4 KB page whose index-2 +big/small entry maps into the ring's 2 MB window) found ZERO candidates in +RAM - the page tables are almost certainly in VIDMEM (r496f's note) or use +a shape the walker does not model. (5) The service's OWN successful read of +one CE init push (`sema-push: h=0xbeec0104 va=0x4d00000 ... rel=0x4b19b10 +pay=1 have_rel=1`, `ops_runs=0`) did NOT apply its release either - +whatever resolved that one read is not reproducible for the GR pushes. +NEW TOOLS: `fence_assist.py` (host: watches a `dmesg -w` capture for the +pump's entry decodes, content-locates the ring pages by their first two +entries in the RAM file, injects the pairs via sema_map_add - proven to +find and inject, but the ring pair alone is insufficient) and +`find_push.py` (guest: tests whether the client CPU-maps the ring GPA). +NEXT (ranked, concrete): (1) find the client VAS page directory: extract the +pdb from the GUEST kernel's RM objects (kcore walk of the VAS/GPU object +for this channel - the GPFIFO ring page content anchors the search), then +register it via the service's VAS table (a new sysfs/debugfs entry point +or the existing candidate path) - this is the "PDB injection" the mission +names; (2) or scan VIDMEM for the pdb with the ring pair as the verifier +(extend `svc_xvas_pdb_discover`'s trigger: call it from the ring-injection +path, not only from the RAM scan latch); (3) make the RAM ring scan re-arm +per channel with the pushes present (its 12-slice give-up is the reason +the auto-discovery never latches on this client); (4) then the submit +decodes, the release lands and `vkWaitForFences` returns -> VKFRAME. diff --git a/tools/guest-diag/wall2/fence_assist.py b/tools/guest-diag/wall2/fence_assist.py new file mode 100644 index 0000000..4ea0b0c --- /dev/null +++ b/tools/guest-diag/wall2/fence_assist.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# fence_assist.py - content-locate the client's ring pages in guest RAM from the +# service's own GPFIFO entry decodes (dmesg) and inject the TRUE VA->PA pairs +# via sema_map_add, so the pump's push reads (and its pushbuffer pairing) have +# a correct anchor. +import mmap, re, struct, subprocess, sys, threading, time + +F = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" + +def gpa_of(fo): + return fo if fo < 0x80000000 else fo - 0x80000000 + 0x100000000 + +def inject(va, pa, ln=0x1000, flag=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va & ~0xfff, pa & ~0xfff, ln, flag, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print("INJECT va=0x%x pa=0x%x : %s" % (va & ~0xfff, pa & ~0xfff, + "ok" if r.returncode == 0 else "FAIL"), flush=True) + +def search(mm, pats, results): + CH = 8 << 20 + size = len(mm) + off = 0 + while off < size: + d = mm[off:off+CH] + for (name, pat) in pats.items(): + if name in results: + continue + i = d.find(pat) + if i >= 0: + results[name] = gpa_of(off + i) + off += CH - 32 + +def main(): + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 120 + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + end = time.time() + dur + pats = {} # name -> pattern bytes + ring_va = {} # name -> ring VA + seen = set() # raw entry lines processed + done = set() # injected names + print("# fence_assist start (%.0fs)" % dur, flush=True) + while time.time() < end: + try: + with open("/tmp/opencode/dmsg_w.txt", "r", errors="ignore") as df: + tail = df.read().splitlines()[-6000:] + out = "\n".join(tail) + except OSError: + out = "" + tail = [] + tail = [ln for ln in tail if "sema-chan:" in ln or "sema-push" in ln] + cur = {} + for ln in tail: + m = re.search(r"sema-chan: entry\[(\d+)\] h=0x([0-9a-f]+) eng=\d+ raw=([0-9a-f]+) va=0x([0-9a-f]+)", ln) + if m: + idx, h, raw, va = int(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4), 16) + cur.setdefault(h, {})[idx] = raw + if idx == 0: + ring_va[h] = va # entry[0] va is the push VA (ring_va - 0x20000) + for h, ents in cur.items(): + key = (h, tuple(sorted(ents.items()))) + if key in seen or len(ents) < 2: + continue + seen.add(key) + name = h + pat = b"".join(struct.pack("= 0: + results[nm] = gpa_of(off + i) + off += CH - 32 + for k in range(4): + a = 0x80000000 + k * (0x80000000 // 4) + b = 0x80000000 + (k + 1) * (0x80000000 // 4) + t = threading.Thread(target=worker, args=(a, b)) + t.start(); threads.append(t) + for t in threads: t.join() + for nm, gpa in results.items(): + if nm in done: continue + done.add(nm) + # ring page content starts with entry[0]; the ring VA is the + # channel's gpfifo VA = we reconstruct: entry0 va is the push, + # ring = first "gpfifo=0x..." dmesg line for this h + m = re.search(r"sema-chan: h=0x%s eng=\d+ cid=\d+ gpfifo=0x([0-9a-f]+)" % nm, out) + rva = int(m.group(1), 16) if m else None + if rva: + print("RING FOUND h=0x%s gpfifo_va=0x%x -> gpa=0x%x" % (nm, rva, gpa), flush=True) + inject(rva, gpa, 0x1000) + time.sleep(0.6) + print("# fence_assist done, injected %d" % len(done), flush=True) + +main() diff --git a/tools/guest-diag/wall2/find_push.py b/tools/guest-diag/wall2/find_push.py new file mode 100644 index 0000000..4bf8ccc --- /dev/null +++ b/tools/guest-diag/wall2/find_push.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# find_push.py - locate the client's CPU mapping that owns the GR ring page +# (GPA 0x12C600000 as content-located on the host) and print the CPU VA<-> +# GPU VA correspondence for the low GPU VAs (0x4ca0000 etc). +import os, struct, sys, subprocess, mmap, time + +RING_GPA = 0x12C600000 +PUSH_GVA = 0x4ca0000 +RING_GVA = 0x4cc0000 + +def pids(): + out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout + return [int(x) for x in out.split()] + +def read_pagemap(pid, va): + try: + with open("/proc/%d/pagemap" % pid, "rb") as f: + f.seek((va // 4096) * 8) + d = f.read(8) + if len(d) < 8: + return None + e = int.from_bytes(d, "little") + if not (e >> 63) & 1: + return None + return (e & ((1 << 55) - 1)) << 12 + except OSError: + return None + +def main(): + for pid in pids(): + print("pid=%d" % pid) + try: + maps = open("/proc/%d/maps" % pid).read().splitlines() + except OSError: + continue + for line in maps: + p = line.split() + if len(p) < 6 or "nvidia" not in p[-1]: + continue + lo, hi = [int(x, 16) for x in p[0].split("-")] + if hi - lo > 0x1000000: + continue + # scan pages; look for RING_GPA + for va in range(lo, hi, 0x10000): + for off in range(0, 0x10000, 0x1000): + pa = read_pagemap(pid, va + off) + if pa == RING_GPA: + print("FOUND ring page: cpu_va=0x%x (map 0x%x-0x%x %s)" % (va+off, lo, hi, p[-1])) + delta_cpu = (va + off) - RING_GVA + push_cpu = PUSH_GVA + delta_cpu + print(" delta_cpu = 0x%x" % delta_cpu) + for gva in (PUSH_GVA, PUSH_GVA+0x1000, 0x4ca059c, 0x4ca0980, 0x4ca5870, 0x4903000): + cva = gva + delta_cpu + pa2 = read_pagemap(pid, cva & ~0xfff) + print(" gva=0x%x -> cpu=0x%x -> pa=%s" % (gva, cva, hex(pa2) if pa2 else None)) + return + print(" ring GPA 0x%x not found in this pid's mappings" % RING_GPA) + +main() -- cgit v1.2.3