From 2954ee131bed4f0fa6aacae46bb6f987799135e2 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 25 Oct 2026 10:38:34 -0400 Subject: wall2: session 88 - push page-5 injection fix (proven +30/sweep), host_resolve tool, RING GAP diagnosis (gpfifo done=0), gdb-SIGKILL RM-lock wedge lesson --- tools/guest-diag/wall2/README-session77.md | 47 ++++++++++++++++ tools/guest-diag/wall2/gr_walk.py | 58 +++++++++++++++++++ tools/guest-diag/wall2/gr_walk2.py | 89 ++++++++++++++++++++++++++++++ tools/guest-diag/wall2/host_resolve.py | 81 +++++++++++++++++++++++++++ 4 files changed, 275 insertions(+) create mode 100644 tools/guest-diag/wall2/gr_walk.py create mode 100644 tools/guest-diag/wall2/gr_walk2.py create mode 100644 tools/guest-diag/wall2/host_resolve.py (limited to 'tools/guest-diag/wall2') diff --git a/tools/guest-diag/wall2/README-session77.md b/tools/guest-diag/wall2/README-session77.md index 6040aff..22247d6 100644 --- a/tools/guest-diag/wall2/README-session77.md +++ b/tools/guest-diag/wall2/README-session77.md @@ -343,3 +343,50 @@ decode of the window page lists. The client (2632) blocks in os_schedule at the allocation — the unread later pushes are the suspected gate. + +## Session 88 addendum - the push-page fix, the gdb-kill wedge, the ring-mapping gap + +**1. PATH-5 RULE (the push pages).** A GR window's pushes do NOT live at +the window base page: the r689-family windows carry their method streams in +WINDOW PAGE 5 (va offsets 0x52d4/0x559c/0x586c/0x596c all live in +0x5000-0x5fff). Standing injections at the base page resolve pa=0x0 for the +pushes (the service's `sema-pushall ... pa=0x0` lines) and the channel +holds/skips entries as "GR no-capture". The fix: page PA = +match_filoff - (va & 0xfff) + 0x80000000 for a content match of the push's +first8 bytes constrained to the same in-page offset; then +`echo 0x,0x,0x1000,1 > .../sema_map_add`. Proven on r689: +channel beec0104's pushes at va=0x4d00000 (page -> PA 0x12f398000) went +from first8=00000000 to first8=20012000 0000c5c0, and `fast sweep` +applied 3281 -> 3809 (+30/sweep sustained). + +**2. host_resolve.py** automates 1: it polls dmesg for sema-pushall lines +with pa=0x0 and non-zero first8, searches /dev/shm/vgpu-guest-ram +(offset-constrained), and injects the page. The service's 6 s no-capture +hold is the race window - run it BEFORE the client and keep it polling. + +**3. NEVER SIGKILL AN ATTACHED GDB.** On r689, killing the stale t91.py +gdb (kill -9 on PID 5334) killed its tracees (client threads +2883/2884/2885/3048); one held the guest RM API rwsem write lock; the lock +leaked and every later RM call blocked (`[vkps] Update blocked for more +than 1228 seconds` in rmapiLockAcquire -> RmGssLegacyRpcCmd -> +_nv04ControlWithSecInfo). The t91 harness design is correct: `timeout 500` +SIGTERMs the gdb and it detaches cleanly. If a gdb must go, use SIGTERM +and wait; SIGKILL only when the guest is about to be cycled anyway. A +leaked kernel rwsem cannot be repaired live. + +**4. The r690 gap: the GR channel ring mapping.** Fresh cycle r690 (mdev +f5a796b0): every RM RPC of the client init is answered (GR ctx commit, +channel allocs class 0x9067 hObject 0xcef00000/0xcef00080, EVPOSTs), but +the service reports `gpfifo done=0 nomap=938 pushes=0` - it cannot read +any GPFIFO ring, so no entry decodes, no pushall line appears, and the +guest's completion poll loops on RM control 0x20800a70 until the ~10 s RM +timeout -> vkCreateDevice returns -4 (VK_ERROR_DEVICE_LOST). r689's +service had done=19108 (rings mapped) - so the ring mapping is the +missing ingredient for a fresh client in r690; candidate paths: the FIFO +alloc params (12 bytes, GPFIFO VA inside) are not logged; the scanner saw +the CE window (ipins 4592) but no GR ring this boot. Next: log the 0x9067 +alloc params (or scan the guest for the ring VA) and inject the ring page. + +**5. gr_walk.py / gr_walk2.py** are the guest-side kcore walkers (fresh-pass +memdesc discovery; 12 records found, kptrs readable in-pass) kept as the +alternate grounding path. diff --git a/tools/guest-diag/wall2/gr_walk.py b/tools/guest-diag/wall2/gr_walk.py new file mode 100644 index 0000000..cbdb02d --- /dev/null +++ b/tools/guest-diag/wall2/gr_walk.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# gr_walk.py - find the RM window memdesc records in the guest's own RAM +# (via /proc/kcore direct map) and walk their kernel page-list pointers in +# the SAME pass (addresses go stale otherwise). Prints VA->PA candidates. +import sys, struct +sys.path.insert(0, "/tmp") +import guest_pin_stream as gps + +PAT_SIZE = struct.pack("= 2: + print(" PA-like: %s" % ["0x%x" % w for w in npas[:16]]) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/gr_walk2.py b/tools/guest-diag/wall2/gr_walk2.py new file mode 100644 index 0000000..7a50bc8 --- /dev/null +++ b/tools/guest-diag/wall2/gr_walk2.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# gr_walk2.py - deeper walk: from the window memdesc records, follow the +# kernel pointers 2-3 levels and search for page-array records +# {vmap_cpu, flags, PA, 0x1000} (2+ consecutive) to recover the page lists. +import sys, struct +sys.path.insert(0, "/tmp") +import guest_pin_stream as gps + +PAT_SIZE = struct.pack("= 2: + out.append((i, run)) + i = j + else: + i += 8 + return out + +def main(): + kc = gps.Kcore() + if not kc.load(): + print("# kcore load failed"); return 1 + pov = kc.page_offset_base + # find memdesc records (fast direct-map scan) + found = [] + CH = 1 << 20 + pa = 0x100000000 + while pa < 0x180000000 and len(found) < 12: + d = kc.read(pov + pa, CH) + if d: + st = 0 + while True: + i = d.find(PAT_SIZE, st) + if i < 0: + break + st = i + 1 + if i + 16 <= len(d) and d[i+8:i+16] == PAT_FLAGS: + found.append((pa + i, d[i:i+0x40])) + pa += CH + print("# memdesc records: %d" % len(found)) + for (gpa, rec) in found: + size, flags = struct.unpack_from(" 0x%x: page-array at +0x%x: %s" % + (kp, p2, arr[0], ["0x%x" % x for x in arr[1][:12]])) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/host_resolve.py b/tools/guest-diag/wall2/host_resolve.py new file mode 100644 index 0000000..258d994 --- /dev/null +++ b/tools/guest-diag/wall2/host_resolve.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# host_resolve.py - host-side push resolver: read the service's sema-pushall +# lines (pa=0x0, non-zero first8), find the content page in +# /dev/shm/vgpu-guest-ram by the push's first8 bytes, and inject the page +# mapping via sema_map_add. Replaces the guest-side kcore walker: the +# service already tells us push VA + content prefix, the host RAM file has +# the bytes. +import mmap, os, re, subprocess, sys, time + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +BASE = 0x80000000 # file offset = pa - BASE + +def inject(va_page, pa): + cmd = "echo 0x%x,0x%x,0x1000,1 > %s" % (va_page, pa, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + ok = r.returncode == 0 + print("inject 0x%x -> 0x%x : %s" % (va_page, pa, "ok" if ok else r.stderr.decode().strip())) + return ok + +def main(): + seen = set() + f = open(RAM, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + injected = 0 + deadline = time.time() + (float(sys.argv[1]) if len(sys.argv) > 1 else 600) + while time.time() < deadline and injected < 40: + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + lines = [l for l in out.splitlines() if "sema-pushall" in l] + for l in lines[-800:]: + m = re.search(r"va=(0x[0-9a-f]+) pa=0x0 len=\d+ allow=\d trusted=\d first8=([0-9a-f]{8}) ([0-9a-f]{8})", l) + if not m: + continue + va = int(m.group(1), 16) + w0, w1 = int(m.group(2), 16), int(m.group(3), 16) + if (w0, w1) == (0, 0) and (w0 == 0 or w1 == 0): + pass + if w0 == 0 and w1 == 0: + continue + key = (va & ~0xFFF, w0, w1) + if key in seen: + continue + pat = w0.to_bytes(4, "little") + w1.to_bytes(4, "little") + inpage = va & 0xFFF + # search with the in-page offset constraint + start = 0 + hit = None + while True: + i = mm.find(pat, start) + if i < 0: + break + if (i & 0xFFF) == inpage: + hit = i + break + start = i + 1 + if hit is None: + # fallback: any hit within the same page offset modulo + start = 0 + while True: + i = mm.find(pat, start) + if i < 0: + break + hit = i + break + # proceed only if the constraint is loose? no - skip + hit = None + if hit is None: + seen.add(key) + print("no match for va=0x%x first8=%08x %08x" % (va, w0, w1)) + continue + pa = (hit & ~0xFFF) + BASE + seen.add(key) + if inject(va & ~0xFFF, pa): + injected += 1 + time.sleep(0.5) + mm.close() + f.close() + print("done: injected=%d" % injected) + +if __name__ == "__main__": + main() -- cgit v1.2.3