From 9813c9b9b6d9fa7e56fe0e77ccd4589fd9e90902 Mon Sep 17 00:00:00 2001
From: Arthur Hanson Rasmusson <arthur@vgpu.io>
Date: Sun, 25 Oct 2026 05:57:04 -0400
Subject: wall-2 session 74: the client's 11s stalls are userspace
 completion-slot spins; the next wall is the polled RM control 0x2080a097

Root-caused the vk_frame client's exact 11.00s gaps: a pure sched_yield +
clock_gettime userspace spin inside libnvidia-eglcore's EGL/GL init, waiting
for completion slots (u16 at slot+0xe == 0xffff) in /dev/nvidiactl mappings.
Guest-side filling of the slot flags lets the client proceed immediately, but
vkCreateDevice still fails -4: with the slots filled the client settles into
an NVOS54 poll of RM control 0x2080a097 (paramsSize=1168) which our emulation
answers with the generic all-zero reply, so the poll never completes.

Also captured: sema_push_raw dumps proving the GR channel-init push contains
NO release (entry0 decodes 2 methods then zeros; entry2 SET_OBJECT 0xc597 then
zeros), and the negative result that neither channelFillGpFifo nor
nvWriteGpEntry runs for this channel's kick.

New probes: gdbpoll6/gdbu16/gdbfill/gdbwatch/gdbparam/gdbgap3/gdbspin (slot
check, flag fill, NVOS54 param dump, backtraces), slotfill{,2}.sh (guest slot
watcher), stracevk/fastrun{,2}.sh (strace+perf recipes), find_ring_va.py and
vmap_watch2.py (vmap/kcore watchers), README-session74.md.
---
 tools/guest-diag/wall2/vmap_watch2.py | 73 +++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 tools/guest-diag/wall2/vmap_watch2.py

(limited to 'tools/guest-diag/wall2/vmap_watch2.py')

diff --git a/tools/guest-diag/wall2/vmap_watch2.py b/tools/guest-diag/wall2/vmap_watch2.py
new file mode 100644
index 0000000..194b0d6
--- /dev/null
+++ b/tools/guest-diag/wall2/vmap_watch2.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# Guest-side live watcher v2: scan ALL vmallocinfo areas via /proc/kcore for
+# (a) GR ring entry patterns, (b) pages starting with a small INC_METHOD header.
+import struct, time
+ENTRIES = [bytes.fromhex("0000480400980500"), bytes.fromhex("8000040004480598"), bytes.fromhex("0003d0000448059c")]
+class Kcore:
+    def __init__(self):
+        self.fd = open("/proc/kcore", "rb")
+        hdr = self.fd.read(64)
+        e_phoff = struct.unpack_from("<Q", hdr, 32)[0]
+        e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54)
+        self.segs = []
+        for i in range(e_phnum):
+            self.fd.seek(e_phoff + i * e_phentsize)
+            ph = self.fd.read(56)
+            p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from("<IIQQQQ", ph, 0)
+            if p_type == 1:
+                self.segs.append((p_va, p_va + p_sz, p_off))
+    def read(self, va, n):
+        for lo, hi, off in self.segs:
+            if lo <= va and va + n <= hi:
+                self.fd.seek(off + (va - lo))
+                return self.fd.read(n)
+        return None
+kc = Kcore()
+def areas():
+    out = []
+    for line in open("/proc/vmallocinfo"):
+        p = line.split()
+        if len(p) < 2 or "-" not in p[0]: continue
+        try:
+            lo, hi = p[0].split("-")
+            lo, hi = int(lo,16), int(hi,16)
+        except Exception:
+            continue
+        out.append((lo, hi, line.strip()[-60:]))
+    return out
+def hdr_ok(page):
+    v = struct.unpack_from("<I", page, 0)[0]
+    op = (v >> 29) & 7
+    cnt = (v >> 16) & 0xfff
+    return op == 1 and 1 <= cnt <= 8
+print("vmap_watch2 start", flush=True)
+found = set()
+t0 = time.time()
+it = 0
+while time.time() - t0 < 300:
+    it += 1
+    nareas = 0
+    for lo, hi, tag in areas():
+        size = hi - lo
+        if size > 8 << 20: continue
+        nareas += 1
+        d = kc.read(lo, size)
+        if not d: continue
+        for pat in ENTRIES:
+            i = d.find(pat)
+            if i >= 0:
+                key = (lo, i & ~0xfff, pat.hex())
+                if key not in found:
+                    found.add(key)
+                    print("%.2f RING %s VA 0x%x (+0x%x) %s" % (time.time()-t0, pat.hex(), lo+(i&~0xfff), i&~0xfff, tag), flush=True)
+        for off in range(0, size - 0x1000, 0x1000):
+            page = d[off:off+0x1000]
+            if hdr_ok(page):
+                key = (lo, off, "h")
+                if key not in found:
+                    found.add(key)
+                    print("%.2f HDR VA 0x%x (+0x%x) %s head=%s" % (time.time()-t0, lo+off, off, tag, page[:0x30].hex()), flush=True)
+    if it % 10 == 0:
+        print("%.1f iter %d areas %d" % (time.time()-t0, it, nareas), flush=True)
+    time.sleep(0.2)
+print("vmap_watch2 done", flush=True)
-- 
cgit v1.2.3