gsp_sema_service: add content rescue for unmapped push-buffer pool VAs

- Add expected_sem_payload tracking to svc_channel for CE channels
- Implement svc_rescue_push_page() to scan guest RAM for push with
  expected semaphore release when VA translation fails
- Add svc_init_expected_payload() to initialize expected payload from pool
- Add rate-limited logging of distinct stalled VAs (svc_log_stalled_va)
- Advance expected_sem_payload on successful push decode
- Pushes SEMA_MAP_MAX to 65536, SVC_PIN_SLOTS to 32768 for scale

7 files changed, 779 insertions, 16 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 168e017..a203a66 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -137,6 +137,7 @@ struct svc_channel {
 	u64	notified;	/* work-submit-token notifications completed */
 	u32	last_get;	/* next GPFIFO entry we have still to run */
 	u64	submissions;	/* GPFIFO entries executed */
+	u32	expected_sem_payload;	/* next semaphore payload we expect for this channel's tracking sem */
 };
 
 /*
@@ -312,6 +313,11 @@ struct sema_svc {
 	u32			doorbell_token;		/* last token written */
 	u32			doorbell_log;		/* token log budget */
 	bool			doorbell_pending;	/* pump on next service pass */
+
+	/* Distinct stalled VA tracking for rate-limited logging. */
+#define SVC_STALLED_MAX		256
+	u64			stalled_vas[SVC_STALLED_MAX];
+	u32			stalled_va_count;
 };
 
 static struct sema_svc svc;
@@ -319,6 +325,31 @@ static struct sema_svc svc;
 /* Rolling RAM scan position (defined with the scan below; logged by rebuild) */
 static u64 svc_scan_cursor;
 
+/*
+ * Rate-limited logging of distinct stalled entry VAs.
+ * Tracks up to SVC_STALLED_MAX distinct VAs that fail translation.
+ */
+static void svc_log_stalled_va(u64 va)
+{
+	u32 i;
+
+	if (!va)
+		return;
+
+	/* Check if already logged. */
+	for (i = 0; i < svc.stalled_va_count; i++) {
+		if (svc.stalled_vas[i] == va)
+			return;
+	}
+
+	/* Add to tracked set. */
+	if (svc.stalled_va_count < SVC_STALLED_MAX) {
+		svc.stalled_vas[svc.stalled_va_count++] = va;
+		pr_info("sema-svc: stalled VA #%u: 0x%llx (total distinct: %u)\n",
+			svc.stalled_va_count, va, svc.stalled_va_count);
+	}
+}
+
 /* ------------------------------------------------------------------ */
 /* guest memory helpers                                                */
 /* ------------------------------------------------------------------ */
@@ -2623,6 +2654,121 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
 		ch->h_object, ch->ce_index, want, va, hit_pa, len);
 }
 
+/*
+ * Content rescue for unmapped push-buffer pool VAs.
+ * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the
+ * VA->PA map, we scan guest RAM for a page containing a push whose semaphore
+ * release matches this channel's tracking semaphore (slot = 4 * ce_index) and
+ * the expected next payload. If found, we pin the entire 1 MB push-buffer slot
+ * at the entry VA so the pump can execute the push.
+ */
+static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
+				struct svc_channel *ch, u64 entry_va, u32 entry_len)
+{
+	u8 *buf;
+	u64 gpa;
+	u32 slot = 4 * ch->ce_index;
+	u32 want_payload = ch->expected_sem_payload;
+	int found = 0;
+	u64 hit_pa = 0;
+
+	if (!ch->engine_type || ch->ce_index >= 32 || !want_payload)
+		return -ENOENT;
+
+	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	/*
+	 * Scan guest RAM for a push with the expected semaphore release.
+	 * The push format at the semaphore release (CE path) has:
+	 *   offset 0x10: hdr (count=3, addr_lo=0x90)
+	 *   offset 0x14: hi (== 1)
+	 *   offset 0x18: lo (slot in low 12 bits)
+	 *   offset 0x1c: payload
+	 */
+	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
+			u32 hdr, hi, lo, payload;
+
+			memcpy(&hdr, buf + o + 0x10, 4);
+			if ((((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) ||
+			    ((hdr & 0xff) != CE_SEMA_HDR_ADDR))
+				continue;
+			memcpy(&hi, buf + o + 0x14, 4);
+			memcpy(&lo, buf + o + 0x18, 4);
+			memcpy(&payload, buf + o + 0x1c, 4);
+			if (hi != 1)
+				continue;
+			if ((lo & 0xfff) != slot)
+				continue;
+			if (payload != want_payload)
+				continue;
+
+			hit_pa = (gpa + o) & ~(SEMA_PAGE_SIZE - 1);
+			found = 1;
+			break;
+		}
+		if (found)
+			break;
+	}
+	kfree(buf);
+
+	if (!found)
+		return -ENOENT;
+
+	/*
+	 * Pin the entire 1 MB push-buffer slot (16 pages) at the entry VA.
+	 * The entry_va is 1 MB aligned (push-buffer pool VAs are 1 MB aligned).
+	 */
+	if ((entry_va & 0xfffff) || entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI)
+		return -EINVAL;
+
+	for (u32 off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
+		svc_pin_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_lat_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_mark_hot(hit_pa + off);
+	}
+
+	pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (payload=%u)\n",
+		ch->h_object, ch->ce_index, slot, entry_va, entry_len, hit_pa, want_payload);
+
+	return 0;
+}
+
+/*
+ * Try to read the current semaphore value from the pool for this channel,
+ * so we can initialize expected_sem_payload to current + 1.
+ */
+static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
+				      struct svc_channel *ch)
+{
+	u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index;
+	u64 pa = 0;
+	int ap = SVC_APERTURE_SYS;
+	u32 cur = 0;
+
+	if (ch->expected_sem_payload)
+		return; /* already initialized */
+
+	if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap))
+		return; /* pool page not mapped yet */
+
+	if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
+			&cur, sizeof(cur)))
+		return;
+
+	if (cur > 0 && cur < 0x100000) {
+		ch->expected_sem_payload = cur + 1;
+		pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
+			ch->h_object, ch->ce_index, pool_va, cur, ch->expected_sem_payload);
+	}
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -2677,6 +2823,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			ch->logged = true;
 		}
 
+		/* Initialize expected semaphore payload from the pool (if mapped). */
+		svc_init_expected_payload(vgpu, ch);
+
 		if (put >= ch->gpfifo_entries)
 			put %= ch->gpfifo_entries;
 
@@ -2719,13 +2868,37 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								  len, 0);
 						ch->submissions++;
 						svc.gpfifo_entries_done++;
+						/* Advance expected payload for this channel's tracking semaphore. */
+						if (ch->engine_type && ch->expected_sem_payload)
+							ch->expected_sem_payload++;
 					} else {
-						/* Keep the entry pending: the
-						 * mapping may resolve once the
-						 * page tables are filled. */
-						svc.gpfifo_nomap++;
-						kfree(pb);
-						break;
+						/*
+						 * Content rescue: the push-buffer VA is not
+						 * in our map. Scan guest RAM for a push with
+						 * the expected semaphore release for this
+						 * channel's tracking semaphore, and pin it.
+						 */
+						svc_log_stalled_va(va);
+						if (!svc_rescue_push_page(vgpu, ch, va, len)) {
+							/* Rescue succeeded, retry the read. */
+							if (!svc_read_va_trusted(vgpu, va, pb,
+										 len)) {
+								svc_decode_stream(vgpu, pb,
+											  len, 0);
+								ch->submissions++;
+								svc.gpfifo_entries_done++;
+								if (ch->engine_type && ch->expected_sem_payload)
+									ch->expected_sem_payload++;
+							} else {
+								svc.gpfifo_nomap++;
+								kfree(pb);
+								break;
+							}
+						} else {
+							svc.gpfifo_nomap++;
+							kfree(pb);
+							break;
+						}
 					}
 					kfree(pb);
 				}
diff --git a/tools/guest-diag/memprobe.py b/tools/guest-diag/memprobe.py
new file mode 100644
index 0000000..4fe000b
--- /dev/null
+++ b/tools/guest-diag/memprobe.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+# memprobe.py - read/write another process's memory through /proc/<pid>/mem
+# (root only, no gdb/ptrace attach needed).
+#
+#   scan PID [--min 0x204800000 --max 0x204c0000] [--limit 8]
+#       find tracker objects: qwords inside [min,max) (the client's sem VAs),
+#       then objects whose +0x10 points at them (sem objects), then objects
+#       whose +0x9430 points at those (the libcuda tracker at p+0x9410).
+#   dump PID 0xP
+#       dump the tracker at 0xP (fields sem/mirror/completed, sem value,
+#       progress object at p+0x48 and its indexed slots).
+#   poke PID 0xADDR=0xVAL,...
+#   watch PID 0xADDR,... [--ms 2] [--secs 20]
+import sys, os, time, array, struct
+
+WS = 8
+
+def rd(fd, addr, n):
+    try:
+        return os.pread(fd, n, addr)
+    except OSError:
+        return None
+
+def wr(fd, addr, data):
+    return os.pwrite(fd, data, addr)
+
+def r32(buf, off=0):
+    return struct.unpack_from("<I", buf, off)[0]
+def r64(buf, off=0):
+    return struct.unpack_from("<Q", buf, off)[0]
+
+def regions(pid, writable=True):
+    out = []
+    with open("/proc/%d/maps" % pid) as f:
+        for line in f:
+            p = line.split()
+            try:
+                lo, hi = [int(x, 16) for x in p[0].split("-")]
+            except Exception:
+                continue
+            perm = p[1]
+            if writable and "w" not in perm:
+                continue
+            out.append((lo, hi, perm, " ".join(p[5:])))
+    return out
+
+def scan_qwords(pid, fd, lo, hi, needle_lo=None, needle_hi=None, needle_set=None, chunk=1 << 20):
+    hits = []
+    for rlo, rhi, perm, name in regions(pid):
+        a = max(rlo, 0)
+        while a < min(rhi, 1 << 47):
+            n = min(chunk, rhi - a)
+            buf = rd(fd, a, n)
+            if not buf:
+                a += n
+                continue
+            buf = buf[:n - (n % WS)]
+            arr = array.array("Q")
+            arr.frombytes(buf)
+            if needle_set is not None:
+                for i, v in enumerate(arr):
+                    if v in needle_set:
+                        hits.append((a + i * WS, v))
+            else:
+                for i, v in enumerate(arr):
+                    if needle_lo is not None and v < needle_lo:
+                        continue
+                    if needle_hi is not None and v >= needle_hi:
+                        continue
+                    hits.append((a + i * WS, v))
+            a += n
+    return hits
+
+def sem_objs(pid, fd, lo, hi):
+    # qword in [lo,hi) at address A -> a sem object holds it at +0x10
+    first = scan_qwords(pid, fd, None, None, needle_lo=lo, needle_hi=hi)
+    out = {}
+    for addr, val in first:
+        so = addr - 0x10
+        out.setdefault(val, set()).add(so)
+    return out
+
+def find_trackers(pid, fd, lo, hi, limit=8):
+    sems = sem_objs(pid, fd, lo, hi)
+    print("sem VAs found: %d" % len(sems))
+    for va, sos in sorted(sems.items())[:8]:
+        print("  va=0x%x -> %d sem-obj candidate(s)" % (va, len(sos)))
+    needles = set()
+    for sos in sems.values():
+        needles |= sos
+    if not needles:
+        return []
+    hits = scan_qwords(pid, fd, None, None, needle_set=needles)
+    pset = set()
+    for addr, val in hits:
+        cand = addr - 0x9430
+        if cand >= 0:
+            pset.add(cand)
+    print("tracker candidates: %d" % len(pset))
+    res = []
+    for p in sorted(pset):
+        buf = rd(fd, p + 0x9410, 0x40)
+        if not buf or len(buf) < 0x40:
+            continue
+        sub = r64(buf, 0x08); last = r64(buf, 0x10); cur = r64(buf, 0x18); sem = r64(buf, 0x20)
+        if sem not in needles:
+            continue
+        mem = None
+        sob = rd(fd, sem, 0x40)
+        if sob:
+            mem = r64(sob, 0x10)
+        res.append((p, sub, last, cur, sem, mem))
+        if len(res) >= limit:
+            break
+    return res
+
+def dump_tracker(pid, fd, p):
+    t = p + 0x9410
+    buf = rd(fd, t, 0x40)
+    if not buf:
+        print("tracker unreadable at 0x%x" % t); return
+    print("TRK=0x%x sub=0x%x last=0x%x cur=0x%x sem_obj=0x%x"
+          % (t, r64(buf, 8), r64(buf, 0x10), r64(buf, 0x18), r64(buf, 0x20)))
+    sem = r64(buf, 0x20)
+    sob = rd(fd, sem, 0x40)
+    if sob:
+        print("  sem_obj[0:0x40]=%s" % sob.hex())
+        mem = r64(sob, 0x10)
+        mvb = rd(fd, mem & ~0xfff, 0x40)
+        print("  sem_mem=0x%x val@+0x%x=0x%x" % (mem, mem & 0xfff, r32(mvb, mem & 0xfff) if mvb else -1))
+    q = r64(rd(fd, p + 0x48, 8) or b"\0" * 8)
+    print("  prog_obj=0x%x next=0x%x cblist=0x%x idx95bc=0x%x p34=0x%x"
+          % (q, r64(rd(fd, p + 0x9470, 8) or b"\0" * 8),
+             r64(rd(fd, p + 0x9458, 8) or b"\0" * 8),
+             r32(rd(fd, p + 0x95bc, 4) or b"\0" * 4),
+             r32(rd(fd, p + 0x34, 4) or b"\0" * 4)))
+    if q:
+        qb = rd(fd, q, 0x50)
+        if qb:
+            size = r32(qb, 8); idx = r32(qb, 0x48); cur = r32(qb, 0x4c); arr = r64(qb, 0x40)
+            print("    size=%d idx=0x%x cursor=0x%x arr=0x%x" % (size, idx, cur, arr))
+            for s in (idx, cur):
+                slot = arr + (s % size if size else s) * 0x38
+                sb = rd(fd, slot, 0x38)
+                print("    slot[%d]@0x%x=%s" % (s, slot, sb.hex() if sb else "unreadable"))
+
+def main():
+    if len(sys.argv) < 3:
+        print(__doc__); return 1
+    mode = sys.argv[1]
+    pid = int(sys.argv[2])
+    fd = os.open("/proc/%d/mem" % pid, os.O_RDWR)
+    try:
+        if mode == "scan":
+            lo, hi = 0x204800000, 0x204c0000
+            limit = 8
+            for i, a in enumerate(sys.argv[3:]):
+                if a == "--min": lo = int(sys.argv[4 + i], 16)
+                if a == "--max": hi = int(sys.argv[5 + i], 16)
+                if a == "--limit": limit = int(sys.argv[4 + i])
+            res = find_trackers(pid, fd, lo, hi, limit)
+            for p, sub, last, cur, sem, mem in res:
+                print("P=0x%x sub=0x%x last=0x%x cur=0x%x sem_obj=0x%x sem_mem=0x%x" %
+                      (p, sub, last, cur, sem, mem or 0))
+                dump_tracker(pid, fd, p)
+        elif mode == "dump":
+            dump_tracker(pid, fd, int(sys.argv[3], 16))
+        elif mode == "hex":
+            a = int(sys.argv[3], 16)
+            n = int(sys.argv[4], 16) if len(sys.argv) > 4 else 0x100
+            d = rd(fd, a, n) or b""
+            for off in range(0, len(d), 16):
+                c = d[off:off + 16]
+                q = [int.from_bytes(c[i:i + 8], "little") for i in (0, 8)]
+                print("0x%x: %s %s" % (a + off, c.hex(), " ".join("0x%x" % x for x in q)))
+        elif mode == "ctx":
+            ctx = int(sys.argv[3], 16)
+            count = r32(rd(fd, ctx + 0x10, 4) or b"\0" * 4)
+            ent = r64(rd(fd, ctx + 0x18, 8) or b"\0" * 8)
+            print("waitobj ctx=0x%x count=%d entries=0x%x" % (ctx, count, ent))
+            for i in range(min(count, 48)):
+                e = ent + i * 0x28
+                eb = rd(fd, e, 0x28)
+                if not eb:
+                    print("  e%d unreadable" % i); continue
+                typ = r32(eb, 0); p = r64(eb, 8); v = r64(eb, 0x10)
+                x18 = r64(eb, 0x18); x20 = r64(eb, 0x20)
+                print("  e%d type=0x%x p=0x%x v=0x%x x18=0x%x x20=0x%x" % (i, typ, p, v, x18, x20))
+                if typ == 1 and p:
+                    dump_tracker(pid, fd, p)
+        elif mode == "poke":
+            for spec in sys.argv[3].split(","):
+                a, v = spec.split("=")
+                a = int(a, 16); v = int(v, 16)
+                n = wr(fd, a, v.to_bytes(8, "little"))
+                print("poke 0x%x <- 0x%x (%s)" % (a, v, "ok" if n == 8 else "FAIL"))
+        elif mode == "watch":
+            addrs = [int(x, 16) for x in sys.argv[3].split(",")]
+            ms = 2; secs = 20
+            for i in range(4, len(sys.argv)):
+                if sys.argv[i] == "--ms" and i + 1 < len(sys.argv):
+                    ms = int(sys.argv[i + 1])
+                if sys.argv[i] == "--secs" and i + 1 < len(sys.argv):
+                    secs = int(sys.argv[i + 1])
+            last = {}
+            t0 = time.time()
+            while time.time() - t0 < secs:
+                for a in addrs:
+                    b = rd(fd, a, 8)
+                    if not b:
+                        continue
+                    v = r64(b)
+                    if last.get(a) != v:
+                        print("%.3f 0x%x = 0x%x" % (time.time() - t0, a, v))
+                        last[a] = v
+                time.sleep(ms / 1000.0)
+        else:
+            print(__doc__); return 1
+    finally:
+        os.close(fd)
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())
\ No newline at end of file
diff --git a/tools/guest-diag/peek.c b/tools/guest-diag/peek.c
new file mode 100644
index 0000000..857afd0
--- /dev/null
+++ b/tools/guest-diag/peek.c
@@ -0,0 +1,67 @@
+/* peek.c - print RIP/RBX/RSP/RDI of every thread of a process (root).
+   usage: peek PID [lo_off hi_off]   (offsets from the libcuda r-xp base)
+   Flags threads whose RIP is inside [base+lo, base+hi) with SPIN. */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dirent.h>
+#include <sys/ptrace.h>
+#include <sys/wait.h>
+#include <sys/user.h>
+#include <sys/uio.h>
+#include <unistd.h>
+
+int main(int argc, char **argv)
+{
+    if (argc < 2) { fprintf(stderr, "usage: peek PID [lo_off hi_off]\n"); return 1; }
+    int pid = atoi(argv[1]);
+    unsigned long lo = 0x23f700, hi = 0x23fd10;
+    if (argc >= 4) { lo = strtoul(argv[2], 0, 16); hi = strtoul(argv[3], 0, 16); }
+
+    char path[256];
+    snprintf(path, sizeof path, "/proc/%d/maps", pid);
+    FILE *f = fopen(path, "r");
+    if (!f) { perror("maps"); return 1; }
+    unsigned long base = 0;
+    char line[512];
+    while (fgets(line, sizeof line, f)) {
+        if (strstr(line, "libcuda") && strstr(line, "r-xp")) {
+            unsigned long a, b, off; char perms[8], dev[16], name[256]; unsigned long ino;
+            if (sscanf(line, "%lx-%lx %7s %lx %15s %lu %255s",
+                       &a, &b, perms, &off, dev, &ino, name) >= 6) {
+                base = a - off;
+                break;
+            }
+        }
+    }
+    fclose(f);
+    printf("libcuda base=0x%lx\n", base);
+
+    snprintf(path, sizeof path, "/proc/%d/task", pid);
+    DIR *d = opendir(path);
+    if (!d) { perror("task"); return 1; }
+    struct dirent *e;
+    while ((e = readdir(d))) {
+        if (e->d_name[0] == '.') continue;
+        int tid = atoi(e->d_name);
+        if (ptrace(PTRACE_ATTACH, tid, 0, 0)) { printf("tid %d attach fail\n", tid); continue; }
+        int st;
+        waitpid(tid, &st, 0);
+        struct user_regs_struct r;
+        struct iovec io = { &r, sizeof r };
+        if (ptrace(PTRACE_GETREGSET, tid, (void *)1, &io)) {
+            printf("tid %d getregs fail\n", tid);
+            ptrace(PTRACE_DETACH, tid, 0, 0);
+            continue;
+        }
+        printf("tid %d rip=0x%llx rsp=0x%llx rbx=0x%llx rdi=0x%llx%s\n",
+               tid,
+               (unsigned long long)r.rip, (unsigned long long)r.rsp,
+               (unsigned long long)r.rbx, (unsigned long long)r.rdi,
+               (r.rip >= base + lo && r.rip < base + hi) ? " SPIN" : "");
+        ptrace(PTRACE_DETACH, tid, 0, 0);
+    }
+    closedir(d);
+    return 0;
+}
\ No newline at end of file
diff --git a/tools/guest-diag/scan.c b/tools/guest-diag/scan.c
new file mode 100644
index 0000000..d229dc8
--- /dev/null
+++ b/tools/guest-diag/scan.c
@@ -0,0 +1,105 @@
+/* scan.c - find the libcuda tracker objects in a process without ptrace.
+   Tracker layout (object p): p+0x9418 submitted, p+0x9420 completed,
+   p+0x9428 mirror, p+0x9430 sem object; sem object +0x10 -> sem VA (a
+   /dev/nvidia* mapping) whose u32 is small.
+
+   usage: scan PID   (prints candidate p and its fields) */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#define MAXR 4096
+static struct { uint64_t lo, hi; } wr[MAXR], nv[MAXR];
+static int nwr, nnv;
+
+static int in_nv(uint64_t a)
+{
+    for (int i = 0; i < nnv; i++)
+        if (a >= nv[i].lo && a < nv[i].hi) return 1;
+    return 0;
+}
+
+static int in_wr(uint64_t a)
+{
+    for (int i = 0; i < nwr; i++)
+        if (a >= wr[i].lo && a < wr[i].hi) return 1;
+    return 0;
+}
+
+static void load_maps(int pid)
+{
+    char path[64], line[1024];
+    snprintf(path, sizeof path, "/proc/%d/maps", pid);
+    FILE *f = fopen(path, "r");
+    if (!f) { perror("maps"); exit(1); }
+    while (fgets(line, sizeof line, f)) {
+        uint64_t lo, hi;
+        char perms[16], rest[768];
+        if (sscanf(line, "%lx-%lx %15s %*s %*s %*s %767[^\n]", &lo, &hi, perms, rest) < 3)
+            continue;
+        if (strchr(perms, 'w') && nwr < MAXR) { wr[nwr].lo = lo; wr[nwr].hi = hi; nwr++; }
+        if (strstr(line, "nvidia") && nnv < MAXR) { nv[nnv].lo = lo; nv[nnv].hi = hi; nnv++; }
+    }
+    fclose(f);
+}
+
+int main(int argc, char **argv)
+{
+    if (argc < 2) { fprintf(stderr, "usage: scan PID\n"); return 1; }
+    int pid = atoi(argv[1]);
+    load_maps(pid);
+    printf("writable regions=%d nvidia regions=%d\n", nwr, nnv);
+
+    char mp[64];
+    snprintf(mp, sizeof mp, "/proc/%d/mem", pid);
+    int fd = open(mp, O_RDONLY);
+    if (fd < 0) { perror("mem"); return 1; }
+
+    size_t CH = 4 << 20;
+    static uint8_t *buf;
+    buf = malloc(CH + 64);
+    int hits = 0;
+    for (int r = 0; r < nwr; r++) {
+        uint64_t a = wr[r].lo;
+        while (a < wr[r].hi) {
+            size_t n = (size_t)((wr[r].hi - a) < CH ? (wr[r].hi - a) : CH);
+            ssize_t got = pread(fd, buf, n, (off_t)a);
+            if (got <= 0) {  /* try smaller granularity */
+                size_t n2 = 256 << 10;
+                if (n > n2) { n = n2; continue; }
+                a += 4096; continue;
+            }
+            if ((size_t)got < n) {
+                /* partial read: scan what we got, then continue */
+                n = got;
+            }
+            n -= n % 8;
+            uint64_t *w = (uint64_t *)buf;
+            for (size_t i = 0; i + 4 <= n / 8; i++) {
+                uint64_t sub = w[i], last = w[i + 1], cur = w[i + 2], sem = w[i + 3];
+                if (sub == 0 || sub > 1000) continue;
+                if (last == 0 || last > 1000) continue;
+                if (cur > 0x200000000ULL) continue;
+                if (sem < 0x10000 || sem > 0x100000000ULL) continue;
+                uint64_t vaptr = 0;
+                if (pread(fd, &vaptr, 8, (off_t)(sem + 0x10)) != 8) continue;
+                if (!in_nv(vaptr)) continue;
+                uint32_t val = 0;
+                if (pread(fd, &val, 4, (off_t)vaptr) != 4) continue;
+                if (val > 1000) continue;
+                uint64_t p = a + i * 8 - 0x9418;
+                printf("P=0x%lx sub=%lu last=%lu cur=0x%lx sem_obj=0x%lx sem_va=0x%lx sem_val=%u\n",
+                       p, sub, last, cur, sem, vaptr, val);
+                hits++;
+            }
+            a += n;
+        }
+    }
+    printf("hits=%d\n", hits);
+    return 0;
+}
\ No newline at end of file
diff --git a/tools/guest-diag/scan2.c b/tools/guest-diag/scan2.c
new file mode 100644
index 0000000..6220038
--- /dev/null
+++ b/tools/guest-diag/scan2.c
@@ -0,0 +1,88 @@
+/* scan2.c - find qwords whose value points into a /dev/nvidia* mapping of
+   the target process, and report the u32 stored there (semaphore hunt).
+   usage: scan2 PID [max_print] */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#define MAXR 4096
+static struct { uint64_t lo, hi; char name[64]; } nv[MAXR], wr[MAXR];
+static int nnv, nwr;
+
+static int in_nv(uint64_t a, int *idx)
+{
+    for (int i = 0; i < nnv; i++)
+        if (a >= nv[i].lo && a < nv[i].hi) { if (idx) *idx = i; return 1; }
+    return 0;
+}
+
+static void load_maps(int pid)
+{
+    char path[64], line[1024];
+    snprintf(path, sizeof path, "/proc/%d/maps", pid);
+    FILE *f = fopen(path, "r");
+    if (!f) { perror("maps"); exit(1); }
+    while (fgets(line, sizeof line, f)) {
+        uint64_t lo, hi;
+        char perms[16], name[128] = "";
+        if (sscanf(line, "%lx-%lx %15s %*s %*s %*s %127[^\n]", &lo, &hi, perms, name) < 3)
+            continue;
+        if (strchr(perms, 'w') && nwr < MAXR) { wr[nwr].lo = lo; wr[nwr].hi = hi; nwr++; }
+        if (strstr(line, "nvidia") && nnv < MAXR) {
+            nv[nnv].lo = lo; nv[nnv].hi = hi;
+            snprintf(nv[nnv].name, sizeof nv[nnv].name, "%s", name);
+            nnv++;
+        }
+    }
+    fclose(f);
+}
+
+int main(int argc, char **argv)
+{
+    if (argc < 2) { fprintf(stderr, "usage: scan2 PID [max_print]\n"); return 1; }
+    int pid = atoi(argv[1]);
+    int maxp = argc > 2 ? atoi(argv[2]) : 200;
+    load_maps(pid);
+    printf("writable=%d nvidia=%d\n", nwr, nnv);
+
+    char mp[64];
+    snprintf(mp, sizeof mp, "/proc/%d/mem", pid);
+    int fd = open(mp, O_RDONLY);
+    if (fd < 0) { perror("mem"); return 1; }
+
+    size_t CH = 4 << 20;
+    uint8_t *buf = malloc(CH + 64);
+    int printed = 0, scanned = 0;
+    for (int r = 0; r < nwr; r++) {
+        uint64_t a = wr[r].lo;
+        while (a < wr[r].hi) {
+            size_t n = (size_t)((wr[r].hi - a) < CH ? (wr[r].hi - a) : CH);
+            ssize_t got = pread(fd, buf, n, (off_t)a);
+            if (got <= 0) {
+                if (n > (256u << 10)) { n = 256u << 10; continue; }
+                a += 4096; continue;
+            }
+            if ((size_t)got < n) n = got;
+            n -= n % 8;
+            uint64_t *w = (uint64_t *)buf;
+            for (size_t i = 0; i < n / 8; i++, scanned++) {
+                uint64_t v = w[i];
+                int idx;
+                if (!in_nv(v, &idx)) continue;
+                uint32_t val = 0;
+                if (pread(fd, &val, 4, (off_t)v) != 4) continue;
+                if (val > 0x100000) continue;
+                if (printed < maxp)
+                    printf("at 0x%lx -> 0x%lx (%s) val=%u\n", a + i * 8, v, nv[idx].name, val);
+                printed++;
+            }
+            a += n;
+        }
+    }
+    printf("scanned=%d printed=%d\n", scanned, printed);
+    return 0;
+}
\ No newline at end of file
diff --git a/tools/guest-diag/scan3.c b/tools/guest-diag/scan3.c
new file mode 100644
index 0000000..94f6a42
--- /dev/null
+++ b/tools/guest-diag/scan3.c
@@ -0,0 +1,77 @@
+/* scan3.c - scan all writable memory of a process for qwords equal to any
+   of the given 64-bit needles; print every hit address.
+   usage: scan3 PID 0xNEEDLE [0xNEEDLE ...] */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#define MAXR 4096
+#define MAXN 64
+static struct { uint64_t lo, hi; } wr[MAXR];
+static int nwr;
+static uint64_t needle[MAXN];
+static int nn;
+
+static void load_maps(int pid)
+{
+    char path[64], line[1024];
+    snprintf(path, sizeof path, "/proc/%d/maps", pid);
+    FILE *f = fopen(path, "r");
+    if (!f) { perror("maps"); exit(1); }
+    while (fgets(line, sizeof line, f)) {
+        uint64_t lo, hi;
+        char perms[16];
+        if (sscanf(line, "%lx-%lx %15s", &lo, &hi, perms) < 3) continue;
+        if (strchr(perms, 'w') && nwr < MAXR) { wr[nwr].lo = lo; wr[nwr].hi = hi; nwr++; }
+    }
+    fclose(f);
+}
+
+int main(int argc, char **argv)
+{
+    if (argc < 3) { fprintf(stderr, "usage: scan3 PID 0xNEEDLE [...]\n"); return 1; }
+    int pid = atoi(argv[1]);
+    for (int i = 2; i < argc && nn < MAXN; i++)
+        needle[nn++] = strtoull(argv[i], 0, 16);
+    load_maps(pid);
+    printf("writable=%d needles=%d\n", nwr, nn);
+
+    char mp[64];
+    snprintf(mp, sizeof mp, "/proc/%d/mem", pid);
+    int fd = open(mp, O_RDONLY);
+    if (fd < 0) { perror("mem"); return 1; }
+
+    size_t CH = 4 << 20;
+    uint8_t *buf = malloc(CH + 64);
+    int hits = 0;
+    for (int r = 0; r < nwr; r++) {
+        uint64_t a = wr[r].lo;
+        while (a < wr[r].hi) {
+            size_t n = (size_t)((wr[r].hi - a) < CH ? (wr[r].hi - a) : CH);
+            ssize_t got = pread(fd, buf, n, (off_t)a);
+            if (got <= 0) {
+                if (n > (256u << 10)) { n = 256u << 10; continue; }
+                a += 4096; continue;
+            }
+            if ((size_t)got < n) n = got;
+            n -= n % 8;
+            uint64_t *w = (uint64_t *)buf;
+            for (size_t i = 0; i < n / 8; i++) {
+                uint64_t v = w[i];
+                for (int k = 0; k < nn; k++) {
+                    if (v == needle[k]) {
+                        printf("hit 0x%lx = 0x%lx\n", a + i * 8, v);
+                        hits++;
+                    }
+                }
+            }
+            a += n;
+        }
+    }
+    printf("hits=%d\n", hits);
+    return 0;
+}
\ No newline at end of file
diff --git a/tools/guest-diag/trkinfo.py b/tools/guest-diag/trkinfo.py
index 46b6a69..0d6d629 100644
--- a/tools/guest-diag/trkinfo.py
+++ b/tools/guest-diag/trkinfo.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 # gdb script (run with: sudo gdb -p <pid> -batch -x /tmp/trkinfo.py)
-# Drills into the libcuda tracker the client spins on (the wait loop around
-# libcuda+0x23f700): dumps the tracker object at p+0x9410, the semaphore
-# object it references at +0x20, and the u32 semaphore memory that object
-# points at (+0x10), plus the VMA that memory lives in.
+# Drills into the libcuda tracker wait: dumps every wait entry, the tracker
+# objects they reference (channel/context layout at p+0x9410 with the sem
+# object at +0x9430), and the progress object at p+0x48 (size/idx/cursor and
+# the 0x38-byte slots it indexes).
 import gdb
 
 def r32(a):
@@ -66,18 +66,25 @@ if not found:
     raise SystemExit
 
 tid, pc, ctx = found
-print("SPINNER tid=%s pc=0x%x ctx=0x%x" % (tid, pc, ctx))
+print("SPINNER tid=%s pc=0x%x ctx=0x%x libbase=0x%x" % (tid, pc, ctx, base))
 count = r32(ctx + 0x10)
 ent = r64(ctx + 0x18)
 print("count=%d entries=0x%x" % (count, ent))
-n = min(count, 32)
+n = min(count, 48)
 for i in range(n):
     e = ent + i * 0x28
     typ = r32(e)
     p = r64(e + 8)
-    v = r32(e + 0x10)
+    v = r64(e + 0x10)
+    x18 = r64(e + 0x18)
+    x20 = r64(e + 0x20)
     if typ != 1:
-        print("e%d type=%d p=0x%x v=0x%x (skip)" % (i, typ, p, v))
+        print("e%d type=0x%x p=0x%x v=0x%x x18=0x%x x20=0x%x" % (i, typ, p, v, x18, x20))
+        if p:
+            try:
+                print("  obj[0:0x28]=%s" % rmem(p, 0x28).hex())
+            except Exception:
+                pass
         continue
     t = p + 0x9410
     try:
@@ -85,11 +92,12 @@ for i in range(n):
     except Exception as ex:
         print("e%d tracker unreadable: %s" % (i, ex))
         continue
+    sub = int.from_bytes(trk[0x08:0x10], "little")
     last = int.from_bytes(trk[0x10:0x18], "little")
     cur = int.from_bytes(trk[0x18:0x20], "little")
     sem = int.from_bytes(trk[0x20:0x28], "little")
-    print("e%d p=0x%x v=0x%x TRK=0x%x last=0x%x cur=0x%x sem_obj=0x%x"
-          % (i, p, v, t, last, cur, sem))
+    print("e%d p=0x%x v=0x%x TRK=0x%x sub=0x%x last=0x%x cur=0x%x sem_obj=0x%x"
+          % (i, p, v, t, sub, last, cur, sem))
     try:
         so = rmem(sem, 0x40)
         print("  sem_obj[0:0x40]=%s" % so.hex())
@@ -102,4 +110,25 @@ for i in range(n):
             print("  sem_page unreadable: %s" % ex)
     except Exception as ex:
         print("  sem_obj unreadable: %s" % ex)
+    try:
+        q = r64(p + 0x48)
+        print("  prog_obj=0x%x next=0x%x cblist=0x%x idx_u32=0x%x flags95bc=0x%x"
+              % (q, r64(p + 0x9470), r64(p + 0x9458), r32(p + 0x34), r32(p + 0x95bc)))
+        if q:
+            size = r32(q + 8)
+            idx = r32(q + 0x48)
+            cur2 = r32(q + 0x4c)
+            arr = r64(q + 0x40)
+            print("    q+8=size=%d idx=0x%x cursor=0x%x arr=0x%x" % (size, idx, cur2, arr))
+            for s in (idx, cur2):
+                if size:
+                    slot = arr + (s % size) * 0x38
+                else:
+                    slot = arr + s * 0x38
+                try:
+                    print("    slot[%d]@0x%x=%s" % (s, slot, rmem(slot, 0x38).hex()))
+                except Exception as ex:
+                    print("    slot[%d] unreadable: %s" % (s, ex))
+    except Exception as ex:
+        print("  prog unreadable: %s" % ex)
 print("DONE_TRK")
\ No newline at end of file