From cc2c2ab40af45e3a14e93b87e4988909303f8a95 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 24 Sep 2026 10:01:22 -0400 Subject: kernel: resolve semaphores through trusted translations only; tools: dump the libcuda tracker wait The vidmem window convention turns any VA above the window base into a vidmem offset, but the CUDA client's GPU semaphores are sysmem objects mapped into its address space at those same VAs. A release applied through the convention therefore landed in the vidmem shadow instead of the page the client polls, and the completion was silently lost: the libcuda tracker it waits on never advanced. Semaphore reads and writes now use trusted translations only (pins, learned cache, discovered map, page walks), so an unresolved semaphore is counted as a miss and stays visible instead of corrupting a shadow. tools/guest-diag/trkinfo.py dumps the libcuda tracker the client spins on: the tracker object behind p+0x9410, the semaphore object it references, the u32 semaphore memory it polls (a /dev/nvidiactl mapping backed by guest RAM) and the VMA that memory lives in. --- tools/guest-diag/push_guest_tools.sh | 1 + tools/guest-diag/trkinfo.py | 105 +++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 tools/guest-diag/trkinfo.py (limited to 'tools') diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh index 665011e..516c64a 100755 --- a/tools/guest-diag/push_guest_tools.sh +++ b/tools/guest-diag/push_guest_tools.sh @@ -25,6 +25,7 @@ push_b64 "$HERE/rmq_probe.c" /tmp/rmqp.c push_b64 "$HERE/ioctl_trace.c" /tmp/iot.c push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c push_b64 "$HERE/dumpctx.py" /tmp/dumpctx.py +push_b64 "$HERE/trkinfo.py" /tmp/trkinfo.py echo "[*] compiling in guest..." "$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \ diff --git a/tools/guest-diag/trkinfo.py b/tools/guest-diag/trkinfo.py new file mode 100644 index 0000000..46b6a69 --- /dev/null +++ b/tools/guest-diag/trkinfo.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# gdb script (run with: sudo gdb -p -batch -x /tmp/trkinfo.py) +# Drills into the libcuda tracker the client spins on (the wait loop around +# libcuda+0x23f700): dumps the tracker object at p+0x9410, the semaphore +# object it references at +0x20, and the u32 semaphore memory that object +# points at (+0x10), plus the VMA that memory lives in. +import gdb + +def r32(a): + return int.from_bytes(gdb.selected_inferior().read_memory(a, 4).tobytes(), "little") +def r64(a): + return int.from_bytes(gdb.selected_inferior().read_memory(a, 8).tobytes(), "little") +def rmem(a, n): + return gdb.selected_inferior().read_memory(a, n).tobytes() + +out = gdb.execute("info proc mappings", to_string=True) +base = None +maps = [] +for line in out.splitlines(): + f = line.split() + if len(f) >= 5: + try: + lo = int(f[0], 16); hi = int(f[1], 16) + maps.append((lo, hi, line.strip())) + except Exception: + pass + if "libcuda" in line and "r-xp" in line and base is None: + base = int(f[0], 16) - int(f[3], 16) + +def vma_of(addr): + for lo, hi, line in maps: + if lo <= addr < hi: + return line + return "(none)" + +if base is None: + print("NO_LIBCUDA_BASE") + raise SystemExit +LO = base + 0x23f700 +HI = base + 0x23fd10 + +found = None +for t in gdb.selected_inferior().threads(): + t.switch() + f = gdb.newest_frame() + for i in range(14): + if f is None: + break + try: + pc = int(f.pc()) + except Exception: + pc = 0 + if LO <= pc < HI: + try: + ctx = int(f.read_register("rbx")) & 0xffffffffffffffff + except Exception: + ctx = None + found = (t.num, pc, ctx) + break + f = f.older() + if found: + break + +if not found: + print("NO_SPIN_FRAME") + raise SystemExit + +tid, pc, ctx = found +print("SPINNER tid=%s pc=0x%x ctx=0x%x" % (tid, pc, ctx)) +count = r32(ctx + 0x10) +ent = r64(ctx + 0x18) +print("count=%d entries=0x%x" % (count, ent)) +n = min(count, 32) +for i in range(n): + e = ent + i * 0x28 + typ = r32(e) + p = r64(e + 8) + v = r32(e + 0x10) + if typ != 1: + print("e%d type=%d p=0x%x v=0x%x (skip)" % (i, typ, p, v)) + continue + t = p + 0x9410 + try: + trk = rmem(t, 0x40) + except Exception as ex: + print("e%d tracker unreadable: %s" % (i, ex)) + continue + last = int.from_bytes(trk[0x10:0x18], "little") + cur = int.from_bytes(trk[0x18:0x20], "little") + sem = int.from_bytes(trk[0x20:0x28], "little") + print("e%d p=0x%x v=0x%x TRK=0x%x last=0x%x cur=0x%x sem_obj=0x%x" + % (i, p, v, t, last, cur, sem)) + try: + so = rmem(sem, 0x40) + print(" sem_obj[0:0x40]=%s" % so.hex()) + mem = int.from_bytes(so[0x10:0x18], "little") + print(" sem_mem=0x%x vma=%s" % (mem, vma_of(mem))) + try: + mv = rmem(mem & ~0xfff, 0x40) + print(" sem_page[0:0x40]=%s val@+0x%x=0x%x" % (mv.hex(), mem & 0xfff, r32(mem))) + except Exception as ex: + print(" sem_page unreadable: %s" % ex) + except Exception as ex: + print(" sem_obj unreadable: %s" % ex) +print("DONE_TRK") \ No newline at end of file -- cgit v1.2.3