tools/guest-diag: add UVM wait diagnostics, tracker poke and VA->PA map injector

7 files changed, 337 insertions, 0 deletions

diff --git a/tools/guest-diag/bp999.py b/tools/guest-diag/bp999.py
new file mode 100644
index 0000000..db83d7d
--- /dev/null
+++ b/tools/guest-diag/bp999.py
@@ -0,0 +1,36 @@
+# gdb script: break on the libcuda instructions that produce CUDA errors
+# 719 (0x2cf) and 999 (0x3e7), then print the stack that led there.
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bp999.py
+import gdb
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        base = int(f[0], 16) - int(f[3], 16)
+        break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+    raise SystemExit
+for off in (0x44fb80, 0x44fb6c):
+    gdb.execute("break *0x%x" % (base + off))
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+try:
+    gdb.execute("continue")
+except gdb.error as e:
+    print("CONTINUE_ERR %s" % e)
+print("HIT_OR_EXIT")
+try:
+    print(gdb.execute("bt 16", to_string=True))
+except Exception as e:
+    print("bt_fail %s" % e)
+try:
+    print(gdb.execute("info registers rdi rsi rdx rax rbx rbp rip", to_string=True))
+except Exception as e:
+    print("regs_fail %s" % e)
+try:
+    gdb.execute("detach")
+except Exception:
+    pass
diff --git a/tools/guest-diag/dumpctx.py b/tools/guest-diag/dumpctx.py
new file mode 100644
index 0000000..983997e
--- /dev/null
+++ b/tools/guest-diag/dumpctx.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# gdb script (run with: sudo gdb -p <pid> -batch -x /tmp/dumpctx.py)
+# Dumps the libcuda client wait-loop context: the ctx object's queue/entry
+# array (ctx+0x10 = count, ctx+0x18 = array of 0x28-byte entries) plus each
+# entry's target object, so the host emulation knows which tracker /
+# notification memory it must write to complete the wait.
+import gdb
+
+def r32(a):
+    return int.from_bytes(gdb.selected_inferior().read_memory(a, 4).tobytes(), "little")
+def r64(a):
+    return int.from_bytes(gdb.selected_inferior().read_memory(a, 8).tobytes(), "little")
+def rmem(a, n):
+    return gdb.selected_inferior().read_memory(a, n).tobytes()
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+    if "libcuda" in line and "r-xp" in line:
+        f = line.split()
+        # gdb format: start end size offset objfile  (offset is field 3)
+        base = int(f[0], 16) - int(f[3], 16)
+        print("MAPLINE " + line.strip())
+        break
+if base is None:
+    print("NO_LIBCUDA_BASE")
+    raise SystemExit
+LO = base + 0x23f700
+HI = base + 0x23fd10
+print("LIBBASE=0x%x SPIN=[0x%x,0x%x)" % (base, LO, HI))
+
+found = None
+for t in gdb.selected_inferior().threads():
+    t.switch()
+    f = gdb.newest_frame()
+    for i in range(14):
+        if f is None:
+            break
+        try:
+            pc = int(f.pc())
+        except Exception:
+            pc = 0
+        if LO <= pc < HI:
+            try:
+                rb = int(f.read_register("rbx")) & 0xffffffffffffffff
+            except Exception:
+                rb = None
+            found = (t, pc, rb, i)
+            break
+        f = f.older()
+    if found:
+        break
+
+if not found:
+    print("NO_SPIN_FRAME")
+    for t in gdb.selected_inferior().threads():
+        t.switch()
+        top = ""
+        try:
+            f0 = gdb.newest_frame()
+            top = f0.name() or ""
+        except Exception:
+            pass
+        print("== tid=%s top=%s" % (t.num, top))
+        try:
+            print(gdb.execute("bt 6", to_string=True).rstrip())
+        except Exception as e:
+            print("bt_fail %s" % e)
+        try:
+            print(gdb.execute("info registers rdi rsi rdx rax rip", to_string=True).rstrip())
+        except Exception as e:
+            print("regs_fail %s" % e)
+        if "ioctl" in top or "poll" in top:
+            try:
+                rax = int(gdb.parse_and_eval("$rax")) & 0xffffffffffffffff
+                rdx = int(gdb.parse_and_eval("$rdx")) & 0xffffffffffffffff
+                print("ARG64@0x%x=%s" % (rdx, rmem(rdx, 0x40).hex()))
+            except Exception as e:
+                print("arg_fail %s" % e)
+    print("REGS_HINT")
+    try:
+        print(gdb.execute("thread apply all info registers rip rbx r13 r12", to_string=True).rstrip())
+    except Exception as e:
+        print("regs_fail %s" % e)
+    print("DONE_NOSPIN")
+    raise SystemExit
+
+t, pc, ctx, depth = found
+print("SPINNER tid=%s pc=0x%x depth=%d ctx=0x%x" % (t.num, pc, depth, ctx))
+if ctx is None:
+    raise SystemExit
+try:
+    count = r32(ctx + 0x10)
+    ent = r64(ctx + 0x18)
+    ctx0 = r64(ctx)
+    gpu = r64(ctx0 + 8)
+    print("CTX=0x%x count=%d entries=0x%x ctx0=0x%x gpu=0x%x gpu+3de0=0x%x" %
+          (ctx, count, ent, ctx0, gpu, r64(gpu + 0x3de0)))
+except Exception as e:
+    print("CTX_READ_FAIL %s" % e)
+    raise SystemExit
+n = min(count, 32)
+for i in range(n):
+    e = ent + i * 0x28
+    try:
+        typ = r32(e + 0)
+        p = r64(e + 8)
+        v = r32(e + 0x10)
+        x18 = r64(e + 0x18)
+        x20 = r64(e + 0x20)
+    except Exception as ex:
+        print("e%d @0x%x UNREADABLE %s" % (i, e, ex))
+        continue
+    line = "e%d @0x%x type=%d p=0x%x v=0x%x x18=0x%x x20=0x%x" % (i, e, typ, p, v, x18, x20)
+    try:
+        hdr = rmem(p, 0x20)
+        line += " p32=%d p64=0x%x p[0:0x20]=%s" % (
+            int.from_bytes(hdr[0:4], "little"),
+            int.from_bytes(hdr[0:8], "little"),
+            hdr.hex())
+    except Exception:
+        line += " p_UNREADABLE"
+    if typ == 1:
+        try:
+            trk = rmem(p + 0x9410, 0x40)
+            vals = [int.from_bytes(trk[o:o+8], "little") for o in (0, 8, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38)]
+            line += " TRK=%s q=%s" % (trk.hex(), ",".join("0x%x" % x for x in vals))
+        except Exception:
+            line += " TRK_UNREADABLE"
+    print(line)
+print("DONE_SPIN")
+
+import os
+if os.environ.get("CTX_POKE"):
+    inf = gdb.selected_inferior()
+    npk = 0
+    for i in range(n):
+        e = ent + i * 0x28
+        try:
+            if r32(e) != 1:
+                continue
+            p = r64(e + 8)
+            inf.write_memory(p + 0x9410 + 0x10, (1).to_bytes(8, "little"))
+            inf.write_memory(p + 0x9410 + 0x18, (1).to_bytes(8, "little"))
+            npk += 1
+        except Exception as ex:
+            print("poke e%d fail %s" % (i, ex))
+    print("POKED=%d" % npk)
diff --git a/tools/guest-diag/inject_maps.sh b/tools/guest-diag/inject_maps.sh
new file mode 100755
index 0000000..2973311
--- /dev/null
+++ b/tools/guest-diag/inject_maps.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+# inject_maps.sh - while cdt runs, collect pushbuffer/ring VAs from the host
+# dmesg (sema-chan entry logs) plus channel GPFIFO ring VAs, translate them
+# through the guest's own pagemap, and inject the VA->PA pairs into the
+# service's learned map with sema_map_add.
+#
+# Usage: tools/guest-diag/inject_maps.sh [iterations]
+set -u
+cd "$(dirname "$0")/../.."
+ITERS="${1:-20}"
+LOG=/tmp/inject_maps.log
+: > "$LOG"
+
+map_add() {   # map_add <va-hex> <pa-hex>
+    echo "0x$1,0x$2" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null 2>&1
+}
+
+inject_list() {  # inject_list <va...>
+    local vas="$*" chunk out n=0
+    [ -n "${vas// /}" ] || return 0
+    # chunk to keep the guest agent output under its size cap
+    chunk=""
+    for va in $vas; do
+        chunk="$chunk $va"
+        if [ "$(echo "$chunk" | wc -w)" -ge 12 ]; then
+            out=$(tools/guest-diag/pmap_va.sh cdt $chunk 2>/dev/null | grep '^VA=')
+            echo "$out" | while read -r line; do
+                local v p
+                v=$(echo "$line" | sed -n 's/.*VA=0x\([0-9a-f]*\).*/\1/p')
+                p=$(echo "$line" | sed -n 's/.*present=1.*PA=0x\([0-9a-f]*\).*/\1/p')
+                [ -n "$v" ] && [ -n "$p" ] && { map_add "$v" "$p"; echo "add 0x$v -> 0x$p" >> "$LOG"; }
+            done
+            chunk=""
+        fi
+    done
+    if [ -n "${chunk// /}" ]; then
+        out=$(tools/guest-diag/pmap_va.sh cdt $chunk 2>/dev/null | grep '^VA=')
+        echo "$out" | while read -r line; do
+            local v p
+            v=$(echo "$line" | sed -n 's/.*VA=0x\([0-9a-f]*\).*/\1/p')
+            p=$(echo "$line" | sed -n 's/.*present=1.*PA=0x\([0-9a-f]*\).*/\1/p')
+            [ -n "$v" ] && [ -n "$p" ] && { map_add "$v" "$p"; echo "add 0x$v -> 0x$p" >> "$LOG"; }
+        done
+    fi
+}
+
+i=0
+while [ "$i" -lt "$ITERS" ]; do
+    i=$((i+1))
+    PID=$(timeout 15 ./scripts/vgpu_vm.py exec --timeout 10 'pgrep -x cdt | head -1' 2>/dev/null | tail -1)
+    [ -n "$PID" ] || { echo "cdt gone at iter $i" >> "$LOG"; break; }
+
+    RINGS=$(sudo dmesg | grep -aoE "gpfifo=0x[0-9a-f]+" | grep -aoE "0x[0-9a-f]+" | sort -u | tr '\n' ' ')
+    RINGVAS=$(sudo dmesg | grep -aoE "entry\[[0-9]+\] raw=[0-9a-f]+ va=0x[0-9a-f]+ len=[0-9]+" | grep -aoE "va=0x[0-9a-f]+" | grep -aoE "0x[0-9a-f]+" | sort -u | tr '\n' ' ')
+
+    echo "== iter $i: rings=[$RINGS] entries=[$RINGVAS]" >> "$LOG"
+    inject_list $RINGS
+    inject_list $RINGVAS
+
+    DONE=$(sudo dmesg | grep -aoE "gpfifo done=[0-9]+ nomap=[0-9]+" | tail -1)
+    TAIL=$(timeout 15 ./scripts/vgpu_vm.py exec --timeout 10 'tail -1 /tmp/cdt.out' 2>/dev/null | tail -1)
+    echo "   $DONE | cdt: $TAIL" >> "$LOG"
+    case "$TAIL" in
+        *cuCtxCreate*FAILED*|*ALL*|*cuMemAlloc*) echo "cdt moved on: $TAIL" >> "$LOG"; break;;
+    esac
+    sleep 10
+done
+echo "INJECT_DONE" >> "$LOG"
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 517296f..665011e 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -24,6 +24,7 @@ push_b64 "$HERE/guest_engines.c" /tmp/ge.c
 push_b64 "$HERE/rmq_probe.c"     /tmp/rmqp.c
 push_b64 "$HERE/ioctl_trace.c"   /tmp/iot.c
 push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c
+push_b64 "$HERE/dumpctx.py"     /tmp/dumpctx.py
 
 echo "[*] compiling in guest..."
 "$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \
diff --git a/tools/guest-diag/uvmprobe.sh b/tools/guest-diag/uvmprobe.sh
new file mode 100644
index 0000000..de7293c
--- /dev/null
+++ b/tools/guest-diag/uvmprobe.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+# uvmprobe.sh - guest-side: probe UVM's external-range/map API return statuses
+# while cdt runs, and capture UVM's own debug-print output.
+nvidia-modprobe -c 0
+nvidia-modprobe -u
+modprobe nvidia-uvm
+echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints 2>/dev/null
+echo > /sys/kernel/tracing/kprobe_events 2>/dev/null
+echo 'p:uvm_crext uvm_api_create_external_range par=%di' >> /sys/kernel/tracing/kprobe_events
+echo 'r:uvm_crext_ret uvm_api_create_external_range status=$retval' >> /sys/kernel/tracing/kprobe_events
+echo 'p:uvm_mapext uvm_api_map_external_allocation par=%di' >> /sys/kernel/tracing/kprobe_events
+echo 'r:uvm_mapext_ret uvm_api_map_external_allocation status=$retval' >> /sys/kernel/tracing/kprobe_events
+echo 1 > /sys/kernel/tracing/events/kprobes/enable
+rm -f /tmp/cdt.out
+setsid /tmp/cdt > /tmp/cdt.out 2>&1 &
+sleep 100
+echo 0 > /sys/kernel/tracing/events/kprobes/enable
+dmesg | grep -ai "uvm" | tail -60 > /tmp/uvmdmesg.txt
+cat /sys/kernel/tracing/trace > /tmp/trace.txt
+tail -1 /tmp/cdt.out > /tmp/cdttail.txt
+echo SCRIPT_DONE > /tmp/uvmprobe.done
diff --git a/tools/guest-diag/uvmwait.sh b/tools/guest-diag/uvmwait.sh
new file mode 100644
index 0000000..06b2dc7
--- /dev/null
+++ b/tools/guest-diag/uvmwait.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# uvmwait.sh - guest-side: run cdt with UVM debug prints on, and snapshot the
+# UVM tracker-wait messages early (before any teardown floods dmesg).
+nvidia-modprobe -c 0
+nvidia-modprobe -u
+modprobe nvidia-uvm
+echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints 2>/dev/null
+dmesg -C
+rm -f /tmp/cdt.out
+setsid /tmp/cdt > /tmp/cdt.out 2>&1 &
+sleep 55
+dmesg | grep -aiE "tracker|semaphore|releasing|channel|page table" > /tmp/uvmwait.txt
+tail -1 /tmp/cdt.out > /tmp/cdttail.txt
+echo DONE > /tmp/uvmwait.done
diff --git a/tools/guest-diag/watch_ctx.sh b/tools/guest-diag/watch_ctx.sh
new file mode 100644
index 0000000..0968571
--- /dev/null
+++ b/tools/guest-diag/watch_ctx.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+# watch_ctx.sh - guest-side watcher: (re)start cdt, snapshot the ioctl stream
+# (fd paths), attach gdb to the libcuda wait as soon as cuCtxCreate is reached
+# and POKE the per-entry trackers (p+0x9410+0x10/0x18 = 1) to test whether that
+# gate releases the wait. Runs under sudo; results land in /tmp/ctxdumps/.
+mkdir -p /tmp/ctxdumps
+rm -f /tmp/ctxdumps/*
+nvidia-modprobe -c 0
+nvidia-modprobe -u
+modprobe nvidia-uvm
+rm -f /tmp/cdt.out
+setsid /tmp/cdt > /tmp/cdt.out 2>&1 &
+i=0
+while [ $i -lt 300 ]; do
+    grep -q 'cuCtxCreate' /tmp/cdt.out 2>/dev/null && break
+    sleep 0.3
+    i=$((i+1))
+done
+P=$(pidof cdt | tr ' ' '\n' | head -1)
+ls -l /proc/$P/fd/ > /tmp/ctxdumps/fds.txt 2>&1
+if command -v strace >/dev/null 2>&1; then
+    timeout 8 strace -f -y -p "$P" -e trace=ioctl -o /tmp/ctxdumps/ioctl.txt 2>/dev/null
+fi
+n=0
+while [ $n -lt 80 ]; do
+    P=$(pidof cdt | tr ' ' '\n' | head -1)
+    [ -n "$P" ] || break
+    gdb -p "$P" -batch -x /tmp/dumpctx.py > "/tmp/ctxdumps/d$n.txt" 2>&1
+    if grep -q DONE_SPIN "/tmp/ctxdumps/d$n.txt"; then
+        cp "/tmp/ctxdumps/d$n.txt" /tmp/ctxdumps/FOUND.txt
+        break
+    fi
+    n=$((n+1))
+    sleep 0.5
+done
+P=$(pidof cdt | tr ' ' '\n' | head -1)
+if [ -f /tmp/ctxdumps/FOUND.txt ] && [ -n "$P" ]; then
+    CTX_POKE=1 gdb -p "$P" -batch -x /tmp/dumpctx.py > /tmp/ctxdumps/poke_run.txt 2>&1
+    k=0
+    while [ $k -lt 12 ]; do
+        echo "--- t=$((k*3))s $(tail -1 /tmp/cdt.out)" >> /tmp/ctxdumps/result.txt
+        k=$((k+1))
+        sleep 3
+    done
+    VAS=$(grep -oE 'type=1 p=0x[0-9a-f]+' /tmp/ctxdumps/FOUND.txt | sed 's/.*p=0x//' | while read -r p; do printf '0x%x ' $((0x$p + 0x9410)); done)
+    echo "PID=$P TRACKER_VAS $VAS" > /tmp/ctxdumps/probe.txt
+    python3 /tmp/pmap.py "$P" $VAS >> /tmp/ctxdumps/probe.txt 2>&1
+fi
+echo WATCH_DONE >> /tmp/ctxdumps/WATCH.log