summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--tools/guest-diag/bp999.py36
-rw-r--r--tools/guest-diag/dumpctx.py148
-rwxr-xr-xtools/guest-diag/inject_maps.sh68
-rwxr-xr-xtools/guest-diag/push_guest_tools.sh1
-rw-r--r--tools/guest-diag/uvmprobe.sh21
-rw-r--r--tools/guest-diag/uvmwait.sh14
-rw-r--r--tools/guest-diag/watch_ctx.sh49
7 files changed, 337 insertions, 0 deletions
diff --git a/tools/guest-diag/bp999.py b/tools/guest-diag/bp999.py
new file mode 100644
index 0000000..db83d7d
--- /dev/null
+++ b/tools/guest-diag/bp999.py
@@ -0,0 +1,36 @@
+# gdb script: break on the libcuda instructions that produce CUDA errors
+# 719 (0x2cf) and 999 (0x3e7), then print the stack that led there.
+# Run: sudo gdb -p <cdt-pid> -batch -x /tmp/bp999.py
+import gdb
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+ if "libcuda" in line and "r-xp" in line:
+ f = line.split()
+ base = int(f[0], 16) - int(f[3], 16)
+ break
+print("BASE=0x%x" % (base or 0))
+if base is None:
+ raise SystemExit
+for off in (0x44fb80, 0x44fb6c):
+ gdb.execute("break *0x%x" % (base + off))
+gdb.execute("set pagination off")
+gdb.execute("set confirm off")
+try:
+ gdb.execute("continue")
+except gdb.error as e:
+ print("CONTINUE_ERR %s" % e)
+print("HIT_OR_EXIT")
+try:
+ print(gdb.execute("bt 16", to_string=True))
+except Exception as e:
+ print("bt_fail %s" % e)
+try:
+ print(gdb.execute("info registers rdi rsi rdx rax rbx rbp rip", to_string=True))
+except Exception as e:
+ print("regs_fail %s" % e)
+try:
+ gdb.execute("detach")
+except Exception:
+ pass
diff --git a/tools/guest-diag/dumpctx.py b/tools/guest-diag/dumpctx.py
new file mode 100644
index 0000000..983997e
--- /dev/null
+++ b/tools/guest-diag/dumpctx.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# gdb script (run with: sudo gdb -p <pid> -batch -x /tmp/dumpctx.py)
+# Dumps the libcuda client wait-loop context: the ctx object's queue/entry
+# array (ctx+0x10 = count, ctx+0x18 = array of 0x28-byte entries) plus each
+# entry's target object, so the host emulation knows which tracker /
+# notification memory it must write to complete the wait.
+import gdb
+
+def r32(a):
+ return int.from_bytes(gdb.selected_inferior().read_memory(a, 4).tobytes(), "little")
+def r64(a):
+ return int.from_bytes(gdb.selected_inferior().read_memory(a, 8).tobytes(), "little")
+def rmem(a, n):
+ return gdb.selected_inferior().read_memory(a, n).tobytes()
+
+out = gdb.execute("info proc mappings", to_string=True)
+base = None
+for line in out.splitlines():
+ if "libcuda" in line and "r-xp" in line:
+ f = line.split()
+ # gdb format: start end size offset objfile (offset is field 3)
+ base = int(f[0], 16) - int(f[3], 16)
+ print("MAPLINE " + line.strip())
+ break
+if base is None:
+ print("NO_LIBCUDA_BASE")
+ raise SystemExit
+LO = base + 0x23f700
+HI = base + 0x23fd10
+print("LIBBASE=0x%x SPIN=[0x%x,0x%x)" % (base, LO, HI))
+
+found = None
+for t in gdb.selected_inferior().threads():
+ t.switch()
+ f = gdb.newest_frame()
+ for i in range(14):
+ if f is None:
+ break
+ try:
+ pc = int(f.pc())
+ except Exception:
+ pc = 0
+ if LO <= pc < HI:
+ try:
+ rb = int(f.read_register("rbx")) & 0xffffffffffffffff
+ except Exception:
+ rb = None
+ found = (t, pc, rb, i)
+ break
+ f = f.older()
+ if found:
+ break
+
+if not found:
+ print("NO_SPIN_FRAME")
+ for t in gdb.selected_inferior().threads():
+ t.switch()
+ top = ""
+ try:
+ f0 = gdb.newest_frame()
+ top = f0.name() or ""
+ except Exception:
+ pass
+ print("== tid=%s top=%s" % (t.num, top))
+ try:
+ print(gdb.execute("bt 6", to_string=True).rstrip())
+ except Exception as e:
+ print("bt_fail %s" % e)
+ try:
+ print(gdb.execute("info registers rdi rsi rdx rax rip", to_string=True).rstrip())
+ except Exception as e:
+ print("regs_fail %s" % e)
+ if "ioctl" in top or "poll" in top:
+ try:
+ rax = int(gdb.parse_and_eval("$rax")) & 0xffffffffffffffff
+ rdx = int(gdb.parse_and_eval("$rdx")) & 0xffffffffffffffff
+ print("ARG64@0x%x=%s" % (rdx, rmem(rdx, 0x40).hex()))
+ except Exception as e:
+ print("arg_fail %s" % e)
+ print("REGS_HINT")
+ try:
+ print(gdb.execute("thread apply all info registers rip rbx r13 r12", to_string=True).rstrip())
+ except Exception as e:
+ print("regs_fail %s" % e)
+ print("DONE_NOSPIN")
+ raise SystemExit
+
+t, pc, ctx, depth = found
+print("SPINNER tid=%s pc=0x%x depth=%d ctx=0x%x" % (t.num, pc, depth, ctx))
+if ctx is None:
+ raise SystemExit
+try:
+ count = r32(ctx + 0x10)
+ ent = r64(ctx + 0x18)
+ ctx0 = r64(ctx)
+ gpu = r64(ctx0 + 8)
+ print("CTX=0x%x count=%d entries=0x%x ctx0=0x%x gpu=0x%x gpu+3de0=0x%x" %
+ (ctx, count, ent, ctx0, gpu, r64(gpu + 0x3de0)))
+except Exception as e:
+ print("CTX_READ_FAIL %s" % e)
+ raise SystemExit
+n = min(count, 32)
+for i in range(n):
+ e = ent + i * 0x28
+ try:
+ typ = r32(e + 0)
+ p = r64(e + 8)
+ v = r32(e + 0x10)
+ x18 = r64(e + 0x18)
+ x20 = r64(e + 0x20)
+ except Exception as ex:
+ print("e%d @0x%x UNREADABLE %s" % (i, e, ex))
+ continue
+ line = "e%d @0x%x type=%d p=0x%x v=0x%x x18=0x%x x20=0x%x" % (i, e, typ, p, v, x18, x20)
+ try:
+ hdr = rmem(p, 0x20)
+ line += " p32=%d p64=0x%x p[0:0x20]=%s" % (
+ int.from_bytes(hdr[0:4], "little"),
+ int.from_bytes(hdr[0:8], "little"),
+ hdr.hex())
+ except Exception:
+ line += " p_UNREADABLE"
+ if typ == 1:
+ try:
+ trk = rmem(p + 0x9410, 0x40)
+ vals = [int.from_bytes(trk[o:o+8], "little") for o in (0, 8, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38)]
+ line += " TRK=%s q=%s" % (trk.hex(), ",".join("0x%x" % x for x in vals))
+ except Exception:
+ line += " TRK_UNREADABLE"
+ print(line)
+print("DONE_SPIN")
+
+import os
+if os.environ.get("CTX_POKE"):
+ inf = gdb.selected_inferior()
+ npk = 0
+ for i in range(n):
+ e = ent + i * 0x28
+ try:
+ if r32(e) != 1:
+ continue
+ p = r64(e + 8)
+ inf.write_memory(p + 0x9410 + 0x10, (1).to_bytes(8, "little"))
+ inf.write_memory(p + 0x9410 + 0x18, (1).to_bytes(8, "little"))
+ npk += 1
+ except Exception as ex:
+ print("poke e%d fail %s" % (i, ex))
+ print("POKED=%d" % npk)
diff --git a/tools/guest-diag/inject_maps.sh b/tools/guest-diag/inject_maps.sh
new file mode 100755
index 0000000..2973311
--- /dev/null
+++ b/tools/guest-diag/inject_maps.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+# inject_maps.sh - while cdt runs, collect pushbuffer/ring VAs from the host
+# dmesg (sema-chan entry logs) plus channel GPFIFO ring VAs, translate them
+# through the guest's own pagemap, and inject the VA->PA pairs into the
+# service's learned map with sema_map_add.
+#
+# Usage: tools/guest-diag/inject_maps.sh [iterations]
+set -u
+cd "$(dirname "$0")/../.."
+ITERS="${1:-20}"
+LOG=/tmp/inject_maps.log
+: > "$LOG"
+
+map_add() { # map_add <va-hex> <pa-hex>
+ echo "0x$1,0x$2" | sudo tee /sys/module/open_iov_vfio/parameters/sema_map_add >/dev/null 2>&1
+}
+
+inject_list() { # inject_list <va...>
+ local vas="$*" chunk out n=0
+ [ -n "${vas// /}" ] || return 0
+ # chunk to keep the guest agent output under its size cap
+ chunk=""
+ for va in $vas; do
+ chunk="$chunk $va"
+ if [ "$(echo "$chunk" | wc -w)" -ge 12 ]; then
+ out=$(tools/guest-diag/pmap_va.sh cdt $chunk 2>/dev/null | grep '^VA=')
+ echo "$out" | while read -r line; do
+ local v p
+ v=$(echo "$line" | sed -n 's/.*VA=0x\([0-9a-f]*\).*/\1/p')
+ p=$(echo "$line" | sed -n 's/.*present=1.*PA=0x\([0-9a-f]*\).*/\1/p')
+ [ -n "$v" ] && [ -n "$p" ] && { map_add "$v" "$p"; echo "add 0x$v -> 0x$p" >> "$LOG"; }
+ done
+ chunk=""
+ fi
+ done
+ if [ -n "${chunk// /}" ]; then
+ out=$(tools/guest-diag/pmap_va.sh cdt $chunk 2>/dev/null | grep '^VA=')
+ echo "$out" | while read -r line; do
+ local v p
+ v=$(echo "$line" | sed -n 's/.*VA=0x\([0-9a-f]*\).*/\1/p')
+ p=$(echo "$line" | sed -n 's/.*present=1.*PA=0x\([0-9a-f]*\).*/\1/p')
+ [ -n "$v" ] && [ -n "$p" ] && { map_add "$v" "$p"; echo "add 0x$v -> 0x$p" >> "$LOG"; }
+ done
+ fi
+}
+
+i=0
+while [ "$i" -lt "$ITERS" ]; do
+ i=$((i+1))
+ PID=$(timeout 15 ./scripts/vgpu_vm.py exec --timeout 10 'pgrep -x cdt | head -1' 2>/dev/null | tail -1)
+ [ -n "$PID" ] || { echo "cdt gone at iter $i" >> "$LOG"; break; }
+
+ RINGS=$(sudo dmesg | grep -aoE "gpfifo=0x[0-9a-f]+" | grep -aoE "0x[0-9a-f]+" | sort -u | tr '\n' ' ')
+ RINGVAS=$(sudo dmesg | grep -aoE "entry\[[0-9]+\] raw=[0-9a-f]+ va=0x[0-9a-f]+ len=[0-9]+" | grep -aoE "va=0x[0-9a-f]+" | grep -aoE "0x[0-9a-f]+" | sort -u | tr '\n' ' ')
+
+ echo "== iter $i: rings=[$RINGS] entries=[$RINGVAS]" >> "$LOG"
+ inject_list $RINGS
+ inject_list $RINGVAS
+
+ DONE=$(sudo dmesg | grep -aoE "gpfifo done=[0-9]+ nomap=[0-9]+" | tail -1)
+ TAIL=$(timeout 15 ./scripts/vgpu_vm.py exec --timeout 10 'tail -1 /tmp/cdt.out' 2>/dev/null | tail -1)
+ echo " $DONE | cdt: $TAIL" >> "$LOG"
+ case "$TAIL" in
+ *cuCtxCreate*FAILED*|*ALL*|*cuMemAlloc*) echo "cdt moved on: $TAIL" >> "$LOG"; break;;
+ esac
+ sleep 10
+done
+echo "INJECT_DONE" >> "$LOG"
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 517296f..665011e 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -24,6 +24,7 @@ push_b64 "$HERE/guest_engines.c" /tmp/ge.c
push_b64 "$HERE/rmq_probe.c" /tmp/rmqp.c
push_b64 "$HERE/ioctl_trace.c" /tmp/iot.c
push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c
+push_b64 "$HERE/dumpctx.py" /tmp/dumpctx.py
echo "[*] compiling in guest..."
"$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \
diff --git a/tools/guest-diag/uvmprobe.sh b/tools/guest-diag/uvmprobe.sh
new file mode 100644
index 0000000..de7293c
--- /dev/null
+++ b/tools/guest-diag/uvmprobe.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+# uvmprobe.sh - guest-side: probe UVM's external-range/map API return statuses
+# while cdt runs, and capture UVM's own debug-print output.
+nvidia-modprobe -c 0
+nvidia-modprobe -u
+modprobe nvidia-uvm
+echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints 2>/dev/null
+echo > /sys/kernel/tracing/kprobe_events 2>/dev/null
+echo 'p:uvm_crext uvm_api_create_external_range par=%di' >> /sys/kernel/tracing/kprobe_events
+echo 'r:uvm_crext_ret uvm_api_create_external_range status=$retval' >> /sys/kernel/tracing/kprobe_events
+echo 'p:uvm_mapext uvm_api_map_external_allocation par=%di' >> /sys/kernel/tracing/kprobe_events
+echo 'r:uvm_mapext_ret uvm_api_map_external_allocation status=$retval' >> /sys/kernel/tracing/kprobe_events
+echo 1 > /sys/kernel/tracing/events/kprobes/enable
+rm -f /tmp/cdt.out
+setsid /tmp/cdt > /tmp/cdt.out 2>&1 &
+sleep 100
+echo 0 > /sys/kernel/tracing/events/kprobes/enable
+dmesg | grep -ai "uvm" | tail -60 > /tmp/uvmdmesg.txt
+cat /sys/kernel/tracing/trace > /tmp/trace.txt
+tail -1 /tmp/cdt.out > /tmp/cdttail.txt
+echo SCRIPT_DONE > /tmp/uvmprobe.done
diff --git a/tools/guest-diag/uvmwait.sh b/tools/guest-diag/uvmwait.sh
new file mode 100644
index 0000000..06b2dc7
--- /dev/null
+++ b/tools/guest-diag/uvmwait.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# uvmwait.sh - guest-side: run cdt with UVM debug prints on, and snapshot the
+# UVM tracker-wait messages early (before any teardown floods dmesg).
+nvidia-modprobe -c 0
+nvidia-modprobe -u
+modprobe nvidia-uvm
+echo 1 > /sys/module/nvidia_uvm/parameters/uvm_debug_prints 2>/dev/null
+dmesg -C
+rm -f /tmp/cdt.out
+setsid /tmp/cdt > /tmp/cdt.out 2>&1 &
+sleep 55
+dmesg | grep -aiE "tracker|semaphore|releasing|channel|page table" > /tmp/uvmwait.txt
+tail -1 /tmp/cdt.out > /tmp/cdttail.txt
+echo DONE > /tmp/uvmwait.done
diff --git a/tools/guest-diag/watch_ctx.sh b/tools/guest-diag/watch_ctx.sh
new file mode 100644
index 0000000..0968571
--- /dev/null
+++ b/tools/guest-diag/watch_ctx.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+# watch_ctx.sh - guest-side watcher: (re)start cdt, snapshot the ioctl stream
+# (fd paths), attach gdb to the libcuda wait as soon as cuCtxCreate is reached
+# and POKE the per-entry trackers (p+0x9410+0x10/0x18 = 1) to test whether that
+# gate releases the wait. Runs under sudo; results land in /tmp/ctxdumps/.
+mkdir -p /tmp/ctxdumps
+rm -f /tmp/ctxdumps/*
+nvidia-modprobe -c 0
+nvidia-modprobe -u
+modprobe nvidia-uvm
+rm -f /tmp/cdt.out
+setsid /tmp/cdt > /tmp/cdt.out 2>&1 &
+i=0
+while [ $i -lt 300 ]; do
+ grep -q 'cuCtxCreate' /tmp/cdt.out 2>/dev/null && break
+ sleep 0.3
+ i=$((i+1))
+done
+P=$(pidof cdt | tr ' ' '\n' | head -1)
+ls -l /proc/$P/fd/ > /tmp/ctxdumps/fds.txt 2>&1
+if command -v strace >/dev/null 2>&1; then
+ timeout 8 strace -f -y -p "$P" -e trace=ioctl -o /tmp/ctxdumps/ioctl.txt 2>/dev/null
+fi
+n=0
+while [ $n -lt 80 ]; do
+ P=$(pidof cdt | tr ' ' '\n' | head -1)
+ [ -n "$P" ] || break
+ gdb -p "$P" -batch -x /tmp/dumpctx.py > "/tmp/ctxdumps/d$n.txt" 2>&1
+ if grep -q DONE_SPIN "/tmp/ctxdumps/d$n.txt"; then
+ cp "/tmp/ctxdumps/d$n.txt" /tmp/ctxdumps/FOUND.txt
+ break
+ fi
+ n=$((n+1))
+ sleep 0.5
+done
+P=$(pidof cdt | tr ' ' '\n' | head -1)
+if [ -f /tmp/ctxdumps/FOUND.txt ] && [ -n "$P" ]; then
+ CTX_POKE=1 gdb -p "$P" -batch -x /tmp/dumpctx.py > /tmp/ctxdumps/poke_run.txt 2>&1
+ k=0
+ while [ $k -lt 12 ]; do
+ echo "--- t=$((k*3))s $(tail -1 /tmp/cdt.out)" >> /tmp/ctxdumps/result.txt
+ k=$((k+1))
+ sleep 3
+ done
+ VAS=$(grep -oE 'type=1 p=0x[0-9a-f]+' /tmp/ctxdumps/FOUND.txt | sed 's/.*p=0x//' | while read -r p; do printf '0x%x ' $((0x$p + 0x9410)); done)
+ echo "PID=$P TRACKER_VAS $VAS" > /tmp/ctxdumps/probe.txt
+ python3 /tmp/pmap.py "$P" $VAS >> /tmp/ctxdumps/probe.txt 2>&1
+fi
+echo WATCH_DONE >> /tmp/ctxdumps/WATCH.log
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 11:54:12 +0000