diff options
Diffstat (limited to 'tools/guest-diag')
| -rw-r--r-- | tools/guest-diag/guest_kprobes.sh | 56 | ||||
| -rwxr-xr-x | tools/guest-diag/push_guest_tools.sh | 1 | ||||
| -rw-r--r-- | tools/guest-diag/semfix_watch.py | 342 |
3 files changed, 301 insertions, 98 deletions
diff --git a/tools/guest-diag/guest_kprobes.sh b/tools/guest-diag/guest_kprobes.sh new file mode 100644 index 0000000..f676e20 --- /dev/null +++ b/tools/guest-diag/guest_kprobes.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# guest_kprobes.sh - (re)install the UVM trace probes used by semfix_watch.py. +# Runs in the guest as root; kprobes do not survive a guest boot, so this must +# run after every VM restart. +# +# uvmt uvm_gpu_tracking_semaphore_update_completed_value sem=%di +# uvch __uvm_push_begin_acquire_on_channel_with_info ch=%di +# uvbp uvm_pushbuffer_begin_push pb=%di +# uvpbf uvm_pushbuffer_create pb=%si +# +# Tries the plain symbol first (module symbols usually resolve), then the +# module:symbol spelling, then without the fetch argument (fetchargs on some +# symbols are rejected). + +TR=/sys/kernel/tracing +[ -d "$TR" ] || TR=/sys/kernel/debug/tracing + +add_probe() { + name=$1; sym=$2; fmt=$3 + echo "-:$name" >> "$TR/kprobe_events" 2>/dev/null + if ! grep -qw "$sym" /proc/kallsyms 2>/dev/null; then + echo "FAIL $name: symbol $sym not in kallsyms" + return 1 + fi + if echo "p:$name $sym $fmt" >> "$TR/kprobe_events" 2>/dev/null; then + if grep -q "p:$name" "$TR/kprobe_events" 2>/dev/null; then + echo "ok $name $sym" + return 0 + fi + fi + echo "-:$name" >> "$TR/kprobe_events" 2>/dev/null + if echo "p:$name nvidia_uvm:$sym $fmt" >> "$TR/kprobe_events" 2>/dev/null; then + if grep -q "p:$name" "$TR/kprobe_events" 2>/dev/null; then + echo "ok-module $name $sym" + return 0 + fi + fi + echo "-:$name" >> "$TR/kprobe_events" 2>/dev/null + if echo "p:$name $sym" >> "$TR/kprobe_events" 2>/dev/null; then + if grep -q "p:$name" "$TR/kprobe_events" 2>/dev/null; then + echo "ok-bare $name $sym" + return 0 + fi + fi + echo "FAIL $name $sym" + return 1 +} + +add_probe uvmt uvm_gpu_tracking_semaphore_update_completed_value 'sem=%di val=+16(%di):u32' +add_probe uvch __uvm_push_begin_acquire_on_channel_with_info 'ch=%di' +add_probe uvbp uvm_pushbuffer_begin_push 'pb=%di' +add_probe uvpbf uvm_pushbuffer_create 'pb=%si' + +echo 1 > "$TR/events/kprobes/enable" 2>/dev/null || true +echo "--- installed probes ---" +grep -E 'uvmt|uvch|uvbp|uvpbf' "$TR/kprobe_events" 2>/dev/null || echo "(none)" diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh index 93f7e6a..a96fe68 100755 --- a/tools/guest-diag/push_guest_tools.sh +++ b/tools/guest-diag/push_guest_tools.sh @@ -34,6 +34,7 @@ push_b64 "$HERE/sem_slots.py" /tmp/sem_slots.py push_b64 "$HERE/read_va.py" /tmp/read_va.py push_b64 "$HERE/guest_pmap.py" /tmp/guest_pmap.py push_b64 "$HERE/track_waits.sh" /tmp/track_waits.sh +push_b64 "$HERE/guest_kprobes.sh" /tmp/kprobes.sh echo "[*] compiling in guest..." "$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \ diff --git a/tools/guest-diag/semfix_watch.py b/tools/guest-diag/semfix_watch.py index 7ae154f..7730e86 100644 --- a/tools/guest-diag/semfix_watch.py +++ b/tools/guest-diag/semfix_watch.py @@ -10,16 +10,26 @@ # `semaphore.page->memory->vas[]` with the pool page's CPU and GPU VAs; # QEMU turns the CPU VA into a GPA (`gva2gpa`) which we pin. # 2. CE pushbuffer: the pushes live in a UVM rm_mem with GPU VA -# `0x120000000` whose CPU VA is vmalloc-mapped guest RAM. Its pages map -# linearly *descending* in GPA (page i -> base_gpa - i*0x1000). Pinning -# the range lets the host pump read the pushes directly and execute the -# PTE memsets/copies instead of only acknowledging their releases. +# `0x120000000` whose CPU VA is vmalloc-mapped guest RAM. Pinning the +# range lets the host pump read the pushes directly and execute the PTE +# memsets/copies instead of only acknowledging their releases. # -# Sources: guest kprobe on uvm_gpu_tracking_semaphore_update_completed_value -# (`sem=0x...`), QEMU QMP `x/..gx` reads and `gva2gpa`. +# Pushbuffer chain (uvm 570.144 layouts): +# uvm_channel_t +0x00 = pool +# uvm_channel_pool_t +0x00 = manager +# uvm_channel_manager_t +0x08 = pushbuffer +# uvm_pushbuffer_t +0x08 = memory (uvm_rm_mem_t) +# uvm_rm_mem_t +0x10 = vas[] (CPU @ +0x30, GPU @ +0x38 observed) +# The channel is reached three ways: kprobe on +# __uvm_push_begin_acquire_on_channel_with_info (`uvch` -> ch), kprobe on +# uvm_pushbuffer_begin_push (`uvbp` -> pb), or the tracking-sem kptr `sem` +# from `uvmt` (sem sits inside the channel struct at a fixed offset). +# +# Sources: guest kprobes, QEMU QMP `x/..gx` reads and `gva2gpa`. # # Usage: tools/guest-diag/semfix_watch.py [interval-seconds] -# Runs until killed; prints one line per newly resolved/pinned page. +# Runs until killed; prints one line per newly resolved/pinned page and a +# structured diagnostic line whenever a chain resolution misses. import json import os @@ -36,9 +46,11 @@ QMP_SOCK = os.environ.get("QMP_SOCK", "/tmp/qemu-qmp.sock") GPU_VA_LO = 0x100000000 GPU_VA_HI = 0x2000000000 -PUSHBUF_VA_LO = 0x120000000 -PUSHBUF_VA_HI = 0x120400000 +PUSHBUF_VA_LO = int(os.environ.get("PUSHBUF_VA_LO", "0x120000000"), 16) +PUSHBUF_VA_HI = int(os.environ.get("PUSHBUF_VA_HI", "0x120400000"), 16) PUSHBUF_MAX_PAGES = 1024 +CHAN_WINDOW_BACK = int(os.environ.get("CHAN_WINDOW_BACK", "0x4000"), 16) +CHAN_MAX_CANDS = int(os.environ.get("CHAN_MAX_CANDS", "256")) class Qmp: @@ -48,6 +60,7 @@ class Qmp: self.s.settimeout(15) self._rd() self.cmd("qmp_capabilities") + self._cache = {} def _rd(self): time.sleep(0.03) @@ -67,11 +80,16 @@ class Qmp: return self.cmd("human-monitor-command", **{"command-line": line}).get("return", "") def read64(self, addr, n=1): + key = (addr, n) + if key in self._cache: + return self._cache[key] out = self.hmp("x/%dgx 0x%x" % (n, addr)) m = re.search(r":(.*)$", out, re.S) if not m: return [] - return [int(x, 16) for x in re.findall(r"0x([0-9a-fA-F]+)", m.group(1))][:n] + vals = [int(x, 16) for x in re.findall(r"0x([0-9a-fA-F]+)", m.group(1))][:n] + self._cache[key] = vals + return vals def gva2gpa(self, va): out = self.hmp("gva2gpa 0x%x" % va) @@ -87,45 +105,200 @@ def is_gpu(v): return v and GPU_VA_LO <= v < GPU_VA_HI +def is_pushbuf_gpu(v): + return v and PUSHBUF_VA_LO <= v < PUSHBUF_VA_HI + + def is_vmalloc(v): # pushbuffer rm_mems are vmalloc-mapped (the sem pools are direct-map) return v and 0xFFFFC90000000000 <= v < 0xFFFFE90000000000 +def diag_add(d, k, n=1): + d[k] = d.get(k, 0) + n + + +def diag_str(d): + return " ".join("%s=%s" % (k, v) for k, v in sorted(d.items())) + + def guest_trace(): - """(sem kptrs, pushbuffer kptrs) from the trace ring, then clear it. + """(sem kptrs, pushbuffer kptrs, channel kptrs) from the trace, then clear. - Read *before* clearing: uvm_pushbuffer_create fires once per client and - would be wiped by a clear that races it. + Read *before* clearing: the create-time probes fire once per client and + would be wiped by a clear that races them. """ - cmd = ("sudo sh -c 'grep -E \"uvmt|uvpb\" /sys/kernel/tracing/trace | tail -200; " - "echo > /sys/kernel/tracing/trace'") + cmd = ("sudo sh -c 'grep -E \"uvmt|uvch|uvbp|uvpbf\" /sys/kernel/tracing/trace " + "| tail -400; echo > /sys/kernel/tracing/trace'") try: out = subprocess.run([VMEXEC, "exec", cmd], capture_output=True, text=True, timeout=90).stdout except Exception: - return set(), set() + return set(), set(), set() sems = {int(m, 16) for m in re.findall(r"sem=0x([0-9a-fA-F]+)", out)} pbs = {int(m, 16) for m in re.findall(r"pb=0x([0-9a-fA-F]+)", out)} - return sems, pbs - - -def resolve_pb_ptr(q, pb): - """pushbuffer pointer -> rm_mem -> vas[] -> (gpu_vas, cpu, gpa).""" - mem = q.read64(pb + 8) - if not mem or not is_kern(mem[0]): - return [], None, None - vas = q.read64(mem[0] + 0x10, 16) + chans = {int(m, 16) for m in re.findall(r"ch=0x([0-9a-fA-F]+)", out)} + return sems, pbs, chans + + +def vas_from_mem(q, mem, d): + """uvm_rm_mem_t pointer -> (gpu_vas, cpu, gpa) or None.""" + if not is_kern(mem): + diag_add(d, "badmem") + return None + vas = q.read64(mem + 0x10, 16) + if not vas: + diag_add(d, "empty_vas") + return None gpu_vas = [v for v in vas if is_gpu(v)] cpu = next((v for v in vas if is_vmalloc(v)), None) - if not gpu_vas or not cpu: - return [], None, None + if not gpu_vas: + diag_add(d, "no_gpuva") + return None + if not cpu: + diag_add(d, "no_vmalloc") + return None gpa = q.gva2gpa(cpu) if not gpa: - return [], None, None + diag_add(d, "no_gpa") + return None return gpu_vas, cpu, gpa +def pushbuffer_of_manager(q, mgr, d): + pb = q.read64(mgr + 8) + if not pb or not is_kern(pb[0]): + diag_add(d, "stop_manager") + return None + mem = q.read64(pb[0] + 8) + if not mem or not is_kern(mem[0]): + diag_add(d, "stop_pushbuffer") + return None + return vas_from_mem(q, mem[0], d) + + +def pushbuffer_of_pool(q, pool, d): + mgr = q.read64(pool) + if not mgr or not is_kern(mgr[0]): + diag_add(d, "stop_pool") + return None + return pushbuffer_of_manager(q, mgr[0], d) + + +def best_candidate(found, d): + """Prefer a chain whose rm_mem has a GPU VA in the CE pushbuffer range.""" + if not found: + return None + for res in found: + if any(is_pushbuf_gpu(v) for v in res[0]): + return res + gpu_vas = found[0][0] + diag_add(d, "range_miss") + d["_near"] = "0x%x" % gpu_vas[0] + return found[0] + + +def chains_from_window(q, words, d): + """Interpret kernel pointers in the scanned window as channel/pool/ + manager/pushbuffer pointers and walk the chain.""" + cands = list(dict.fromkeys(w for w in words if is_kern(w))) + d["cands"] = len(cands) + found = [] + for w in cands[:CHAN_MAX_CANDS]: + res = pushbuffer_of_pool(q, w, d) + if res: + found.append(res) + continue + ch = q.read64(w) + if ch and is_kern(ch[0]): + res = pushbuffer_of_pool(q, ch[0], d) + if res: + found.append(res) + continue + res = pushbuffer_of_manager(q, w, d) + if res: + found.append(res) + continue + pb = q.read64(w + 8) + if pb and is_kern(pb[0]): + res = vas_from_mem(q, pb[0], d) + if res: + found.append(res) + return best_candidate(found, d) + + +def resolve_from_channel(q, ch, d): + """uvm_channel_t * -> pool -> manager -> pushbuffer -> rm_mem.""" + d["ch"] = 1 + pool = q.read64(ch) + if not pool or not is_kern(pool[0]): + diag_add(d, "stop_channel") + return None + res = pushbuffer_of_pool(q, pool[0], d) + return best_candidate([res] if res else [], d) + + +def resolve_from_pushbuffer(q, pb, d): + d["pb"] = 1 + mem = q.read64(pb + 8) + if not mem or not is_kern(mem[0]): + diag_add(d, "stop_pushbuffer") + return None + res = vas_from_mem(q, mem[0], d) + return best_candidate([res] if res else [], d) + + +def resolve_from_sem(q, sem, d): + """Scan the pages around the channel's tracking_sem for the pool pointer + (channel+0 = pool) and walk pool -> manager -> pushbuffer -> rm_mem.""" + d["sem"] = 1 + page = sem & ~0xFFF + words = [] + nback = max(1, CHAN_WINDOW_BACK // 0x1000) + for i in range(nback, -2, -1): + chunk = q.read64(page - i * 0x1000, 512) + if not chunk: + diag_add(d, "empty_chunks") + continue + words += chunk + diag_add(d, "chunks") + d["words"] = len(words) + if not words: + return None + return chains_from_window(q, words, d) + + +def resolve_pushbuffer_wide(q, sem, d): + """Legacy 256 KB window scan (kept as a last resort).""" + base = sem & ~0xFFF + words = [] + for i in range(64): + chunk = q.read64(base - 0x40000 + i * 0x1000, 512) + if not chunk: + diag_add(d, "empty_wide_chunks") + continue + words += chunk + d["wide_words"] = len(words) + d["wide_empty"] = d.get("empty_wide_chunks", 0) + if not words: + return None + cands = list(dict.fromkeys(w for w in words + if is_kern(w) and (w >> 32) == (sem >> 32))) + d["wide_cands"] = len(cands) + found = [] + for M in cands[:400]: + res = pushbuffer_of_manager(q, M, d) + if res: + found.append(res) + continue + pb = q.read64(M + 8) + if pb and is_kern(pb[0]): + res = vas_from_mem(q, pb[0], d) + if res: + found.append(res) + return best_candidate(found, d) + + def pin(va, pa): cmd = "printf '0x%x,0x%x' > %s/sema_map_add" % (va, pa, PARAM) for _ in range(40): @@ -155,62 +328,6 @@ def resolve_sem_pool(q, sem): return [(gv, gpa) for gv in gpu_vas] -def resolve_pushbuffer(q, sem): - """Find the pushbuffer rm_mem near the channel and return (gpu_vas, cpu, gpa).""" - base = sem & ~0xFFF - words = [] - for i in range(64): - chunk = q.read64(base - 0x40000 + i * 0x1000, 512) - if not chunk: - continue - words += chunk - # The manager/pushbuffer live in the same slab region as the channel. - prefix = sem >> 32 - cands = list(dict.fromkeys(w for w in words - if is_kern(w) and (w >> 32) == prefix)) - # Chain 1: value is the channel manager; +8 pushbuffer, +8 rm_mem. - deadline = time.time() + 45 - for M in cands[:1200]: - if time.time() > deadline: - break - pb = q.read64(M + 8) - if not pb or not is_kern(pb[0]): - continue - mem = q.read64(pb[0] + 8) - if not mem or not is_kern(mem[0]): - continue - vas = q.read64(mem[0] + 0x10, 16) - gpu_vas = [v for v in vas if is_gpu(v)] - if not gpu_vas: - continue - cpu = next((v for v in vas if is_vmalloc(v)), None) - if not cpu: - continue - gpa = q.gva2gpa(cpu) - if gpa: - return gpu_vas, cpu, gpa - # Chain 2: value is the pushbuffer itself; +8 = rm_mem. - # Two QMP reads per candidate, so cap the scan (bounded in time). - deadline = time.time() + 30 - for V in cands[:400]: - if time.time() > deadline: - break - mem = q.read64(V + 8) - if not mem or not is_kern(mem[0]): - continue - g5 = q.read64(mem[0] + 0x38) - if not g5 or not is_gpu(g5[0]): - continue - g4 = q.read64(mem[0] + 0x30) - cpu = g4[0] if g4 and is_vmalloc(g4[0]) else None - if not cpu: - continue - gpa = q.gva2gpa(cpu) - if gpa: - return [g5[0]], cpu, gpa - return [], None, None - - def pin_pushbuffer(q, gpu_vas, cpu, gpa): """Pin the whole pushbuffer with *per-page* translations (the layout is not linear across the allocation: page 32 jumped on a real client).""" @@ -241,6 +358,7 @@ def main(): q = Qmp(QMP_SOCK) seen = set() seen_pb = set() + seen_ch = set() pinned_ranges = set() def pin_range_once(gpu_vas, cpu, gpa): @@ -252,19 +370,35 @@ def main(): print("semfix: watching (interval %.1fs)" % interval, flush=True) while True: - sems, pbs = guest_trace() + sems, pbs, chans = guest_trace() + for ch in chans - seen_ch: + d = {} + try: + res = resolve_from_channel(q, ch, d) + except Exception as e: + res = None + d["exception"] = str(e)[:40] + if res: + seen_ch.add(ch) + pin_range_once(*res) + print("semfix: ch=0x%x -> %s" % (ch, diag_str(d)), flush=True) + else: + print("semfix: ch=0x%x pushbuf miss [%s]" % (ch, diag_str(d)), flush=True) for pb in pbs - seen_pb: - seen_pb.add(pb) + d = {} try: - gpu_vas, cpu, gpa = resolve_pb_ptr(q, pb) + res = resolve_from_pushbuffer(q, pb, d) except Exception as e: - print("semfix: pb=0x%x resolve failed: %s" % (pb, e), flush=True) - continue - if gpu_vas: - pin_range_once(gpu_vas, cpu, gpa) + res = None + d["exception"] = str(e)[:40] + if res: + seen_pb.add(pb) + pin_range_once(*res) + print("semfix: pb=0x%x -> %s" % (pb, diag_str(d)), flush=True) else: - print("semfix: pb=0x%x no pushbuffer rm_mem (retry)" % pb, flush=True) + print("semfix: pb=0x%x pushbuf miss [%s]" % (pb, diag_str(d)), flush=True) for sem in sems - seen: + d = {} try: pairs = resolve_sem_pool(q, sem) except Exception as e: @@ -278,15 +412,27 @@ def main(): ok = pin(gv, gpa) print("semfix: sem=0x%x va=0x%x -> gpa=0x%x pin=%s" % (sem, gv, gpa, "ok" if ok else "FAILED"), flush=True) + d = {} try: - gpu_vas, cpu, gpa = resolve_pushbuffer(q, sem) + res = resolve_from_sem(q, sem, d) except Exception as e: - print("semfix: pushbuf resolve failed: %s" % e, flush=True) - gpu_vas = [] - if gpu_vas: - pin_range_once(gpu_vas, cpu, gpa) + res = None + d["exception"] = str(e)[:40] + if not res: + for k in ("chunks", "words", "cands", "stop_pool", "no_gpuva", + "no_vmalloc", "empty_vas"): + d.pop(k, None) + try: + res = resolve_pushbuffer_wide(q, sem, d) + except Exception as e: + res = None + d["exception"] = str(e)[:40] + if res: + pin_range_once(*res) + print("semfix: sem=0x%x pushbuf -> %s" % (sem, diag_str(d)), flush=True) else: - print("semfix: pushbuf not found near sem=0x%x (retry)" % sem, flush=True) + print("semfix: sem=0x%x pushbuf not found [%s]" + % (sem, diag_str(d)), flush=True) time.sleep(interval) |