diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 82 | ||||
| -rwxr-xr-x | tools/guest-diag/semfix_uvmt.py | 305 |
2 files changed, 385 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index ce2e0c9..ae8f7da 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -350,6 +350,7 @@ struct sema_svc { u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 releases_faked; /* skipped-entry fallback */ + u64 counter_rel_applied; /* releases to registered counters */ u64 builds; u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ @@ -1766,11 +1767,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.releases_stale, svc.releases_jumped, svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits, svc_scan_cursor); - pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu\n", + pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu crel=%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail, svc.ring_ops, svc.ops_denied, - svc.false_trans, svc.gpfifo_skipped, svc.releases_faked); + svc.false_trans, svc.gpfifo_skipped, svc.releases_faked, + svc.counter_rel_applied); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -2059,6 +2061,56 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len) static u64 svc_rel_va_seen; static u32 svc_dbg_streams; +/* + * Apply a decoded release directly to a channel's registered completion + * counter slot. + * + * GR pushes release the client's per-channel completion counter and the client + * tracks push completion by polling that u32. The release VA is known from the + * push, but its VA->PA translation may be missing (the counter page is a + * client UVM allocation that is not in any scanned window) or stale (recycled + * client), while the slot's guest PA is known exactly: it is registered + * through the sema_counter parameter from the client's own pagemap. Writing + * the payload there is what lets the client's wait observe the release + * regardless of the VA map. Forward-only, so a stale copy of an + * already-completed push cannot rewind the counter. + */ +static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, + u32 payload) +{ + unsigned long flags; + u64 pa = 0; + u32 cur = 0; + int i; + + if (!h || !payload) + return; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].h_object == h) { + pa = svc.chan[i].counter_pa; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!pa) + return; + + if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) + return; + if (payload <= cur || payload - cur > 0x100000) + return; + if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &payload, + sizeof(payload))) + return; + svc.releases_applied++; + svc.counter_rel_applied++; + svc.post_hint_until = jiffies + 10 * HZ; + pr_info_ratelimited("sema-chan: counter release h=0x%x pa=0x%llx val=%u (was %u)\n", + h, pa, payload, cur); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, u32 ch_h, u32 ch_eng, u64 push_va, @@ -2482,6 +2534,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.post_hint_until = jiffies + 10 * HZ; } } + + /* + * Exact-slot fallback: the releasing channel's + * completion counter was enumerated from the + * client's pagemap, so the release can be + * applied to the very slot the client polls + * even when the release VA has no (or a stale) + * translation. Same forward-only guard as the + * VA path. + */ + svc_counter_release_apply(vgpu, ch_h, + st.sem_payload); /* release ends the push */ n_ops = 0; } @@ -3741,6 +3805,7 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { u32 val; + u32 cur = 0; if (!ch->counter_pa) return; @@ -3748,6 +3813,19 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, val = put; if (val == ch->counter_last) return; + /* + * A decoded release (svc_counter_release_apply) may already have + * advanced the slot past the put position the client last submitted; + * the client polls this u32, so never write it backwards. + */ + if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, + sizeof(cur))) + return; + if (val <= cur) { + if (cur > ch->counter_last) + ch->counter_last = cur; + return; + } if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &val, sizeof(val))) return; diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py new file mode 100755 index 0000000..12577ab --- /dev/null +++ b/tools/guest-diag/semfix_uvmt.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +# semfix_uvmt.py - single-QMP tracking-semaphore/resolution watcher. +# +# One QMP client does everything semfix_watch.py and uvmt_poll.py did, because +# QEMU's QMP socket accepts a single client: running the two tools side by side +# silently broke semfix's pinning. +# +# Jobs: +# 1. keep the client's tracking-semaphore pool pages and push-buffer pages +# pinned with fresh guest truth (semfix_watch.py, imported); +# 2. poll every tracking semaphore seen in the guest trace and print the exact +# word the client's wait polls: pool GPU VA (gpu_base + idx*4), its guest PA +# (gva2gpa of the CPU mapping), the payload value and the +0x14/+0x28 +# fields. Every change is appended to a timeline file, so the last wait +# before a failure and the release that should clear it are both visible; +# 3. mirror the host's decoded releases (dmesg `sema-push ... rel=... pay=...`) +# into the same timeline, so a wait can be matched to the channel/push that +# releases its word. +# +# Usage: tools/guest-diag/semfix_uvmt.py [interval-seconds] +# Env: TIMELINE=/tmp/opencode/uvmt_timeline.log +# UVMT_HEARTBEAT=30 (seconds between unchanged-state heartbeats) +import os +import re +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from semfix_watch import ( # noqa: E402 + Qmp, QMP_SOCK, is_kern, is_gpu, diag_str, guest_trace, + resolve_sem_pool, pin, pin_bulk, resolve_from_channel, + resolve_from_pushbuffer, resolve_from_sem, resolve_pushbuffer_wide, + pin_pushbuffer, +) + +TIMELINE = os.environ.get("TIMELINE", "/tmp/opencode/uvmt_timeline.log") +HEARTBEAT = float(os.environ.get("UVMT_HEARTBEAT", "30")) +DLOG = os.environ.get("UVMT_DMESG", "1") == "1" +VMEXEC = os.path.normpath(os.path.join(HERE, "..", "..", "scripts", "vgpu_vm.py")) +COUNTER_LO = int(os.environ.get("COUNTER_LO", "0x20400f000"), 16) +COUNTER_HI = int(os.environ.get("COUNTER_HI", "0x204010000"), 16) + + +def log(msg): + line = "[%s] %s" % (time.strftime("%H:%M:%S"), msg) + print(line, flush=True) + try: + with open(TIMELINE, "a") as f: + f.write(line + "\n") + except OSError: + pass + + +def uvmt_state(q, sem): + """Read one tracking semaphore; returns a dict or None. + + Layout (uvm 570.144, verified by probe): sem+0 = tracking semaphore page + object, sem+4 low u16 = pool index; page+0 = uvm_rm_mem; mem+0x10 = vas[]; + the payload the client's update_completed_value polls is the u32 at + cpu_va + idx*4 (its GPU alias is gpu_va + idx*4, the VA the releasing push + writes). + """ + vals = q.read64(sem, 12) + if len(vals) < 2 or not is_kern(vals[0]): + return None + pv = q.read64(vals[0]) + if not pv or not is_kern(pv[0]): + return None + mem = pv[0] + vas = q.read64(mem + 0x10, 16) + cpu = next((v for v in vas if is_kern(v)), None) + gpu = [v for v in vas if is_gpu(v)] + idx = vals[1] & 0xffff + payload = None + word_pa = None + if cpu is not None: + w = q.read64(cpu + idx * 4) + payload = w[0] if w else None + word_pa = q.gva2gpa(cpu + idx * 4) + word_va = (gpu[0] + idx * 4) if gpu else None + return { + "page": vals[0], + "mem": mem, + "idx": idx, + "gpu": gpu, + "cpu": cpu, + "payload": payload, + "word_va": word_va, + "word_pa": word_pa, + "w14": vals[5] if len(vals) > 5 else 0, + "w28": vals[10] if len(vals) > 10 else 0, + } + + +def state_key(st): + if not st: + return None + return (st["idx"], st["payload"], st["w14"], st["w28"], st["word_pa"], + tuple(st["gpu"])) + + +def state_str(sem, st): + if not st: + return "sem=0x%x unreadable" % sem + return ("sem=0x%x idx=%d word_va=%s word_pa=%s payload=%s " + "+14=0x%x +28=0x%x gpu=%s" % + (sem, st["idx"], + "0x%x" % st["word_va"] if st["word_va"] is not None else "?", + "0x%x" % st["word_pa"] if st["word_pa"] is not None else "?", + "0x%x" % st["payload"] if st["payload"] is not None else "?", + st["w14"], st["w28"], + ",".join("0x%x" % v for v in st["gpu"]))) + + +def dmesg_releases(seen): + """Print/append new host-decoded releases (sema-push lines).""" + try: + r = subprocess.run( + ["sudo", "-n", "sh", "-c", + "dmesg -T | grep 'sema-push' | tail -300"], + capture_output=True, text=True, timeout=10) + except Exception: + return + for line in r.stdout.splitlines(): + line = line.strip() + if not line or line in seen: + continue + seen.add(line) + log("push: %s" % line.replace("gsp-resman-vfio: ", "")) + + +def cdt_pid(): + try: + r = subprocess.run([VMEXEC, "exec", "pgrep -x cdt | head -1"], + capture_output=True, text=True, timeout=30) + except Exception: + return None + pids = re.findall(r"(?m)^(\d+)\s*$", r.stdout) + return int(pids[-1]) if pids else None + + +def pin_counter_page(pid, pinned_pids): + """Pin the client's completion-counter page from its own pagemap. + + The GR channels release the counter page VA range (0x20400fec0..0x20400fff0 + observed); translating it from fresh guest truth before the first releases + decode is what lets those releases land on the slots the client polls. + """ + cmd = ("sudo python3 /tmp/guest_pmap_win.py %d 0x%x 0x%x 64" + % (pid, COUNTER_LO, COUNTER_HI)) + try: + r = subprocess.run([VMEXEC, "exec", cmd], capture_output=True, + text=True, timeout=60) + except Exception: + return False + n = 0 + for m in re.finditer(r"^(0x[0-9a-f]+),(0x[0-9a-f]+),(\d+)$", + r.stdout, re.M): + va, pa, pages = int(m.group(1), 16), int(m.group(2), 16), int(m.group(3)) + if pin_bulk(va, pa, pages * 0x1000): + n += pages + if n: + pinned_pids.add(pid) + log("counter: pinned %d page(s) at pid %d (0x%x..0x%x)" + % (n, pid, COUNTER_LO, COUNTER_HI)) + return n > 0 + + +def main(): + interval = float(sys.argv[1]) if len(sys.argv) > 1 else 2.0 + os.environ.setdefault("KEEP_TRACE", "1") + q = Qmp(QMP_SOCK) + os.makedirs(os.path.dirname(TIMELINE), exist_ok=True) + log("semfix_uvmt: watching (interval %.1fs, qmp %s)" % (interval, QMP_SOCK)) + + known_sems = set() + seen_sem_pin = set() + seen_pb = set() + seen_ch = set() + pinned_ranges = set() + last_state = {} + last_seen_t = {} + dseen = set() + counter_pids = set() + last_pid = None + + def pin_range_once(gpu_vas, cpu, gpa, size=0): + key = tuple(gpu_vas), cpu, size + if key in pinned_ranges: + return + pinned_ranges.add(key) + pin_pushbuffer(q, gpu_vas, cpu, gpa, size) + + while True: + sems, pbs, chans = guest_trace() + + # Poll every sem ever seen: the client's final wait may only appear in + # the trace once, and the trace ring wraps while it spins. + for sem in sorted(sems - known_sems): + known_sems.add(sem) + + for sem in sorted(known_sems): + try: + st = uvmt_state(q, sem) + except Exception: + st = None + key = state_key(st) + now = time.time() + if key != last_state.get(sem): + last_state[sem] = key + last_seen_t[sem] = now + log("uvmt %s" % state_str(sem, st)) + elif now - last_seen_t.get(sem, 0) >= HEARTBEAT: + last_seen_t[sem] = now + log("uvmt(hb) %s" % state_str(sem, st)) + + # Pin the pool page once per (sem, word) resolution. The payload + # word is what the releasing push writes; pinning the page makes + # the host able to apply that release. + if st and sem not in seen_sem_pin: + try: + pairs = resolve_sem_pool(q, sem) + except Exception as e: + pairs = [] + log("semfix: sem=0x%x resolve failed: %s" % (sem, e)) + if pairs: + seen_sem_pin.add(sem) + for gv, gpa in pairs: + ok = pin(gv, gpa) + log("semfix: sem=0x%x pool va=0x%x -> gpa=0x%x pin=%s" + % (sem, gv, gpa, "ok" if ok else "FAILED")) + d = {} + try: + res = resolve_from_sem(q, sem, d) + except Exception as e: + res = None + d["exception"] = str(e)[:40] + if not res: + for k in ("chunks", "words", "cands", "stop_pool", + "no_gpuva", "no_vmalloc", "empty_vas"): + d.pop(k, None) + try: + res = resolve_pushbuffer_wide(q, sem, d) + except Exception as e: + res = None + d["exception"] = str(e)[:40] + if res: + pin_range_once(*res) + log("semfix: sem=0x%x pushbuf -> %s" + % (sem, diag_str(d))) + else: + log("semfix: sem=0x%x pushbuf not found [%s]" + % (sem, diag_str(d))) + else: + log("semfix: sem=0x%x pool unresolved (retry)" % sem) + + for ch in sorted(chans - seen_ch): + d = {} + try: + res = resolve_from_channel(q, ch, d) + except Exception as e: + res = None + d["exception"] = str(e)[:40] + if res: + seen_ch.add(ch) + pin_range_once(*res) + log("semfix: ch=0x%x -> %s" % (ch, diag_str(d))) + else: + log("semfix: ch=0x%x pushbuf miss [%s]" % (ch, diag_str(d))) + for pb in sorted(pbs - seen_pb): + d = {} + try: + res = resolve_from_pushbuffer(q, pb, d) + except Exception as e: + res = None + d["exception"] = str(e)[:40] + if res: + seen_pb.add(pb) + pin_range_once(*res) + log("semfix: pb=0x%x -> %s" % (pb, diag_str(d))) + else: + log("semfix: pb=0x%x pushbuf miss [%s]" % (pb, diag_str(d))) + + if DLOG: + dmesg_releases(dseen) + + # Pin the completion-counter page of a newly started client, before its + # first GR pushes decode their releases. + pid = cdt_pid() + if pid and pid != last_pid: + last_pid = pid + log("counter: client pid %d" % pid) + if pid and pid not in counter_pids: + try: + pin_counter_page(pid, counter_pids) + except Exception as e: + log("counter: pin failed: %s" % e) + + time.sleep(interval) + + +if __name__ == "__main__": + main() |