From 6a88588bfddc9532becea5d14492ddb51d4eb6d4 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 25 Oct 2026 06:58:12 -0400 Subject: vgsp-rm r680 + wall-2 session-76 forensics: 0x2080a097 reply record-area tunables (wall2_a097_count/rec0..rec7, default 0xffffffff = echo) and the evidence set that re-questions the a097 gate The real host RM 0x2080a097 handler (GSP-RM 570.144 control table entry, params 0x490) overwrites params+4..+0x23 with up to four {u32,u32} mode pairs (engine value + a x1000 clock/percent) and the client copies back the first 16 bytes; the tick at eglcore 0xd5e3e0 consumes the second word of the first pair as the per-engine progress (session +0xe104, compared against per-boot config threshold 95 at lib+0x2019320+0xffc with 10/20 bit-history cutoffs). New module params let one boot sweep the record matrix; the r680 sweep (flag/v488/v48c + count/rec matrix incl. progress 100..1000000) changed nothing observable, and hw watchpoints show E+0x150 stays 0x00200000 with no writer over 5+ minutes while the a097 poller is a worker thread and the main thread sits in a 10 s poll() loop on /dev/nvidia0 inside vkCreateDevice. Tools: t75g.py (in-guest gdb state/watch probes), diag76.py (thread bt + address resolution), slots_probe.sh, run75g.sh; README-session76.md records the firmware handler decode and the sweep. --- tools/guest-diag/wall2/README-session76.md | 92 ++++++++++++++++ tools/guest-diag/wall2/diag76.py | 47 ++++++++ tools/guest-diag/wall2/run75g.sh | 41 +++++++ tools/guest-diag/wall2/slots_probe.sh | 11 ++ tools/guest-diag/wall2/t75g.py | 170 +++++++++++++++++++++++++++++ 5 files changed, 361 insertions(+) create mode 100644 tools/guest-diag/wall2/README-session76.md create mode 100644 tools/guest-diag/wall2/diag76.py create mode 100755 tools/guest-diag/wall2/run75g.sh create mode 100644 tools/guest-diag/wall2/slots_probe.sh create mode 100644 tools/guest-diag/wall2/t75g.py (limited to 'tools/guest-diag/wall2') diff --git a/tools/guest-diag/wall2/README-session76.md b/tools/guest-diag/wall2/README-session76.md new file mode 100644 index 0000000..d78ed39 --- /dev/null +++ b/tools/guest-diag/wall2/README-session76.md @@ -0,0 +1,92 @@ +# session 76 notes (r680) — the 0x2080a097 engine-monitor re-examined + +## What the firmware says (ground truth for the a097 family) + +The shipped GSP-RM 570.144 image embeds the host-side control +descriptor table (file offset 0x41df8..0x42450, entry = 0x20 bytes +{u32 cmd, u32 params_size, u32 0x4049580, u32 0, u32 handler_ptr, +u32 0, u32 out_size, u32 0}). Entries seen: + + 0x2080a07f size 0x408 handler vaddr 0x551b2c0 + 0x2080a079 size 0x14804 + 0x2080a097 size 0x490 handler vaddr 0x55166f4 + 0x2080a0d1 size 0x7e8 + ... + +The real 0x2080a097 handler (0x55166f4) writes up to FOUR {u32,u32} +mode pairs into params+4..+0x23 (params[0] low byte = the request's +count flag): for i=1..4 it calls an engine-status helper +(0x5516334 / 0x55d7f04 / 0x55d7c1c / 0x55d7d60) with mode=i and emits +{pair} = {engine value, 1000 * u16(engine+0x1bc0)}; the loop stops at +the first helper error. So the real reply overwrites the client's +request record {1,0,0,0}. + +## What the client does with the reply + +libnvidia-eglcore's local handler (file offset 0xecd2e0, the userspace +implementation of the internal cmd 0x2080a040) copies back: + + result+0 = reply+0 (count) + result+0x10 = reply+0x204 (flag) + result+0x20 = reply+0x488 + result+0x24 = reply+0x48c + count*16 bytes from reply+4 -> *(result+8) + +For the engine tick (file offset 0xd5e3e0, called from the wait loop +0xd6ef40 per iteration) the result struct IS the caller's request at +rsp+0x10, and *(result+8) points at a 16-byte "inner" buffer +(rsp+0). The tick then reads inner+4 and stores it to session+0xe104 +(the per-engine "progress" cache; poll interval ~1.25 s from the same +function). A second consumer of the tick's return value compares it +against per-boot config at lib+0x2019320: + + 0xfc8 = 1000, 0xffc = 95 (progress threshold), + 0xfec = 10, 0xfed = 20 (bit-history cutoffs), + 0x1000 = 500 + +and ORs the comparison bit into a round-robin bitfield (+0xb4) of the +state machine object. + +## Live evidence on r680 (fresh boot, full-param recipe) + +* The 0x2080a097 polls (NVOS54 0x2a, 1.25 s cadence) come from a + WORKER thread (gdb bt: start_thread -> eglcore a11b2a -> wait loop + 0xd6ef40 -> tick 0xd5e3e0). The engine object E (0x604101365e90) + sits at E+0x150 = 0x00200000 (bit 21 only), E+0x1b038 = 1, + E+0x1b039 = 1, *(E+0x1b030)+8 != 0 — all static for 5+ minutes. +* MAIN thread (vkCreateDevice) is in a poll(fd=/dev/nvidia0, events + POLLIN|POLLPRI, timeout=10s) loop inside eglcore (bt: poll <- + eglcore+0xa12de4 <- 0xd69159 <- 0xd6a02b <- 0xdcebc6 <- ... <- + libvulkan vkCreateDevice). No -4 is ever printed in this state: the + client waits indefinitely (killed by its own timeout in runs). +* GR channel kicks continue concurrently (sema-entry lines for + h=0xbeec0100 va=0x4ca0000 len=1432 etc.) and end as + "skipping stale entry ... after N failed rescues" (gr_wedge>0). +* Reply sweep on r680 (module params) — flag=0/2/1/1..0xffffffff, + v488, v48c AND the new record matrix count/rec0..rec7 = + 0/{1,0,0,0}/100/1000/100000/1000000/0xdeadbeef — produced ZERO + observable change: same a097 cadence, no writers to + E+0x150/E+0x1b038/E+0x1b039 ever caught, no client progress. The + hw watchpoints on those words fired zero times over ~5 min. + +## Tools + +* t75g.py — in-guest gdb python: dispatcher-type log, a097 issuer + dump, tick (d5e449) state + cfg dump, E/S capture, hw watchpoints on + E+0x150 / E+0x1b038 / E+0x1b039 / S+0x120, POST a040 consumption + dump. +* diag76.py — in-guest gdb python: thread-apply bt + map-relative + address resolution (used to find the main-thread poll wait). +* slots_probe.sh — read the u16 completion-slot words of every small + /dev/nvidiactl rw-s mapping (SESSION 74's slotfill ABI). +* run75g.sh — trace + record-word sweep driver. + +## Verdict + +The a097 monitor is a background worker, not an obviously blocking +gate: every knob the client reads from it can be varied with no +observable state change, while the main thread blocks elsewhere +(/dev/nvidia0 poll loop) and the GR channel-init pushes still cannot +be read by the sema service. Next lever is the main thread's wait +(what would make the /dev/nvidia0 poll return readable) or the GR +channel-init completion — not the a097 reply words. diff --git a/tools/guest-diag/wall2/diag76.py b/tools/guest-diag/wall2/diag76.py new file mode 100644 index 0000000..989abd3 --- /dev/null +++ b/tools/guest-diag/wall2/diag76.py @@ -0,0 +1,47 @@ +import gdb + +inf = gdb.selected_inferior() +pid = inf.pid +maps = [] +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + maps.append((lo, hi, off, parts[5] if len(parts) > 5 else "")) + except Exception: + pass + +def resolve(addr): + for lo, hi, off, name in maps: + if lo <= addr < hi: + return "%s+0x%x" % (name.split('/')[-1], off + (addr - lo)) + return "?" + +gdb.execute("set pagination off") +print("=== THREADS ===") +for t in inf.threads(): + try: + t.switch() + except Exception as e: + print("switch err", e) + continue + pc = int(gdb.parse_and_eval("$pc")) + print("TID %s pc=%s" % (t.ptid, resolve(pc))) + try: + bt = gdb.execute("bt 12", to_string=True) + except Exception as e: + bt = "bt err %s" % e + for ln in bt.splitlines(): + ln = ln.strip() + if ln.startswith("#"): + try: + a = int(ln.split()[1], 16) + print(" ", ln, "=>", resolve(a)) + except Exception: + print(" ", ln) + else: + print(" ", ln) +print("DIAG-DONE") +gdb.execute("detach") +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/run75g.sh b/tools/guest-diag/wall2/run75g.sh new file mode 100755 index 0000000..001a80f --- /dev/null +++ b/tools/guest-diag/wall2/run75g.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# run75g.sh - trace + record-word sweep on r680. +cd /tmp/opencode +rm -f /tmp/opencode/t75g-sweep.log +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/t75g.py ubuntu@127.0.0.1:/tmp/t75g.py >/dev/null 2>&1 +( + sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 \ + 'sudo bash /tmp/slotfill2.sh > /tmp/slotfill75g.log 2>&1 & sleep 0.5; timeout 400 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk_t75g.out 2>&1 & sleep 8; PID=$(pgrep -x vk_frame | head -1); echo "PID=$PID"; sudo timeout 380 gdb -q -batch -p $PID -x /tmp/t75g.py > /tmp/t75g.out 2>&1; echo GDB_DONE rc=$?' +) > /tmp/opencode/t75g-ssh.log 2>&1 & +GPID=$! +sleep 35 +setvar() { echo "$2" | sudo tee /sys/module/vgsp/parameters/$1 >/dev/null; } +mark() { echo "[$(date +%s.%N)] $*" >> /tmp/opencode/t75g-sweep.log; } +# phase 1: baseline echo (client already polling) +mark "baseline echo" +sleep 20 +# phase 2: all-zero records +setvar wall2_a097_rec0 0; setvar wall2_a097_rec1 0; setvar wall2_a097_rec2 0; setvar wall2_a097_rec3 0 +setvar wall2_a097_rec4 0; setvar wall2_a097_rec5 0; setvar wall2_a097_rec6 0; setvar wall2_a097_rec7 0 +mark "records all zero"; sleep 20 +# phase 3: progress=100 +setvar wall2_a097_rec1 100 +mark "rec1=100"; sleep 20 +# phase 4: progress=1000 +setvar wall2_a097_rec1 1000 +mark "rec1=1000"; sleep 20 +# phase 5: progress=100000 +setvar wall2_a097_rec1 100000 +mark "rec1=100000"; sleep 20 +# phase 6: progress=1000000 + rec0=0 +setvar wall2_a097_rec1 1000000 +setvar wall2_a097_rec0 0 +mark "rec1=1000000 rec0=0"; sleep 20 +# phase 7: all ones +setvar wall2_a097_rec0 3735928559; setvar wall2_a097_rec1 3735928559; setvar wall2_a097_rec2 3735928559; setvar wall2_a097_rec3 3735928559 +mark "rec0-3=0xdeadbeef"; sleep 20 +# phase 8: back to echo +setvar wall2_a097_rec0 4294967295; setvar wall2_a097_rec1 4294967295; setvar wall2_a097_rec2 4294967295; setvar wall2_a097_rec3 4294967295 +mark "baseline echo again" +wait $GPID +echo SWEEP-DONE diff --git a/tools/guest-diag/wall2/slots_probe.sh b/tools/guest-diag/wall2/slots_probe.sh new file mode 100644 index 0000000..71e6fb7 --- /dev/null +++ b/tools/guest-diag/wall2/slots_probe.sh @@ -0,0 +1,11 @@ +#!/bin/bash +PID=$1 +sudo grep -E 'rw-s' /proc/$PID/maps | grep nvidiactl | while read -r line; do + RNG=$(echo "$line" | awk '{print $1}') + LO=${RNG%-*}; HI=${RNG#*-} + SZ=$((0x$HI-0x$LO)) + [ $SZ -gt 262144 ] && continue + V2E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0x2e)) count=2 2>/dev/null | xxd -p) + V0E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0xe)) count=2 2>/dev/null | xxd -p) + echo "$RNG sz=$SZ w2e=$V2E w0e=$V0E" +done diff --git a/tools/guest-diag/wall2/t75g.py b/tools/guest-diag/wall2/t75g.py new file mode 100644 index 0000000..2a0881a --- /dev/null +++ b/tools/guest-diag/wall2/t75g.py @@ -0,0 +1,170 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid + +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found") + gdb.execute("detach") + sys.exit(1) +print("EGLBASE 0x%x PID %d" % (base, pid)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"out": 0, "E": 0, "S": 0, "armed150": False, "armedS": False, + "bt_done": False, "post_done": False} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def a97(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + r14 = int(gdb.parse_and_eval("$r14")) + out = u64(rbp+0x10) + state["out"] = out + q = rd(out, 0x28) + rec_ptr = int.from_bytes(q[8:16], 'little') + print("A97 out=0x%x count=%d f10=%08x f20=%08x f24=%08x recp=0x%x" % + (out, int.from_bytes(q[0:4], 'little'), int.from_bytes(q[0x10:0x14], 'little'), + int.from_bytes(q[0x20:0x24], 'little'), int.from_bytes(q[0x24:0x28], 'little'), rec_ptr)) + if rec_ptr and rec_ptr > 0x1000: + try: + rec = rd(rec_ptr, 16) + print("A97 rec=%08x %08x %08x %08x" % + tuple(int.from_bytes(rec[i*4:i*4+4], 'little') for i in range(4))) + except Exception as e: + print("A97 rec-unreadable", e) + return False + +def post(bp): + if state["post_done"]: + return False + state["post_done"] = True + rsp = int(gdb.parse_and_eval("$rsp")) + q = rd(rsp, 0x40) + print("POST rsp=0x%x out=0x%x" % (rsp, state["out"])) + for i in range(0, 0x40, 8): + v = int.from_bytes(q[i:i+8], 'little') + print("POST rsp+%02x = 0x%016x" % (i, v)) + if state["out"]: + oq = rd(state["out"], 0x40) + for i in range(0, 0x40, 8): + v = int.from_bytes(oq[i:i+8], 'little') + print("POST out+%02x = 0x%016x" % (i, v)) + return False + +def t150(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + if not state.get("vt_done"): + state["vt_done"] = True + rdi = u64(rbp+0x148) + vt = u64(rdi) if rdi else 0 + m118 = u64(vt+0x118) if vt else 0 + m170 = u64(vt+0x170) if vt else 0 + print("VT E148=0x%x vt=0x%x m118=0x%x m170=0x%x" % (rdi, vt, m118, m170)) + if not state.get("eng_done"): + state["eng_done"] = True + n = u32(rbp+8) + ids = [u32(rbp+0x54+i*4) for i in range(min(n,8))] + print("ENG n=%d ids=%s" % (n, ",".join("0x%x" % x for x in ids))) + + # cfg area pointer is rip-relative 0x2019320 at runtime = base + (0x2019320 - 0) + # eglcore base == min load addr; file offset == vaddr for the .text; data seg present? + try: + vals = rd(base + 0x2019320 + 0xf80, 0xe0) + for i in range(0, 0xe0, 0x10): + v4 = [int.from_bytes(vals[i+j:i+j+4],'little') for j in (0,4,8,12)] + print("CFG +0x%03x = %s" % (0xf80+i, ["0x%x"%x for x in v4])) + except Exception as e: + print("CFG dump err", e) + v = u32(rbp+0x150) + b38 = rd(rbp+0x1b038, 1)[0]; b39 = rd(rbp+0x1b039, 1)[0] + p = u64(rbp+0x1b030); s8 = u64(p+8) if p else 0 + print("T150 E=0x%x v=0x%08x b038=%d b039=%d p=0x%x p8=0x%x" % + (rbp, v, b38, b39, p, s8)) + if not state["bt_done"]: + state["bt_done"] = True + print(gdb.execute("bt 8", to_string=True)) + if not state["armed150"]: + state["armed150"] = True + state["E"] = rbp + try: + gdb.execute("frame 1") + S = int(gdb.parse_and_eval("$rbp")) + gdb.execute("frame 0") + print("S=0x%x S120=%08x S128=%08x" % (S, u32(S+0x120), u32(S+0x128))) + state["S"] = S + except Exception as e: + print("S capture err", e) + S = 0 + W("*(unsigned int *)0x%x" % (rbp+0x150), rbp, 0x150, 4, "W150") + W("*(unsigned char *)0x%x" % (rbp+0x1b038), rbp, 0x1b038, 1, "W038") + W("*(unsigned char *)0x%x" % (rbp+0x1b039), rbp, 0x1b039, 1, "W039") + if S: + W("*(unsigned int *)0x%x" % (S+0x120), S, 0x120, 4, "W120") + print("WATCHES armed E=0x%x S=0x%x" % (rbp, S)) + return False + +def sl(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + print("SLOOP S=0x%x S120=%08x" % (rbp, u32(rbp+0x120))) + if not state["armedS"]: + state["armedS"] = True + W("*(unsigned int *)0x%x" % (rbp+0x120), rbp, 0x120, 4, "W120") + print("W120 armed on S+0x120") + return False + +class W(gdb.Breakpoint): + def __init__(self, spec, E, off, size, tag): + super().__init__(spec, type=gdb.BP_WATCHPOINT) + self.silent = True; self.E = E; self.off = off; self.size = size + self.tag = tag; self.hits = 0 + def stop(self): + self.hits += 1 + cur = int.from_bytes(rd(self.E+self.off, self.size), 'little') + pc = int(gdb.parse_and_eval("$pc")) + print("%s v=0x%x pc=0x%x (rel 0x%x)" % (self.tag, cur, pc, pc-base)) + if self.hits <= 8: + print(gdb.execute("bt 10", to_string=True)) + return self.hits >= 400 + +B("*0x%x" % (base+0xecd43d), a97, 220) +B("*0x%x" % (base+0xd5e4d3), post, 100000) +B("*0x%x" % (base+0xd5e449), t150, 220) +B("*0x%x" % (base+0xd6ef76), sl, 240) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +print("TRACE-END") +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") -- cgit v1.2.3