diff options
39 files changed, 2394 insertions, 0 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 49758b4..b338a6f 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -3440,6 +3440,60 @@ MODULE_PARM_DESC(gsp_rm_binapi07_fwd, "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)"); /* + * session 75: reply words for the client-pending control 0x2080a097 (the + * libnvidia-eglcore engine-readiness poll). Runtime-tunable so one boot can + * sweep reply variants: the client reads +0x204, +0x488 and +0x48c back + * (file offsets 0xecd45f..0xecd475) before its engine loop re-issues the + * control. Defaults: flag=1 (the request sets it to 0), the other words 0. + */ +static uint gsp_rm_a097_flag = 1; +module_param_named(wall2_a097_flag, gsp_rm_a097_flag, uint, 0644); +MODULE_PARM_DESC(wall2_a097_flag, "0x2080a097 reply word at params+0x204 (default 1)"); +static uint gsp_rm_a097_v488; +module_param_named(wall2_a097_v488, gsp_rm_a097_v488, uint, 0644); +MODULE_PARM_DESC(wall2_a097_v488, "0x2080a097 reply word at params+0x488 (default 0)"); +static uint gsp_rm_a097_v48c; +module_param_named(wall2_a097_v48c, gsp_rm_a097_v48c, uint, 0644); +MODULE_PARM_DESC(wall2_a097_v48c, "0x2080a097 reply word at params+0x48c (default 0)"); + +/* + * session 76: the 0x2080a097 reply's RECORD area (params+4..+0x23). The + * real host RM handler overwrites up to four {u32, u32} mode pairs there + * (per-mode engine status + a x1000 clock/percent value); the client + * (count=1) copies back the first 16 bytes into its per-engine query + * buffer, and the second word of the first pair is consumed as the + * "engine progress" word (libnvidia-eglcore d5e3e0: *(inner+4) -> + * session+0xe104, compared against per-boot config thresholds). + * Each word defaults to 0xffffffff = leave the echoed request word + * untouched; any other value replaces it, so one boot can sweep the + * record matrix without rebuilding. + */ +static uint gsp_rm_a097_count = ~0u; +module_param_named(wall2_a097_count, gsp_rm_a097_count, uint, 0644); +MODULE_PARM_DESC(wall2_a097_count, "0x2080a097 reply count word at params+0 (0xffffffff = echo)"); +static uint gsp_rm_a097_rec0 = ~0u, gsp_rm_a097_rec1 = ~0u; +static uint gsp_rm_a097_rec2 = ~0u, gsp_rm_a097_rec3 = ~0u; +static uint gsp_rm_a097_rec4 = ~0u, gsp_rm_a097_rec5 = ~0u; +static uint gsp_rm_a097_rec6 = ~0u, gsp_rm_a097_rec7 = ~0u; +module_param_named(wall2_a097_rec0, gsp_rm_a097_rec0, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec0, "0x2080a097 reply word at params+0x4 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec1, gsp_rm_a097_rec1, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec1, "0x2080a097 reply word at params+0x8 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec2, gsp_rm_a097_rec2, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec2, "0x2080a097 reply word at params+0xc (0xffffffff = echo)"); +module_param_named(wall2_a097_rec3, gsp_rm_a097_rec3, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec3, "0x2080a097 reply word at params+0x10 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec4, gsp_rm_a097_rec4, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec4, "0x2080a097 reply word at params+0x14 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec5, gsp_rm_a097_rec5, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec5, "0x2080a097 reply word at params+0x18 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec6, gsp_rm_a097_rec6, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec6, "0x2080a097 reply word at params+0x1c (0xffffffff = echo)"); +module_param_named(wall2_a097_rec7, gsp_rm_a097_rec7, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec7, "0x2080a097 reply word at params+0x20 (0xffffffff = echo)"); + + +/* * The live session's oracle client/object for the 0x20810107 experiment, * created on demand; 0 until the first query builds it. One device per module * load, so file-statics are enough; reset by unload. @@ -4215,6 +4269,72 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; } + case 0x2080a097: { /* client-pending engine-context query (session 75) + * + * libnvidia-eglcore issues this via NVOS54 and then + * POLLS the reply's params words: the request sets + * +0x204 = 0, and the caller re-issues the whole + * control every ~1.8 s while its readiness check + * stays negative (observed: 32-36 polls against the + * generic all-zero reply, then the client gives up + * and vkCreateDevice returns -4). The params are an + * in/out record block: u32 count at +0, then count + * 16-byte records, then +0x204, and the client reads + * back +0, +0x204, +0x488 and +0x48c after the call + * (libnvidia-eglcore.so.570.144 file offsets + * 0xecd45f..0xecd475). The readiness wait itself + * lives in the client's engine loop + * (0xd6ef40..0xd6f0ab polling byte +0x1b038 of the + * engine object, which a91ea0 recomputes from many + * state fields), so the reply words below are + * runtime-tunable for a single-cycle sweep without + * rebuilding: the wall2_a097_* module parameters. + * + * Reply: verbatim echo (the request records are the + * client's own query data) with the tunable words + * applied. Defaults keep the completion-style flag. */ + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 0x490 && + req.paramsSize + GSP_RM_CTRL_HDR_SIZE <= + GSPRM_MAX_SINGLE_ELEM_DATA) { + u32 count = 0, flag; + static const u32 *const recp[8] = { + &gsp_rm_a097_rec0, &gsp_rm_a097_rec1, + &gsp_rm_a097_rec2, &gsp_rm_a097_rec3, + &gsp_rm_a097_rec4, &gsp_rm_a097_rec5, + &gsp_rm_a097_rec6, &gsp_rm_a097_rec7, + }; + int i; + + memcpy(rp, pp, req.paramsSize); + memcpy(&count, pp, 4); + memcpy(&flag, pp + 0x204, 4); + if (gsp_rm_a097_count != ~0u) + *(u32 *)(rp + 0) = gsp_rm_a097_count; + for (i = 0; i < 8; i++) { + if (*recp[i] != ~0u) + *(u32 *)(rp + 4 + i * 4) = *recp[i]; + } + *(u32 *)(rp + 0x204) = gsp_rm_a097_flag; + *(u32 *)(rp + 0x488) = gsp_rm_a097_v488; + *(u32 *)(rp + 0x48c) = gsp_rm_a097_v48c; + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: ctrl 0x2080a097: echoed %u bytes (count=%u, req flag=%u) -> flag=%u v488=0x%x v48c=0x%x", + req.paramsSize, count, flag, + gsp_rm_a097_flag, gsp_rm_a097_v488, + gsp_rm_a097_v48c); + out_status = 0; + out_params_size = req.paramsSize; + } else { + out_status = 0; + out_params_size = 0; + } + break; + } + default: /* Unknown control command - return success with zeroed params. * Do NOT echo the request: pure queries pass uninitialized @@ -4656,6 +4776,23 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) * For vGPU, we just acknowledge success. */ nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC UPDATE_BAR_PDE(%u) seq=%u len=%u -> SUCCESS", function, sequence, hdr->length); + { + const u8 *pl = (const u8 *)hdr + sizeof(*hdr); + u32 n = min_t(u32, hdr->length, 64); + char dump[256]; + int o = 0; + u32 i; + + for (i = 0; i < n; i += 4) { + u32 v = 0; + + memcpy(&v, pl + i, 4); + o += snprintf(dump + o, sizeof(dump) - o, + "%08x ", v); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: bar-pde payload: %s", dump); + } if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: UPDATE_BAR_PDE reply send failed"); else @@ -4684,6 +4821,31 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)", gsp_rm_rpc_name(function), function, sequence, hdr->length, rm->unknown_rpcs); + /* + * Bounded payload dump for the first unknown RPCs: the vGPU + * guest maps its client VA spaces through some RPC; the + * mapping data (va pages / PAs / memdesc handles) must be + * visible here to be captured. Never echo it back (pure + * queries would eat their own stack garbage). + */ + if (rm->unknown_rpcs <= 96 && hdr->length) { + const u8 *pl = (const u8 *)hdr + sizeof(*hdr); + u32 n = min_t(u32, hdr->length, 64); + char dump[256]; + int o = 0; + u32 i; + + for (i = 0; i < n; i += 4) { + u32 v = 0; + + memcpy(&v, pl + i, 4); + o += snprintf(dump + o, sizeof(dump) - o, + "%08x ", v); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: unknown fn=%u seq=%u len=%u payload: %s", + function, sequence, hdr->length, dump); + } if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function); else diff --git a/tools/guest-diag/wall2/README-session74.md b/tools/guest-diag/wall2/README-session74.md new file mode 100644 index 0000000..58587eb --- /dev/null +++ b/tools/guest-diag/wall2/README-session74.md @@ -0,0 +1,66 @@ +# WALL #2 / device-init completion — session 74 findings (r675) + +## The client's "11 s stalls" fully root-caused (this is NOT the channel-init release) +strace (ioctl-only) shows THREE exact 11.00 s gaps with NO ioctl in flight; a +full strace shows the client doing a pure `sched_yield` spin (~57K/s) + +`clock_gettime` (perf) inside **libnvidia-eglcore.so.570.144** — i.e. a +userspace timeout loop. gdb backtraces place the spin inside the Vulkan ICD's +EGL/GL init (`vkCreateInstance` -> libGLX_nvidia vk_icdNegotiate... -> +libEGL -> eglcore wait loop). + +## The wait target: completion slots in /dev/nvidiactl mappings +- The wait loop (eglcore file offsets ~0xb284b0 delay, ~0xb33d00 wait loop, + ~0xb1c750 slot check) polls a slot table on the client's heap object: + table at object+0x1428, count at object+0x22efc. +- Slot check: `esi` is offset by 0x43; for esi-0x43 > 4 it reads a **u16 at + slot+0xe** and treats `0xffff` as "empty/not complete". +- gdb sampling during the stall: 13164 checks with `u16@slot+0xe = 0xffff`, + 1 with `0` — the client spins until the flag changes. +- The slot pointer = a **/dev/nvidiactl rw-s mapping** (page-sized; base+0x20 + = slot). These are the client's mappings of RM memory (the channel + notifier pages: service logs `notif=0x11627f060` per channel). + +## Filling the slots (guest-side) WORKS mechanically but is not the -4 cause +- `slotfill.sh` / `slotfill2.sh`: while vk_frame runs, write 0 to + `base+0x2e` of every small /dev/nvidiactl rw-s mapping when it reads 0xffff. +- Result: the 11 s gaps vanish (5 fills within ~2 s; the client proceeds + immediately) — **but vkCreateDevice still fails -4**. So the 11 s waits are + not the failure; they are an init-side slowdown the client tolerates. + +## The next wall: the polled RM control 0x2080a097 +- With the slots filled, the client settles into **NVOS54 (ioctl nr=0x2a) + polling every ~1.25 s** (strace), and the host dmesg shows + `GSP_RM_CONTROL cmd=0x2080a097 hClient=0xc1d000bb hObject=0xbeef0004 + paramsSize=1168 flags=0x0` -> `unknown cmd=0x2080a097 -> generic SUCCESS` + (all-zero reply), 36 polls in 45 s. +- The NVOS54 struct (gdb dump): {hClient=0xc1d000c0, hObject=0xbeef0004, + cmd=0x2080a097, flags=0, params=ptr, paramsSize=0x490, status=ptr, ...}. +- The params buffer = 1168 bytes of 32-byte records with a `0x25` field at + +0x10 (dump in /tmp/opencode evidence; gdbparam.sh captures it live). +- The client polls until the control's status/reply signals completion; our + generic all-zero reply never does. **This is the wall that must be solved to + get vkCreateDevice past -4** (either a real reply for 0x2080a097 or making + the polled status field change). + +## Other session-74 results +- The GR ring-entry writer was NOT found by disassembly-guessing: nvidia.ko + `channelFillGpFifo` and nvidia-modeset `nvWriteGpEntry` never fire for this + channel's kick (gdb hbreaks); nvidia/modeset kprobes are `notrace`; QEMU gdb + hbreak works. +- `sema_push_raw` dumps (module param) captured the service's reads of the + push: entry0 decodes only 2 methods then zeros (`consumed=0x288/1432`), + entry2 starts `SET_OBJECT 0xc597` then zeros; **no release in the push** + (`have_rel=0` for all three entries) — the channel-init release theory for + these waits is dead. +- `find_ring_va.py` / `vmap_watch2.py`: the ring entries are NOT in any + vmallocinfo area during the run; the nv_vm_map_pages areas hold the CE + pushes; the GR push content is not found by content searches. + +## Files +- gdbpoll6.gdb, gdbu16.gdb, gdbfill.gdb, gdbwatch.gdb, gdbparam.sh, + gdbgap3.sh, gdbspin.sh — the gdb probes (slot check, u16 flag, fill, + NVOS54 param dump, backtraces). +- slotfill.sh / slotfill2.sh — the guest-side slot-fill watcher. +- stracevk.sh, fastrun.sh, fastrun2.sh — the strace/perf recipes. +- perfstall5.sh — perf sampling with map-relative offsets. +- find_ring_va.py, vmap_watch2.py — the vmap/kcore watchers. diff --git a/tools/guest-diag/wall2/README-session76.md b/tools/guest-diag/wall2/README-session76.md new file mode 100644 index 0000000..d78ed39 --- /dev/null +++ b/tools/guest-diag/wall2/README-session76.md @@ -0,0 +1,92 @@ +# session 76 notes (r680) — the 0x2080a097 engine-monitor re-examined + +## What the firmware says (ground truth for the a097 family) + +The shipped GSP-RM 570.144 image embeds the host-side control +descriptor table (file offset 0x41df8..0x42450, entry = 0x20 bytes +{u32 cmd, u32 params_size, u32 0x4049580, u32 0, u32 handler_ptr, +u32 0, u32 out_size, u32 0}). Entries seen: + + 0x2080a07f size 0x408 handler vaddr 0x551b2c0 + 0x2080a079 size 0x14804 + 0x2080a097 size 0x490 handler vaddr 0x55166f4 + 0x2080a0d1 size 0x7e8 + ... + +The real 0x2080a097 handler (0x55166f4) writes up to FOUR {u32,u32} +mode pairs into params+4..+0x23 (params[0] low byte = the request's +count flag): for i=1..4 it calls an engine-status helper +(0x5516334 / 0x55d7f04 / 0x55d7c1c / 0x55d7d60) with mode=i and emits +{pair} = {engine value, 1000 * u16(engine+0x1bc0)}; the loop stops at +the first helper error. So the real reply overwrites the client's +request record {1,0,0,0}. + +## What the client does with the reply + +libnvidia-eglcore's local handler (file offset 0xecd2e0, the userspace +implementation of the internal cmd 0x2080a040) copies back: + + result+0 = reply+0 (count) + result+0x10 = reply+0x204 (flag) + result+0x20 = reply+0x488 + result+0x24 = reply+0x48c + count*16 bytes from reply+4 -> *(result+8) + +For the engine tick (file offset 0xd5e3e0, called from the wait loop +0xd6ef40 per iteration) the result struct IS the caller's request at +rsp+0x10, and *(result+8) points at a 16-byte "inner" buffer +(rsp+0). The tick then reads inner+4 and stores it to session+0xe104 +(the per-engine "progress" cache; poll interval ~1.25 s from the same +function). A second consumer of the tick's return value compares it +against per-boot config at lib+0x2019320: + + 0xfc8 = 1000, 0xffc = 95 (progress threshold), + 0xfec = 10, 0xfed = 20 (bit-history cutoffs), + 0x1000 = 500 + +and ORs the comparison bit into a round-robin bitfield (+0xb4) of the +state machine object. + +## Live evidence on r680 (fresh boot, full-param recipe) + +* The 0x2080a097 polls (NVOS54 0x2a, 1.25 s cadence) come from a + WORKER thread (gdb bt: start_thread -> eglcore a11b2a -> wait loop + 0xd6ef40 -> tick 0xd5e3e0). The engine object E (0x604101365e90) + sits at E+0x150 = 0x00200000 (bit 21 only), E+0x1b038 = 1, + E+0x1b039 = 1, *(E+0x1b030)+8 != 0 — all static for 5+ minutes. +* MAIN thread (vkCreateDevice) is in a poll(fd=/dev/nvidia0, events + POLLIN|POLLPRI, timeout=10s) loop inside eglcore (bt: poll <- + eglcore+0xa12de4 <- 0xd69159 <- 0xd6a02b <- 0xdcebc6 <- ... <- + libvulkan vkCreateDevice). No -4 is ever printed in this state: the + client waits indefinitely (killed by its own timeout in runs). +* GR channel kicks continue concurrently (sema-entry lines for + h=0xbeec0100 va=0x4ca0000 len=1432 etc.) and end as + "skipping stale entry ... after N failed rescues" (gr_wedge>0). +* Reply sweep on r680 (module params) — flag=0/2/1/1..0xffffffff, + v488, v48c AND the new record matrix count/rec0..rec7 = + 0/{1,0,0,0}/100/1000/100000/1000000/0xdeadbeef — produced ZERO + observable change: same a097 cadence, no writers to + E+0x150/E+0x1b038/E+0x1b039 ever caught, no client progress. The + hw watchpoints on those words fired zero times over ~5 min. + +## Tools + +* t75g.py — in-guest gdb python: dispatcher-type log, a097 issuer + dump, tick (d5e449) state + cfg dump, E/S capture, hw watchpoints on + E+0x150 / E+0x1b038 / E+0x1b039 / S+0x120, POST a040 consumption + dump. +* diag76.py — in-guest gdb python: thread-apply bt + map-relative + address resolution (used to find the main-thread poll wait). +* slots_probe.sh — read the u16 completion-slot words of every small + /dev/nvidiactl rw-s mapping (SESSION 74's slotfill ABI). +* run75g.sh — trace + record-word sweep driver. + +## Verdict + +The a097 monitor is a background worker, not an obviously blocking +gate: every knob the client reads from it can be varied with no +observable state change, while the main thread blocks elsewhere +(/dev/nvidia0 poll loop) and the GR channel-init pushes still cannot +be read by the sema service. Next lever is the main thread's wait +(what would make the /dev/nvidia0 poll return readable) or the GR +channel-init completion — not the a097 reply words. diff --git a/tools/guest-diag/wall2/README.md b/tools/guest-diag/wall2/README.md new file mode 100644 index 0000000..1ef9558 --- /dev/null +++ b/tools/guest-diag/wall2/README.md @@ -0,0 +1,85 @@ +# WALL #2 (GR channel-init release) — session 73 findings + tools + +## Goal +The vk_frame Vulkan client fails `vkCreateDevice -> VKFAIL -4` because the +kernel RM kicks its GR channel (h=0xbeee0100, ring VA 0x44a0000, push VA +0x4480000) and waits ~11 s per attempt for a completion the sema service +cannot apply: it cannot translate the push VA ("false translation for push +va=0x4480000 ... rescuing"), so no release is decoded and the wait times out. + +## Findings (r674/r675, 8+ client runs) +1. The push is RM-built (nvidia.ko), NOT UVM: a kprobe on + `uvm_hal_turing_host_semaphore_release` never fired during a client run. +2. nvidia.ko / nvidia-modeset.ko functions are `notrace` -> `kprobe_events` + refuses them ("Could not probe notrace function ... [nvidia]"). UVM module + probes DO work. QEMU gdbstub hardware breakpoints (hbreak) work fine. +3. The service's rescue matcher is CE-shaped only (hdr 0x20038090, hi==1, + lo&0xfff==slot, w5/v5 shape) -> it can never match a GR push. Entries 0/1/2 + are consumed as "skipping stale entry ... after ~150 failed rescues"; + gr_wedge=1. +4. Host-side content searches of /dev/shm/vgpu-guest-ram during the stall found + NO push data: neither the CE release group (0x20038090) nor the host-class + release header (0x20050017) appears outside driver code pages. +5. Ring entry encoding (from `channelFillGpFifo` in nvidia.ko and + `nvWriteGpEntry` in nvidia-modeset.ko): [0] = va & ~3, [1] = + ((va>>32)&0xff) | ((len<<8)&0x7ffffc00) | 0x80000000 (flag bit seen on + entry[1]). +6. Neither `channelFillGpFifo` (nvidia.ko) nor `nvWriteGpEntry` + (nvidia-modeset.ko) is called during the client run (gdb hbreaks, 0 hits + for the GR channel) -> the GR channel's ring entries are written by an + unidentified writer (possibly the emulation/GSP path); the push content + was never located in guest RAM. +7. RM window records + memdescs (host-readable, stable layout): + - a page holds the client channel's window records + {VA=0x4480000, SIZE=0x20000} (push) and {VA=0x44a0000, SIZE=0x40000} + (ring), each inside head/tail list nodes; + - memdescs `{SIZE, 0x03008000, FIRST_PA, ptr}` live in the same page: + the ring memdesc's FIRST_PA == the service's validated ring pa + (verified twice: 0x1158ce000, 0x129415000) -> memdesc PA == the PA of the + window's first VA page; + - the push memdesc `{0x20000, 0x03008000, PA}` was injected as + 0x4480000 -> PA (resolver4, r675: PA 0x11595c000; "injected mapping + va=0x4480000 ... (sysmem)" in dmesg) and the service then read the page + WITHOUT the "false translation" line - but decoded no release (the page + content does not look like the method stream; later candidates read as + zero). +8. The kernel-side stall shows no GSP RPCs and no ioctls; the wait function + `channelWaitForFinishPayload` (polls `channelReadChannelMemdesc(channel, + channel->0x50)`, which reads `*(channel+0xe8)+off`) was NEVER called during + a client run (gdb hbreak) -> the RM's wait path for this kick is elsewhere. + +## Tools (this directory) +- wall2_resolver2.py — v2: CE release-group content search + tuple arrays + (pa&~0xfff, strides 0x20/0x40/0x60). No candidates live. +- wall2_resolver3.py — v3: ring memdesc -> same-page push memdesc recipe. +- wall2_resolver4.py — v4: window-record pair + ring memdesc cross-check + (this one injected successfully; rdesc[0] bug fixed in v5). +- wall2_resolver5.py — v5: continuous scan (no ring-line gate), injects every + 0x20000 memdesc PA found in pages holding both window records. +- wall2_livecap{,2,3}.py — live stall-time scanners for the CE group and the + host release header (both negative; useful for re-checks). +- scan_r673*.py — RAM structure analysis (ring-PA refs, record pages, memdesc + scans). +- trace_push674.py — VA-reference tracer with direct-map pointer following. +- gdb675*.gdb — QEMU gdbstub scripts (channelFillGpFifo / nvWriteGpEntry + probes, channel struct dumps). + +## Next steps (ranked) +1. Find the GR ring-entry writer (gdb watchpoint on the ring page's CPU VA; + get the VA from the channel struct; find the channel struct via the ring + memdesc page's head/tail pointers). The writer's caller holds the push's + CPU mapping -> PA -> inject -> service decodes the release. +2. Or: gdb-break at the stall on the RM's wait (sample the sleeping task's + kernel stack; the current candidates never fired) and read the semaphore + value it polls; bump it host-side via the ring-page PA (the finish payload + may live at ring_cpu + channel->0x50). +3. Or: fix the service's rescue matcher to accept GR-shaped releases once the + push content is captured (see item 1). + +## Ops +- pkill -f self-matches: the bracket trick fails when the pattern text appears + elsewhere in the agent's own command line; kill by explicit PID from + `ps -eo pid,comm,args | awk '$2=="python3" && /<script>/ {print $1}'`. +- r674 wedged after a gdb-interrupted run; fresh cycle r675 used (no host + reboot). gdb674d/gdb675d sessions left attached (never kill gdbs). +- Never reload the guest nvidia modules; never kill clients mid-flight. diff --git a/tools/guest-diag/wall2/diag76.py b/tools/guest-diag/wall2/diag76.py new file mode 100644 index 0000000..989abd3 --- /dev/null +++ b/tools/guest-diag/wall2/diag76.py @@ -0,0 +1,47 @@ +import gdb + +inf = gdb.selected_inferior() +pid = inf.pid +maps = [] +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + maps.append((lo, hi, off, parts[5] if len(parts) > 5 else "")) + except Exception: + pass + +def resolve(addr): + for lo, hi, off, name in maps: + if lo <= addr < hi: + return "%s+0x%x" % (name.split('/')[-1], off + (addr - lo)) + return "?" + +gdb.execute("set pagination off") +print("=== THREADS ===") +for t in inf.threads(): + try: + t.switch() + except Exception as e: + print("switch err", e) + continue + pc = int(gdb.parse_and_eval("$pc")) + print("TID %s pc=%s" % (t.ptid, resolve(pc))) + try: + bt = gdb.execute("bt 12", to_string=True) + except Exception as e: + bt = "bt err %s" % e + for ln in bt.splitlines(): + ln = ln.strip() + if ln.startswith("#"): + try: + a = int(ln.split()[1], 16) + print(" ", ln, "=>", resolve(a)) + except Exception: + print(" ", ln) + else: + print(" ", ln) +print("DIAG-DONE") +gdb.execute("detach") +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/fastrun.sh b/tools/guest-diag/wall2/fastrun.sh new file mode 100644 index 0000000..73b9d80 --- /dev/null +++ b/tools/guest-diag/wall2/fastrun.sh @@ -0,0 +1,10 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /tmp/slotfill2.log 2>&1 & ) +sleep 0.5 +sudo timeout 60 strace -f -T -tt -e trace=ioctl -o /tmp/vk_strace3.out /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk77.out 2>&1 +echo "=== client ===" +cat /tmp/vk77.out +echo "=== last ioctls ===" +tail -12 /tmp/vk_strace3.out diff --git a/tools/guest-diag/wall2/fastrun2.sh b/tools/guest-diag/wall2/fastrun2.sh new file mode 100644 index 0000000..3c7b943 --- /dev/null +++ b/tools/guest-diag/wall2/fastrun2.sh @@ -0,0 +1,12 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /dev/null 2>&1 & ) +sleep 0.5 +sudo timeout 45 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk78.out 2>&1 +echo "=== client ===" +cat /tmp/vk78.out +echo "=== repeated controls (top) ===" +sudo dmesg -T | grep -oE "GSP_RM_CONTROL cmd=0x[0-9a-f]+" | sort | uniq -c | sort -rn | head -12 +echo "=== last controls ===" +sudo dmesg -T | grep "GSP_RM_CONTROL cmd" | tail -8 diff --git a/tools/guest-diag/wall2/find_ring_va.py b/tools/guest-diag/wall2/find_ring_va.py new file mode 100644 index 0000000..c02ea40 --- /dev/null +++ b/tools/guest-diag/wall2/find_ring_va.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Guest-side: read the nv_vm_map_pages vmap areas via /proc/kcore and find the +# GPFIFO entry pattern (entry0 raw 0x0005980004480000 for va=0x4480000 len=1432). +import struct, sys +pat = bytes.fromhex("0000480400980500") # 0x0005980004480000 LE +class Kcore: + def __init__(self): + self.fd = open("/proc/kcore", "rb") + hdr = self.fd.read(64) + e_phoff = struct.unpack_from("<Q", hdr, 32)[0] + e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54) + self.segs = [] + for i in range(e_phnum): + self.fd.seek(e_phoff + i * e_phentsize) + ph = self.fd.read(56) + p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from("<IIQQQQ", ph, 0) + if p_type == 1: + self.segs.append((p_va, p_va + p_sz, p_off)) + def read(self, va, n): + for lo, hi, off in self.segs: + if lo <= va and va + n <= hi: + self.fd.seek(off + (va - lo)) + return self.fd.read(n) + return None +kc = Kcore() +areas = [] +for line in open("/proc/vmallocinfo"): + p = line.split() + if len(p) < 2: continue + if "nv_vm_map_pages" not in line: continue + lo, hi = p[0].split("-") + areas.append((int(lo, 16), int(hi, 16))) +print("nv_vm_map_pages areas:", len(areas)) +for lo, hi in areas: + size = hi - lo + d = kc.read(lo, size) + if not d: + print(" 0x%x-0x%x unreadable" % (lo, hi)); continue + idx = d.find(pat) + print(" 0x%x-0x%x size=0x%x hit=%s" % (lo, hi, size, ("+0x%x -> VA 0x%x" % (idx, lo + idx)) if idx >= 0 else "no")) + if idx >= 0: + # dump the page around the hit + base = lo + (idx & ~0xfff) + page = kc.read(base, 0x100) + print(" page head: %s" % page.hex()) diff --git a/tools/guest-diag/wall2/gdb675.gdb b/tools/guest-diag/wall2/gdb675.gdb new file mode 100644 index 0000000..e4b73ce --- /dev/null +++ b/tools/guest-diag/wall2/gdb675.gdb @@ -0,0 +1,21 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc0d0b9f0 +condition 1 *(unsigned int*)($rdi+0x110) == 0xc46f +commands 1 +silent +printf "GRFILL ch=%p idx=%d len=%d\n", $rdi, $esi, $edx +printf " +0x048=%p +0x0b8=%p +0x0e8=%p\n", *(unsigned long*)($rdi+0x48), *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0xe8) +printf " +0x060=%p +0x0c0=%p +0x0c8=%p +0x0d0=%p +0x0d8=%p\n", *(unsigned long*)($rdi+0x60), *(unsigned long*)($rdi+0xc0), *(unsigned long*)($rdi+0xc8), *(unsigned long*)($rdi+0xd0), *(unsigned long*)($rdi+0xd8) +printf " +0x0e0=%p +0x0f0=%p +0x0f8=%p +0x108=%p +0x140=%p +0x148=%p\n", *(unsigned long*)($rdi+0xe0), *(unsigned long*)($rdi+0xf0), *(unsigned long*)($rdi+0xf8), *(unsigned long*)($rdi+0x108), *(unsigned long*)($rdi+0x140), *(unsigned long*)($rdi+0x148) +x/8gx $rdi+0x40 +x/8gx $rdi+0x80 +x/8gx $rdi+0xc0 +x/8gx $rdi+0xe0 +x/8gx $rdi+0x100 +x/8gx $rdi+0x140 +continue +end +continue diff --git a/tools/guest-diag/wall2/gdb675c.gdb b/tools/guest-diag/wall2/gdb675c.gdb new file mode 100644 index 0000000..25b5ca0 --- /dev/null +++ b/tools/guest-diag/wall2/gdb675c.gdb @@ -0,0 +1,12 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc0d0b9f0 +commands 1 +silent +printf "FILL ch=%p cls100=0x%x b8=0x%x stride48=0x%x idx=%d len=%d\n", $rdi, *(unsigned int*)($rdi+0x100), *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0x48), $esi, $edx +printf " d0=%p d8=%p e8=%p f8=%p 60=%p\n", *(unsigned long*)($rdi+0xd0), *(unsigned long*)($rdi+0xd8), *(unsigned long*)($rdi+0xe8), *(unsigned long*)($rdi+0xf8), *(unsigned long*)($rdi+0x60) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdb675d.gdb b/tools/guest-diag/wall2/gdb675d.gdb new file mode 100644 index 0000000..f9dc460 --- /dev/null +++ b/tools/guest-diag/wall2/gdb675d.gdb @@ -0,0 +1,11 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc1605fd0 +commands 1 +silent +printf "NVWRITE ch=%p idx=%u len=%u b8=0x%x f68=%p f50=0x%x fb0=0x%x fc0=0x%x f198=%p\n", $rdi, $esi, $edx, *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0x68), *(unsigned int*)($rdi+0x50), *(unsigned int*)($rdi+0xb0), *(unsigned int*)($rdi+0xc0), *(unsigned long*)($rdi+0x198) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbfill.gdb b/tools/guest-diag/wall2/gdbfill.gdb new file mode 100644 index 0000000..41d1ea3 --- /dev/null +++ b/tools/guest-diag/wall2/gdbfill.gdb @@ -0,0 +1,13 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +set $e = *(unsigned long*)($rdi+0x1428) +if *(unsigned short*)($e+0x2e) == 0xffff + set {unsigned short}($e+0x2e) = 0 + printf "FILLED slot at %p+0x2e -> 0\n", $e +end +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbgap3.sh b/tools/guest-diag/wall2/gdbgap3.sh new file mode 100644 index 0000000..bf19370 --- /dev/null +++ b/tools/guest-diag/wall2/gdbgap3.sh @@ -0,0 +1,28 @@ +#!/bin/bash +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk65.out 2>&1 &) +sleep 5 +PID=$(pgrep -x vk_frame | head -1) +cp /proc/$PID/maps /tmp/maps_d.txt +gdb -p $PID -batch -ex "set pagination off" -ex "bt 12" 2>&1 | grep -E "^#" > /tmp/bt_gap3.txt +python3 - <<'PY' +import re +maps=open('/tmp/maps_d.txt').read().splitlines() +segs=[] +for m in maps: + p=m.split() + if len(p)<6: continue + lo,hi=[int(x,16) for x in p[0].split('-')] + off=int(p[2],16) + segs.append((lo,hi,off,p[5])) +for line in open('/tmp/bt_gap3.txt'): + m=re.match(r'#(\d+)\s+0x([0-9a-f]+)', line) + if not m: continue + a=int(m.group(2),16) + for lo,hi,off,name in segs: + if lo<=a<hi: + print('#%s %s + 0x%x' % (m.group(1), name.split('/')[-1], off+(a-lo))) + break + else: + print('#%s 0x%x unmapped' % (m.group(1), a)) +PY diff --git a/tools/guest-diag/wall2/gdbparam.sh b/tools/guest-diag/wall2/gdbparam.sh new file mode 100644 index 0000000..3f30d49 --- /dev/null +++ b/tools/guest-diag/wall2/gdbparam.sh @@ -0,0 +1,16 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /dev/null 2>&1 & ) +sleep 0.5 +(timeout 60 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk79.out 2>&1 &) +sleep 5 +PID=$(pgrep -x vk_frame | head -1) +gdb -p $PID -batch \ + -ex "set pagination off" \ + -ex "catch syscall ioctl" \ + -ex "condition 1 \$rsi == 0xc020462a" \ + -ex "continue" \ + -ex "printf \"ioctl nr=0x2a arg=%p\n\", \$rdx" \ + -ex "x/40wx \$rdx" \ + -ex "detach" 2>/dev/null | tail -45 diff --git a/tools/guest-diag/wall2/gdbpoll6.gdb b/tools/guest-diag/wall2/gdbpoll6.gdb new file mode 100644 index 0000000..d7fccfd --- /dev/null +++ b/tools/guest-diag/wall2/gdbpoll6.gdb @@ -0,0 +1,9 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +printf "CHK rsi=%d cnt=%d e0=%p v=%x\n", $rsi, *(int*)($rdi+0x22efc), *(unsigned long*)($rdi+0x1428), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)+($rsi<<4)) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbspin.sh b/tools/guest-diag/wall2/gdbspin.sh new file mode 100644 index 0000000..989e584 --- /dev/null +++ b/tools/guest-diag/wall2/gdbspin.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk57.out 2>&1 &) +sleep 25 +PID=$(pgrep -x vk_frame | head -1) +echo "pid=$PID" +BASE=$(grep 'eglcore' /proc/$PID/maps | head -1 | cut -d- -f1) +echo "base=$BASE" +SPIN=$(printf '0x%x' $((0x$BASE + 0xa10e90))) +echo "spin=$SPIN" +gdb -p $PID -batch \ + -ex "set pagination off" \ + -ex "break *$SPIN" \ + -ex "continue" \ + -ex "printf \"HIT rip=%p rdi=%p rsi=%p rdx=%p rbx=%p rbp=%p\n\", \$rip, \$rdi, \$rsi, \$rdx, \$rbx, \$rbp" \ + -ex "bt" \ + -ex "info registers r12 r13 r14 r15" \ + -ex "x/12i \$rip-24" \ + -ex "detach" 2>&1 | tail -40 diff --git a/tools/guest-diag/wall2/gdbu16.gdb b/tools/guest-diag/wall2/gdbu16.gdb new file mode 100644 index 0000000..61f335b --- /dev/null +++ b/tools/guest-diag/wall2/gdbu16.gdb @@ -0,0 +1,9 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +printf "CHK rsi=%d cnt=%d e0=%p u16@2e=%x u32@20=%x u32@0=%x\n", $rsi, *(int*)($rdi+0x22efc), *(unsigned long*)($rdi+0x1428), *(unsigned short*)(*(unsigned long*)($rdi+0x1428)+0x2e), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)+0x20), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbwatch.gdb b/tools/guest-diag/wall2/gdbwatch.gdb new file mode 100644 index 0000000..36b3339 --- /dev/null +++ b/tools/guest-diag/wall2/gdbwatch.gdb @@ -0,0 +1,18 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +set $e = *(unsigned long*)($rdi+0x1428) +watch *(unsigned short*)($e+0x2e) +printf "WATCH set at %p+0x2e\n", $e +disable 1 +continue +end +commands 2 +silent +printf "WRITE to slot+0x2e: old=%x new=%x by pc=%p\n", *(unsigned short*)($e+0x2e), $eax, $pc +bt 3 +continue +end +continue diff --git a/tools/guest-diag/wall2/perfstall5.sh b/tools/guest-diag/wall2/perfstall5.sh new file mode 100644 index 0000000..ae1c161 --- /dev/null +++ b/tools/guest-diag/wall2/perfstall5.sh @@ -0,0 +1,9 @@ +#!/bin/sh +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk55.out 2>&1 &) +sleep 25 +PID=$(pgrep -x vk_frame | head -1) +cp /proc/$PID/maps /tmp/vk_maps2.txt +sudo perf record -p $PID -g -o /tmp/perf6.data -- sleep 3 2>/dev/null +sudo perf script -i /tmp/perf6.data -F ip,dso 2>/dev/null | grep eglcore | awk '{print $1}' | sort | uniq -c | sort -rn | head -8 > /tmp/spin_addrs.txt +cat /tmp/spin_addrs.txt diff --git a/tools/guest-diag/wall2/run75g.sh b/tools/guest-diag/wall2/run75g.sh new file mode 100755 index 0000000..001a80f --- /dev/null +++ b/tools/guest-diag/wall2/run75g.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# run75g.sh - trace + record-word sweep on r680. +cd /tmp/opencode +rm -f /tmp/opencode/t75g-sweep.log +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/t75g.py ubuntu@127.0.0.1:/tmp/t75g.py >/dev/null 2>&1 +( + sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 \ + 'sudo bash /tmp/slotfill2.sh > /tmp/slotfill75g.log 2>&1 & sleep 0.5; timeout 400 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk_t75g.out 2>&1 & sleep 8; PID=$(pgrep -x vk_frame | head -1); echo "PID=$PID"; sudo timeout 380 gdb -q -batch -p $PID -x /tmp/t75g.py > /tmp/t75g.out 2>&1; echo GDB_DONE rc=$?' +) > /tmp/opencode/t75g-ssh.log 2>&1 & +GPID=$! +sleep 35 +setvar() { echo "$2" | sudo tee /sys/module/vgsp/parameters/$1 >/dev/null; } +mark() { echo "[$(date +%s.%N)] $*" >> /tmp/opencode/t75g-sweep.log; } +# phase 1: baseline echo (client already polling) +mark "baseline echo" +sleep 20 +# phase 2: all-zero records +setvar wall2_a097_rec0 0; setvar wall2_a097_rec1 0; setvar wall2_a097_rec2 0; setvar wall2_a097_rec3 0 +setvar wall2_a097_rec4 0; setvar wall2_a097_rec5 0; setvar wall2_a097_rec6 0; setvar wall2_a097_rec7 0 +mark "records all zero"; sleep 20 +# phase 3: progress=100 +setvar wall2_a097_rec1 100 +mark "rec1=100"; sleep 20 +# phase 4: progress=1000 +setvar wall2_a097_rec1 1000 +mark "rec1=1000"; sleep 20 +# phase 5: progress=100000 +setvar wall2_a097_rec1 100000 +mark "rec1=100000"; sleep 20 +# phase 6: progress=1000000 + rec0=0 +setvar wall2_a097_rec1 1000000 +setvar wall2_a097_rec0 0 +mark "rec1=1000000 rec0=0"; sleep 20 +# phase 7: all ones +setvar wall2_a097_rec0 3735928559; setvar wall2_a097_rec1 3735928559; setvar wall2_a097_rec2 3735928559; setvar wall2_a097_rec3 3735928559 +mark "rec0-3=0xdeadbeef"; sleep 20 +# phase 8: back to echo +setvar wall2_a097_rec0 4294967295; setvar wall2_a097_rec1 4294967295; setvar wall2_a097_rec2 4294967295; setvar wall2_a097_rec3 4294967295 +mark "baseline echo again" +wait $GPID +echo SWEEP-DONE diff --git a/tools/guest-diag/wall2/scan_r673.py b/tools/guest-diag/wall2/scan_r673.py new file mode 100644 index 0000000..71a0128 --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# Analyze the live r673 guest RAM: locate every reference to the validated ring +# PA (0x12a783000) including flag-bit variants, and characterize the record +# arrays around each hit. Also dump the two pages named in the handoff. +import numpy as np, struct, sys, time + +RAM="/dev/shm/vgpu-guest-ram" +RING=0x12a783000 +MASK=~0xfff & ((1<<64)-1) + +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) + +img=open(RAM,"rb") +import os +fsz=os.path.getsize(RAM) +print("ram size 0x%x" % fsz) + +t0=time.time() +hits=[] +CH=64<<20 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero((a & np.uint64(MASK))==np.uint64(RING))[0] + for i in idx: + off=pos+int(i)*8 + val=int(a[i]) + hits.append((off,val)) +print("hits=%d scan=%.1fs" % (len(hits), time.time()-t0)) +for off,val in hits: + print(" off=0x%x gpa=0x%x val=0x%x (low=0x%x)" % (off,off2gpa(off),val,val&0xfff)) + +# characterize each hit page +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +def plausible_pa(v): return 0x100000000 <= (v & MASK) < 0x180000000 +def ptrish(v): + return (0xffff800000000000 <= v < 0xffffffffffffffff) + +for off,val in hits: + gpa=off2gpa(off) + page=gpa & MASK + rec_off=gpa-page + d=rd(page,0x1000) + print("=== hit gpa=0x%x page=0x%x rec_off=0x%x ===" % (gpa,page,rec_off)) + for stride in (0x20,0x40,0x60,0x30,0x50,0x28,0x48): + # record start such that the hit qword falls inside the record + for rel in range(0,stride,8): + start=rec_off-rel + if start<0: continue + # walk records; count plausible PA fields anywhere in each record + n_pa=0; n_ptr=0; n_1000=0 + for o in range(start,0x1000-stride,stride): + rec=d[o:o+stride] + qs=struct.unpack_from("<%dQ"%(stride//8),rec,0) + if any(plausible_pa(q) for q in qs): n_pa+=1 + if any(ptrish(q) for q in qs): n_ptr+=1 + if 0x1000 in qs: n_1000+=1 + if n_pa>=8: + print(" stride=0x%x start=0x%x: PA-records=%d ptr-records=%d 0x1000=%d" % (stride,start,n_pa,n_ptr,n_1000)) + # dump the first 0x100 bytes of the page and around the hit + print(" page head: %s" % d[:0x60].hex()) + s=max(0,rec_off-0x40); e=min(0x1000,rec_off+0x80) + print(" around hit [0x%x..0x%x]: %s" % (s,e,d[s:e].hex())) + +# dump the two named pages +for pa in (0x101499000,0x127cf2000): + d=rd(pa,0x1000) + print("=== page 0x%x ===" % pa) + if d is None: print(" unreadable"); continue + for o in range(0,0x120,0x20): + qs=struct.unpack_from("<4Q",d,o) + print(" +0x%03x: %016x %016x %016x %016x" % ((o,)+qs)) diff --git a/tools/guest-diag/wall2/scan_r673b.py b/tools/guest-diag/wall2/scan_r673b.py new file mode 100644 index 0000000..11613b1 --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673b.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# r673: dump the full record page 0x101499000 and search RAM for the ring/push +# VA structures to identify which record is the 16-page push (VA 0x4500000). +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +MASK=~0xfff & ((1<<64)-1) +img=open(RAM,"rb") +fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +print("== full record page 0x101499000 ==") +d=rd(0x101499000,0x1000) +nrec=0 +for o in range(0,0x1000,32): + p,f,pa,sz=struct.unpack_from("<QQQQ",d,o) + if p==0 and f==0 and pa==0 and sz==0: continue + nrec+=1 + print(" rec[%3d] vmap=%016x flags=%016x pa=%012x size=0x%x" % (o//32,p,f,pa,sz)) +print("nonempty records:",nrec) + +print() +print("== search RAM for qword 0x4520000 (ring VA) ==") +t0=time.time(); CH=64<<20 +for pat_name,pat in (("0x4520000",0x4520000),("0x4500000",0x4500000)): + print("-- pattern %s --" % pat_name) + hits=[] + for pos in range(0,fsz,CH): + img.seek(pos); dd=img.read(min(CH,fsz-pos)) + n=len(dd)//8 + a=np.frombuffer(dd[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(pat))[0] + for i in idx: + hits.append(pos+int(i)*8) + print(" hits=%d (%.1fs)" % (len(hits),time.time()-t0)) + for off in hits[:40]: + gpa=off2gpa(off); page=gpa&MASK + # show 32 bytes before/after + dd=rd(page,0x1000) + if not dd: continue + ro=gpa-page + s=max(0,ro-32); e=min(0x1000,ro+64) + print(" gpa=0x%x page=0x%x off=0x%x ctx=%s" % (gpa,page,ro,dd[s:e].hex())) + if len(hits)>40: print(" ... %d more" % (len(hits)-40)) diff --git a/tools/guest-diag/wall2/scan_r673c.py b/tools/guest-diag/wall2/scan_r673c.py new file mode 100644 index 0000000..db69b5d --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673c.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# r673: find the push page. Patterns: ring entry raw qwords, the sem VA, and +# method-stream signatures. Dump contexts. +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +pats=[ + ("entry0raw", 0x0005980004500000), + ("entry1raw", 0x0000040004500598), + ("entry2raw", 0x0003d0000450059c), + ("sem_va", 0x4500598), + ("sem_va2", 0x450059c), + ("m20038090", 0x20038090), + ("push_va", 0x4500000), +] +CH=64<<20 +for name,pat in pats: + hits=[] + t0=time.time() + for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(pat))[0] + for i in idx: hits.append(pos+int(i)*8) + print("== %s pat=0x%x hits=%d (%.1fs)" % (name,pat,len(hits),time.time()-t0)) + for off in hits[:12]: + gpa=off2gpa(off); page=gpa&~0xfff + d=rd(page,0x1000) + ro=gpa-page + s=max(0,ro-24); e=min(0x1000,ro+40) + print(" gpa=0x%x page=0x%x off=0x%x ctx=%s" % (gpa,page,ro,d[s:e].hex() if d else "?")) + if len(hits)>12: print(" ... %d more" % (len(hits)-12)) diff --git a/tools/guest-diag/wall2/scan_r673d.py b/tools/guest-diag/wall2/scan_r673d.py new file mode 100644 index 0000000..1fa89ca --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673d.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# r673 deep-dive: all ring-PA refs with low bits, ring page dump, RM record +# pages, and a RAM-wide scan for 0xa3bf20 VA-table records mentioning 0x4500000. +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +RING=0x12a783000; TAG=0xa3bf20 +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +MASK=~0xfff & ((1<<64)-1) +print("== all qwords (v & ~0xfff)==ring_pa ==") +CH=64<<20 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero((a & np.uint64(MASK))==np.uint64(RING))[0] + for i in idx: + off=pos+int(i)*8 + gpa=off2gpa(off); page=gpa&~0xfff + dd=rd(page,0x1000); ro=gpa-page + s=max(0,ro-32); e=min(0x1000,ro+48) + print(" gpa=0x%x low=0x%x ctx=%s" % (gpa,int(a[i])&0xfff,dd[s:e].hex() if dd else "?")) + +print() +print("== ring page 0x%x first 0x120 ==" % RING) +d=rd(RING,0x120) +for o in range(0,0x120,0x20): + print(" +0x%03x: %s" % (o,d[o:o+0x20].hex())) + +for pa in (0x1013a6000,0x104acb000,0x114760000,0x1202cd000,0x13e2c1000,0x13e62e000): + print() + print("== page 0x%x first 0x140 ==" % pa) + d=rd(pa,0x140) + for o in range(0,0x140,0x20): + q=struct.unpack_from("<4Q",d,o) + print(" +0x%03x: %016x %016x %016x %016x" % ((o,)+q)) + +print() +print("== RAM-wide: 32-byte records {va, cpu, 0xa3bf20, pages} with va==0x4500000 ==") +t0=time.time() +found=0 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(TAG))[0] + for i in idx: + off=pos+int(i)*8 + gpa=off2gpa(off); page=gpa&~0xfff + dd=rd(page,0x1000) + ro=gpa-page + if ro>=16 and ro+8<=0x1000: + va,cpu=struct.unpack_from("<QQ",dd,ro-16) + pgs=struct.unpack_from("<Q",dd,ro+8)[0] + if va==0x4500000 or (0x4400000<=va<=0x4600000): + print(" rec gpa=0x%x va=0x%x cpu=0x%x pages=0x%x" % (gpa,va,cpu,pgs)) + found+=1 +print("total", found, "(%.1fs)" % (time.time()-t0)) diff --git a/tools/guest-diag/wall2/session73-notes.md b/tools/guest-diag/wall2/session73-notes.md new file mode 100644 index 0000000..d2079d3 --- /dev/null +++ b/tools/guest-diag/wall2/session73-notes.md @@ -0,0 +1,51 @@ +# SESSION 73 notes (wall #2 live validation, r674 + r675) + +## r674 cycle results (8 client runs, all VKFAIL -4) +- Client vk_frame reproducibly fails at vkCreateDevice (-4) as before. +- wall2_resolver2 (content-first CE group 0x20038090 + tuple arrays): no candidates found + during the stall; tuple arrays contain the ring PA but not the push. +- livecap/livecap2/livecap3 (0x20038090 shape / 0x20050017 host-release shape): NO data + hits in guest RAM during the stall; only driver CODE pages match. +- Conclusion: the GR channel-init push content is NOT findable in /dev/shm/vgpu-guest-ram + by these patterns, even live. (The ring page content IS there and matches the service's + decode.) + +## Structural findings +- The push is RM-built (nvidia.ko), NOT UVM: kprobe on + uvm_hal_turing_host_semaphore_release never fired during the client run. +- nvidia.ko functions are `notrace` -> kprobe_events refuses them + ("Could not probe notrace function ... [nvidia]"). UVM module probes DO work. + QEMU gdbstub hardware breakpoints work fine (hbreak *addr). +- Service rescue matcher is CE-shaped only (hdr 0x20038090, hi==1, lo&0xfff==slot, + w5/v5 shape) -> can never match a GR push. Entries 0/1/2 are consumed as "stale" + after ~150 failed rescues each; gr_wedge=1. +- channelFillGpFifo(channel, idx, len) disasm: rdi=channel; +0x178=gpu; + +0xe8=cached ring CPU VA; +0xb8=push GPU VA base; +0x48=stride; + +0x110=class (0xc5b5 seen for a CE channel; GR = 0xc46f). + Entry encoding: [0]=va&~3 (u32), [1]=(va>>32 &0xff)|((len<<8)&0x7ffffc00). +- r674 ring PAs per run: 0x12a783000(?) no: run1 0x1050b6000, run2 0x10297d000, + later 0x102317000 (ring line format: "h=0x... gpfifo=0x44a0000 ent=32768 userd=0xffe00 as=2 put=3 get=0 ring rc=0 pa=0x... ap=1"). +- Push VA = 0x4480000, ring VA = 0x44a0000 (delta 0x20000); entry0 len=1432, + entry1 {va=0x4480598,len=4}, entry2 {va=0x448059c,len=976}. +- The pushbuffer (UVM) = 0x20000 (32 pages) aligned; UVM_PUSHBUFFER_SIZE=0x20000 + (UVM_MAX_PUSH_SIZE=0x400). UVM_RM_MEM_TYPE_SYS with CPU map. +- RM memdesc records {size=0x20000, flags=0x03008000, first_PA, ptr} exist in RAM + (11 found post-run) but their PAs did not contain the release group. +- RM window record {0x4480000, 0x20000} + head/tail list nodes exist in RAM. +- The r673 VA-table record {0x10, 0x4500000, 0x786fd9598ab0, 0xa3bf20} has a + "cpu" field that may be a user VA of the client (0x64..-0x78.. range). + +## Ops +- r674 wedged after the gdb-interrupted vk31 run (SSH + guest agent dead, vCPUs idle). + Fresh cycle r675 started. gdb674d still attached to the dead QEMU (never kill gdbs). +- /dev/shm 53%. + +## r675 plan +1. gdb675.gdb: hbreak channelFillGpFifo if class(+0x110)==0xc46f; on hit dump the + channel struct pointer fields (see gdb675.gdb) -> identify the push's CPU mapping + (direct-map or vmalloc) by content. +2. Compute PA (direct-map: VA - 0xffff8916c0000000 style base) and read the push + content from the RAM file; find the release group. +3. Inject 0x4480000 -> PA via sema_map_add BEFORE/DURING the stall; watch the service + apply the release and vkCreateDevice pass. +4. Durability: integrate into pin_gr_stream.py demand path or a service hook. diff --git a/tools/guest-diag/wall2/slotfill.sh b/tools/guest-diag/wall2/slotfill.sh new file mode 100644 index 0000000..a3416a9 --- /dev/null +++ b/tools/guest-diag/wall2/slotfill.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Watch the vk_frame client and fill the completion-slot u16 (base+0x2e) in +# every small /dev/nvidiactl rw-s mapping while it waits. +cd /tmp +rm -f /tmp/slotfill.log +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk76.out 2>&1 &) +PID="" +for i in $(seq 1 100); do PID=$(pgrep -x vk_frame | head -1); [ -n "$PID" ] && break; sleep 0.1; done +echo "pid=$PID" >> /tmp/slotfill.log +END=$((SECONDS+60)) +while [ $SECONDS -lt $END ]; do + [ -d /proc/$PID ] || break + for m in $(grep 'nvidiactl' /proc/$PID/maps | awk '$2 ~ /rw-s/ {print $1}'); do + lo=${m%-*}; hi=${m#*-} + size=$((0x$hi - 0x$lo)) + [ $size -gt 65536 ] && continue + off=$((0x$lo + 0x2e)) + v=$(dd if=/proc/$PID/mem bs=1 skip=$off count=2 2>/dev/null | xxd -p) + if [ "$v" = "ffff" ]; then + printf '\x00\x00' | dd of=/proc/$PID/mem bs=1 seek=$off count=2 conv=notrunc 2>/dev/null + echo "$(date +%s.%N) FILL $m+0x2e was ffff" >> /tmp/slotfill.log + fi + done + sleep 0.2 +done +echo "done" >> /tmp/slotfill.log +echo "=== client ===" +cat /tmp/vk76.out +echo "=== fills ===" +grep -c FILL /tmp/slotfill.log diff --git a/tools/guest-diag/wall2/slotfill2.sh b/tools/guest-diag/wall2/slotfill2.sh new file mode 100644 index 0000000..33ad702 --- /dev/null +++ b/tools/guest-diag/wall2/slotfill2.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cd /tmp +PID="" +for i in $(seq 1 200); do PID=$(pgrep -x vk_frame | head -1); [ -n "$PID" ] && break; sleep 0.05; done +END=$((SECONDS+45)) +while [ $SECONDS -lt $END ]; do + [ -d /proc/$PID ] || break + for m in $(grep 'nvidiactl' /proc/$PID/maps 2>/dev/null | awk '$2 ~ /rw-s/ {print $1}'); do + lo=${m%-*}; hi=${m#*-} + size=$((0x$hi - 0x$lo)) + [ $size -gt 65536 ] && continue + off=$((0x$lo + 0x2e)) + v=$(dd if=/proc/$PID/mem bs=1 skip=$off count=2 2>/dev/null | xxd -p) + if [ "$v" = "ffff" ]; then + printf '\x00\x00' | dd of=/proc/$PID/mem bs=1 seek=$off count=2 conv=notrunc 2>/dev/null + fi + done + sleep 0.1 +done diff --git a/tools/guest-diag/wall2/slots_probe.sh b/tools/guest-diag/wall2/slots_probe.sh new file mode 100644 index 0000000..71e6fb7 --- /dev/null +++ b/tools/guest-diag/wall2/slots_probe.sh @@ -0,0 +1,11 @@ +#!/bin/bash +PID=$1 +sudo grep -E 'rw-s' /proc/$PID/maps | grep nvidiactl | while read -r line; do + RNG=$(echo "$line" | awk '{print $1}') + LO=${RNG%-*}; HI=${RNG#*-} + SZ=$((0x$HI-0x$LO)) + [ $SZ -gt 262144 ] && continue + V2E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0x2e)) count=2 2>/dev/null | xxd -p) + V0E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0xe)) count=2 2>/dev/null | xxd -p) + echo "$RNG sz=$SZ w2e=$V2E w0e=$V0E" +done diff --git a/tools/guest-diag/wall2/stracevk.sh b/tools/guest-diag/wall2/stracevk.sh new file mode 100644 index 0000000..b2752c4 --- /dev/null +++ b/tools/guest-diag/wall2/stracevk.sh @@ -0,0 +1,7 @@ +#!/bin/sh +cd /tmp +sudo timeout 90 strace -f -T -tt -e trace=ioctl -o /tmp/vk_strace.out /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk58.out 2>&1 +echo "=== client out ===" +cat /tmp/vk58.out +echo "=== ioctl tail ===" +tail -30 /tmp/vk_strace.out diff --git a/tools/guest-diag/wall2/t75g.py b/tools/guest-diag/wall2/t75g.py new file mode 100644 index 0000000..2a0881a --- /dev/null +++ b/tools/guest-diag/wall2/t75g.py @@ -0,0 +1,170 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid + +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found") + gdb.execute("detach") + sys.exit(1) +print("EGLBASE 0x%x PID %d" % (base, pid)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"out": 0, "E": 0, "S": 0, "armed150": False, "armedS": False, + "bt_done": False, "post_done": False} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def a97(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + r14 = int(gdb.parse_and_eval("$r14")) + out = u64(rbp+0x10) + state["out"] = out + q = rd(out, 0x28) + rec_ptr = int.from_bytes(q[8:16], 'little') + print("A97 out=0x%x count=%d f10=%08x f20=%08x f24=%08x recp=0x%x" % + (out, int.from_bytes(q[0:4], 'little'), int.from_bytes(q[0x10:0x14], 'little'), + int.from_bytes(q[0x20:0x24], 'little'), int.from_bytes(q[0x24:0x28], 'little'), rec_ptr)) + if rec_ptr and rec_ptr > 0x1000: + try: + rec = rd(rec_ptr, 16) + print("A97 rec=%08x %08x %08x %08x" % + tuple(int.from_bytes(rec[i*4:i*4+4], 'little') for i in range(4))) + except Exception as e: + print("A97 rec-unreadable", e) + return False + +def post(bp): + if state["post_done"]: + return False + state["post_done"] = True + rsp = int(gdb.parse_and_eval("$rsp")) + q = rd(rsp, 0x40) + print("POST rsp=0x%x out=0x%x" % (rsp, state["out"])) + for i in range(0, 0x40, 8): + v = int.from_bytes(q[i:i+8], 'little') + print("POST rsp+%02x = 0x%016x" % (i, v)) + if state["out"]: + oq = rd(state["out"], 0x40) + for i in range(0, 0x40, 8): + v = int.from_bytes(oq[i:i+8], 'little') + print("POST out+%02x = 0x%016x" % (i, v)) + return False + +def t150(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + if not state.get("vt_done"): + state["vt_done"] = True + rdi = u64(rbp+0x148) + vt = u64(rdi) if rdi else 0 + m118 = u64(vt+0x118) if vt else 0 + m170 = u64(vt+0x170) if vt else 0 + print("VT E148=0x%x vt=0x%x m118=0x%x m170=0x%x" % (rdi, vt, m118, m170)) + if not state.get("eng_done"): + state["eng_done"] = True + n = u32(rbp+8) + ids = [u32(rbp+0x54+i*4) for i in range(min(n,8))] + print("ENG n=%d ids=%s" % (n, ",".join("0x%x" % x for x in ids))) + + # cfg area pointer is rip-relative 0x2019320 at runtime = base + (0x2019320 - 0) + # eglcore base == min load addr; file offset == vaddr for the .text; data seg present? + try: + vals = rd(base + 0x2019320 + 0xf80, 0xe0) + for i in range(0, 0xe0, 0x10): + v4 = [int.from_bytes(vals[i+j:i+j+4],'little') for j in (0,4,8,12)] + print("CFG +0x%03x = %s" % (0xf80+i, ["0x%x"%x for x in v4])) + except Exception as e: + print("CFG dump err", e) + v = u32(rbp+0x150) + b38 = rd(rbp+0x1b038, 1)[0]; b39 = rd(rbp+0x1b039, 1)[0] + p = u64(rbp+0x1b030); s8 = u64(p+8) if p else 0 + print("T150 E=0x%x v=0x%08x b038=%d b039=%d p=0x%x p8=0x%x" % + (rbp, v, b38, b39, p, s8)) + if not state["bt_done"]: + state["bt_done"] = True + print(gdb.execute("bt 8", to_string=True)) + if not state["armed150"]: + state["armed150"] = True + state["E"] = rbp + try: + gdb.execute("frame 1") + S = int(gdb.parse_and_eval("$rbp")) + gdb.execute("frame 0") + print("S=0x%x S120=%08x S128=%08x" % (S, u32(S+0x120), u32(S+0x128))) + state["S"] = S + except Exception as e: + print("S capture err", e) + S = 0 + W("*(unsigned int *)0x%x" % (rbp+0x150), rbp, 0x150, 4, "W150") + W("*(unsigned char *)0x%x" % (rbp+0x1b038), rbp, 0x1b038, 1, "W038") + W("*(unsigned char *)0x%x" % (rbp+0x1b039), rbp, 0x1b039, 1, "W039") + if S: + W("*(unsigned int *)0x%x" % (S+0x120), S, 0x120, 4, "W120") + print("WATCHES armed E=0x%x S=0x%x" % (rbp, S)) + return False + +def sl(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + print("SLOOP S=0x%x S120=%08x" % (rbp, u32(rbp+0x120))) + if not state["armedS"]: + state["armedS"] = True + W("*(unsigned int *)0x%x" % (rbp+0x120), rbp, 0x120, 4, "W120") + print("W120 armed on S+0x120") + return False + +class W(gdb.Breakpoint): + def __init__(self, spec, E, off, size, tag): + super().__init__(spec, type=gdb.BP_WATCHPOINT) + self.silent = True; self.E = E; self.off = off; self.size = size + self.tag = tag; self.hits = 0 + def stop(self): + self.hits += 1 + cur = int.from_bytes(rd(self.E+self.off, self.size), 'little') + pc = int(gdb.parse_and_eval("$pc")) + print("%s v=0x%x pc=0x%x (rel 0x%x)" % (self.tag, cur, pc, pc-base)) + if self.hits <= 8: + print(gdb.execute("bt 10", to_string=True)) + return self.hits >= 400 + +B("*0x%x" % (base+0xecd43d), a97, 220) +B("*0x%x" % (base+0xd5e4d3), post, 100000) +B("*0x%x" % (base+0xd5e449), t150, 220) +B("*0x%x" % (base+0xd6ef76), sl, 240) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +print("TRACE-END") +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/trace_push674.py b/tools/guest-diag/wall2/trace_push674.py new file mode 100644 index 0000000..ced4168 --- /dev/null +++ b/tools/guest-diag/wall2/trace_push674.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# r674: find every reference to the push VA 0x4480000 and follow RM structures +# (direct-map pointers: PA = ptr - page_offset_base) toward a page list. +import numpy as np, struct, os, time, sys + +RAM="/dev/shm/vgpu-guest-ram" +VA=0x4480000 +BASE=0xffff8916c0000000 # r674 page_offset_base (from dmesg rebuild #15) +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa<0x80000000: return pa + if pa<0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off<0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) +def ptr2pa(v): + if BASE<=v<BASE+0x180000000: + return v-BASE + return None + +CH=64<<20 +hits=[] +t0=time.time() +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(VA))[0] + for i in idx: hits.append(off2gpa(pos+int(i)*8)) +print("VA hits:", len(hits), "(%.1fs)" % (time.time()-t0)) + +def tag(v): + bs=struct.pack("<Q",v) + return bs.decode() if all(32<=c<127 for c in bs) else "" + +for gpa in hits: + page=gpa&~0xfff + d=rd(page,0x1000) + if not d: continue + ro=gpa-page + print("== hit gpa=0x%x page=0x%x off=0x%x ==" % (gpa,page,ro)) + # print 0x60 before, 0x80 after + s=max(0,ro-0x60); e=min(0x1000,ro+0x80) + for o in range(s,e,0x10): + a,b=struct.unpack_from("<QQ",d,o) + print(" +0x%03x: %016x %016x %s %s" % (o,a,b,tag(a),tag(b))) + # follow direct-map pointers in this page + for o in range(0,0x1000-8,8): + v=struct.unpack_from("<Q",d,o)[0] + pa=ptr2pa(v) + if pa is None: continue + dd=rd(pa&~0xfff,0x1000) + if not dd: continue + # search that page for PA runs (>=8 page-aligned RAM PAs ascending/descending) + runs=[]; cur=[] + for oo in range(0,0xff8,8): + vv=struct.unpack_from("<Q",dd,oo)[0] + if 0x100000000<=vv<0x180000000 and (vv&0xfff)==0: + cur.append((oo,vv)) + else: + if len(cur)>=8: runs.append(cur) + cur=[] + if len(cur)>=8: runs.append(cur) + if runs: + print(" ptr +0x%03x -> pa=0x%x has %d PA-run(s):" % (o,pa&~0xfff,len(runs))) + for r in runs[:4]: + print(" run at +0x%x: %d pages first=0x%x last=0x%x" % (r[0][0],len(r),r[0][1],r[-1][1])) diff --git a/tools/guest-diag/wall2/vmap_watch2.py b/tools/guest-diag/wall2/vmap_watch2.py new file mode 100644 index 0000000..194b0d6 --- /dev/null +++ b/tools/guest-diag/wall2/vmap_watch2.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# Guest-side live watcher v2: scan ALL vmallocinfo areas via /proc/kcore for +# (a) GR ring entry patterns, (b) pages starting with a small INC_METHOD header. +import struct, time +ENTRIES = [bytes.fromhex("0000480400980500"), bytes.fromhex("8000040004480598"), bytes.fromhex("0003d0000448059c")] +class Kcore: + def __init__(self): + self.fd = open("/proc/kcore", "rb") + hdr = self.fd.read(64) + e_phoff = struct.unpack_from("<Q", hdr, 32)[0] + e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54) + self.segs = [] + for i in range(e_phnum): + self.fd.seek(e_phoff + i * e_phentsize) + ph = self.fd.read(56) + p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from("<IIQQQQ", ph, 0) + if p_type == 1: + self.segs.append((p_va, p_va + p_sz, p_off)) + def read(self, va, n): + for lo, hi, off in self.segs: + if lo <= va and va + n <= hi: + self.fd.seek(off + (va - lo)) + return self.fd.read(n) + return None +kc = Kcore() +def areas(): + out = [] + for line in open("/proc/vmallocinfo"): + p = line.split() + if len(p) < 2 or "-" not in p[0]: continue + try: + lo, hi = p[0].split("-") + lo, hi = int(lo,16), int(hi,16) + except Exception: + continue + out.append((lo, hi, line.strip()[-60:])) + return out +def hdr_ok(page): + v = struct.unpack_from("<I", page, 0)[0] + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return op == 1 and 1 <= cnt <= 8 +print("vmap_watch2 start", flush=True) +found = set() +t0 = time.time() +it = 0 +while time.time() - t0 < 300: + it += 1 + nareas = 0 + for lo, hi, tag in areas(): + size = hi - lo + if size > 8 << 20: continue + nareas += 1 + d = kc.read(lo, size) + if not d: continue + for pat in ENTRIES: + i = d.find(pat) + if i >= 0: + key = (lo, i & ~0xfff, pat.hex()) + if key not in found: + found.add(key) + print("%.2f RING %s VA 0x%x (+0x%x) %s" % (time.time()-t0, pat.hex(), lo+(i&~0xfff), i&~0xfff, tag), flush=True) + for off in range(0, size - 0x1000, 0x1000): + page = d[off:off+0x1000] + if hdr_ok(page): + key = (lo, off, "h") + if key not in found: + found.add(key) + print("%.2f HDR VA 0x%x (+0x%x) %s head=%s" % (time.time()-t0, lo+off, off, tag, page[:0x30].hex()), flush=True) + if it % 10 == 0: + print("%.1f iter %d areas %d" % (time.time()-t0, it, nareas), flush=True) + time.sleep(0.2) +print("vmap_watch2 done", flush=True) diff --git a/tools/guest-diag/wall2/wall2_livecap.py b/tools/guest-diag/wall2/wall2_livecap.py new file mode 100644 index 0000000..af68c03 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# wall2_livecap.py - live capture of the GR channel-init push page. +# +# The release group in the push is {0x20038090, sem_va_hi, sem_va_lo, payload, +# continuation}: sem_va = (hi<<32)|lo, and for the channel-init entry the sem +# sits at push_va + entry_len (entry0: len=1432=0x598, sem at +0x598), with the +# group itself at push_va + entry_len - 24 = +0x580. +# +# This watcher scans guest RAM for 0x20038090 groups during the client's stall, +# classifies them, and injects push_va -> page_pa via sema_map_add when the +# group's sem VA and offset identify the push page. Prints full evidence. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap.log" +MASK = ~0xfff & ((1 << 64) - 1) + +ENTRY_LEN = int(os.environ.get("W2_LEN", "1432")) +WATCH_S = int(os.environ.get("W2_WATCH", "120")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def cont_ok(v): + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return (op in (1, 3, 5) and cnt != 0) or op in (4, 7) + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + g = d[i:i+24] + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, hi, lo, pay, w5, v5, g)) + pos += CH + return out + +def main(): + log("=== livecap start len=%d watch=%ds ===" % (ENTRY_LEN, WATCH_S)) + injected = {} + deadline = time.time() + WATCH_S + while time.time() < deadline: + hits = scan() + cands = [] + for (page, off, hi, lo, pay, w5, v5, g) in hits: + sem = (hi << 32) | lo + # classify + pool = (hi == 1 and lo < 0x10000 and 0 < pay < (1 << 24) + and (w5 >> 29) == 1 and ((w5 >> 13) & 7) == 4 + and ((w5 & 0x1fff) << 2) == 0x300 and (v5 & 3) == 0) + if pool: + continue # CE pool release family, not the push + if not (0x4000000 <= sem < 0x100000000): + continue + if not (0 < pay < (1 << 24)): + continue + if not cont_ok(w5): + continue + push_va = sem - ENTRY_LEN + cands.append((page, off, sem, pay, push_va, w5, v5, g)) + if cands: + # prefer the canonical group offset 0x580 / sem-consistent push VA + cands.sort(key=lambda c: (c[1] != (ENTRY_LEN - 24), -c[3])) + page, off, sem, pay, push_va, w5, v5, g = cands[0] + d = rd(page, 0x100) + log("livecap: %d cand(s); page=0x%x off=0x%x sem=0x%x pay=%u push_va=0x%x w5=0x%08x v5=0x%08x" + % (len(cands), page, off, sem, pay, push_va, w5, v5)) + log(" page head: %s" % d[:0x40].hex()) + if (push_va & 0xfff) == 0 and push_va not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (push_va, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x (off=0x%x)" % (push_va, page, off)) + injected[push_va] = page + except OSError as e: + log(" inject FAIL", e) + time.sleep(1) + log("=== livecap done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap2.py b/tools/guest-diag/wall2/wall2_livecap2.py new file mode 100644 index 0000000..a33b25b --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap2.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# wall2_livecap2.py - capture the GR channel-init push page by its release +# group {0x20038090, hi, lo, payload, continuation} at page offset 0x598. +# The push VA comes from the service's own entry[0] decode in dmesg. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap2.log" +MASK = ~0xfff & ((1 << 64) - 1) +WATCH_S = int(os.environ.get("W2_WATCH", "180")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def prefix_ok(d): + off = 0 + while off + 4 <= len(d): + hdr = struct.unpack_from("<I", d, off)[0] + if hdr: break + off += 4 + if off + 4 > len(d): return False + op = (hdr >> 29) & 7 + if op in (4, 7): return True + if op not in (1, 3, 5): return False + return ((hdr >> 16) & 0xfff) != 0 + +def cont_ok(v): + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return (op in (1, 3, 5) and cnt != 0) or op in (4, 7) + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + g = d[i:i+24] + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, hi, lo, pay, w5, v5)) + pos += CH + return out + +def main(): + log("=== livecap2 start watch=%ds ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + seen_pages = {} + while time.time() < deadline: + hits = scan() + for (page, off, hi, lo, pay, w5, v5) in hits: + if not (0 < pay < (1 << 24)): + continue + if not cont_ok(w5): + continue + d = rd(page, 0x1000) + if not d: continue + pok = prefix_ok(d[:0x100]) + key = (page, off) + if key in seen_pages: + continue + seen_pages[key] = 1 + log("hit page=0x%x off=0x%x sem=0x%x pay=%u w5=0x%08x v5=0x%08x prefix_ok=%s" + % (page, off, (hi << 32) | lo, pay, w5, v5, pok)) + log(" head: %s" % d[:0x30].hex()) + log(" @0x590: %s" % d[0x590:0x5c0].hex()) + if pok and off in (0x598, 0x590, 0x580, 0x588): + va = 0x4480000 + if va not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (va, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x" % (va, page)) + injected.add(va) + except OSError as e: + log(" inject FAIL", e) + time.sleep(0.5) + log("=== livecap2 done: %d injections ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap3.py b/tools/guest-diag/wall2/wall2_livecap3.py new file mode 100644 index 0000000..db28bc2 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap3.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# wall2_livecap3.py - capture the UVM GR channel-init push page via the host +# (C46F) semaphore release group emitted by uvm_hal_turing_host_semaphore_release: +# {0x20050017 (INC_METHOD count=5 addr=0x5c), sem_lo, sem_hi, payload, 0, exec} +# The group sits at push_va + entry_len - 24 = push_va + 0x580 (entry0 len 1432), +# with the header at +0x598 (entry1's 4-byte push). The push VA comes from the +# service's own entry[0] decode in dmesg. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap3.log" +MASK = ~0xfff & ((1 << 64) - 1) +WATCH_S = int(os.environ.get("W2_WATCH", "240")) +PUSH_VA_DEFAULT = 0x4480000 + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def push_va_now(): + # newest entry[0] line for the Vulkan client's channel + va = None + for ln in dmesg().splitlines(): + m = re.search(r"entry\[0\] h=0x([0-9a-f]+) eng=0 raw=\S+ va=0x([0-9a-f]+) len=(\d+)", ln) + if m: + va = int(m.group(2), 16) + return va + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<I", 0x20050017) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + lo, hi, pay, w4, w5 = struct.unpack_from("<5I", d, i + 4) + # data-word sanity: payload_hi==0, RELEASE op, sane payload + if w4 != 0: continue + if (w5 & 0x7) != 1: continue + if not (0 < pay < (1 << 24)): continue + sem = ((hi & 0xffffffff) << 32) | ((lo & 0x3fffffff) << 2) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, sem, pay, w5)) + pos += CH + return out + +def main(): + log("=== livecap3 start watch=%ds ===" % WATCH_S) + injected = {} + seen = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + hits = scan() + if hits: + for (page, off, sem, pay, w5) in hits: + key = (page, off) + if key in seen: continue + seen.add(key) + d = rd(page, 0x1000) + head = d[:0x20].hex() if d else "?" + log("hit page=0x%x off=0x%x sem=0x%x pay=%u exec=0x%08x head=%s" + % (page, off, sem, pay, w5, head)) + log(" @0x580: %s" % (d[0x580:0x5b0].hex() if d else "?")) + if off in (0x598, 0x580, 0x590): + pv = push_va_now() or PUSH_VA_DEFAULT + if pv not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (pv, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x" % (pv, page)) + injected[pv] = page + except OSError as e: + log(" inject FAIL", e) + time.sleep(0.5) + log("=== livecap3 done: %d injections ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver2.py b/tools/guest-diag/wall2/wall2_resolver2.py new file mode 100644 index 0000000..785e5e2 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver2.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# wall2_resolver2.py - HOST-SIDE wall#2 resolver v2 (r674+). +# +# Why: the vk_frame client's channel-init push (va=0x4500000 len=1432) is +# kernel-built and kernel-only; the sema service cannot translate it +# ("false translation for push ... rescuing"), the client times out (~11 s) +# per attempt, retries, then VKFAIL -4. Guest RAM is host-readable, so the +# push page can be found and injected via sema_map_add. +# +# v2 recipe (per the r672/r673 structure analysis): +# 1. wait for the channel ring line in dmesg (validated RING_PA + handle) +# 2. scan guest RAM for the push page: +# a) CONTENT: the 24-byte release group the service's own rescue matches +# (hdr=0x20038090, hi=1, payload sane, w5/v5 shape). The channel-init +# entry (va=0x4500000 len=1432) puts that group at page offset 0x580. +# b) TUPLES: pages of 32-byte records {vmap_ptr; flags; PA; size} (both +# observed layouts) matched on (PA & ~0xfff) == RING_PA with strides +# 0x20/0x40/0x60; entries below the ring entry in VA order map +# 0x4500000.. (push VA = ring VA 0x4520000 - 0x20000 = 32 pages). +# 3. inject (va -> pa, len) through /sys/module/vgsp/parameters/sema_map_add +# 4. keep scanning/re-injecting for the watch window (retries allocate new +# pages) and log everything. +# +# Run this FIRST (foreground), then launch the client in the guest. +import subprocess, struct, time, sys, re, os, signal + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver2.log" +RING_VA = 0x4520000 +PUSH_VA = 0x4500000 +DELTA = (RING_VA - PUSH_VA) // 0x1000 # 32 pages +REL_OFF = 0x580 # group offset in the push page +MASK = ~0xfff & ((1 << 64) - 1) + +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "150")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,%d\n" % (va, pa, ln, ap)) + log("INJECT va=0x%x -> pa=0x%x len=0x%x" % (va, pa, ln)) + return True + except OSError as e: + log("inject FAIL", e) + return False + +def prefix_ok(d): + off = 0 + while off + 4 <= len(d): + hdr = struct.unpack_from("<I", d, off)[0] + if hdr: + break + off += 4 + if off + 4 > len(d): + return False + op = (hdr >> 29) & 7 + if op in (4, 7): + return True + if op not in (1, 3, 5): + return False + return ((hdr >> 16) & 0xfff) != 0 + +def rel_shape(g): + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + return (hdr == 0x20038090 and hi == 1 and 0 < pay < (1 << 24) + and (w5 >> 29) == 1 and ((w5 >> 13) & 7) == 4 + and ((w5 & 0x1fff) << 2) == 0x300 and (v5 & 3) == 0), lo & 0xfff, pay + +img = None +def open_ram(): + global img + if img is None: + img = open(RAM, "rb", buffering=0) + return img + +def gpa2off(pa): + if pa < 0x80000000: + return pa + if pa < 0x180000000: + return 0x80000000 + (pa - 0x100000000) + return None + +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) + +def rd(pa, n): + off = gpa2off(pa) + if off is None: + return None + f = open_ram() + try: + f.seek(off) + return f.read(n) + except OSError: + return None + +def find_content_candidates(): + """Pages containing the release group; returns [(page_pa, off, slot, pay)].""" + f = open_ram() + f.seek(0, 2) + fsz = f.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + f.seek(pos) + d = f.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: + break + st = i + 1 + if i + 24 > len(d): + continue + ok, slot, pay = rel_shape(d[i:i + 24]) + if not ok: + continue + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, slot, pay)) + pos += CH + return out + +def find_tuple_arrays(ring_pa): + """Find record pages containing the ring PA; returns list of + (page_gpa, stride, ring_index, entries[list of pa]).""" + f = open_ram() + f.seek(0, 2) + fsz = f.tell() + pat = struct.pack("<Q", ring_pa) + arrays = [] + CH = 64 << 20 + pos = 0 + hits = [] + while pos < fsz: + f.seek(pos) + d = f.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: + break + hits.append(off2gpa(pos + i)) + st = i + 1 + pos += CH + seen = set() + for gpa in hits: + page = gpa & MASK + if page in seen: + continue + seen.add(page) + d = rd(page, 0x1000) + if not d: + continue + rec_off = gpa - page + for stride in (0x20, 0x40, 0x60): + for rel in range(0, min(stride, rec_off + 1), 8): + start = rec_off - rel + ents = [] + for o in range(start, 0x1000 - stride + 1, stride): + # PA field anywhere in the record: try +0x10 first, then any qword + vals = struct.unpack_from("<%dQ" % (stride // 8), d, o) + pa = None + for cand_off in (0x10, 0x8, 0x18, 0x0, 0x20, 0x28): + if cand_off + 8 <= stride: + v = vals[cand_off // 8] + if (v & MASK) == ring_pa: + pa = v & MASK + break + if pa is None: + continue + ents.append((o, pa)) + if len(ents) >= 4: + idx = [e[1] for e in ents].index(ring_pa) + arrays.append((page, stride, start, idx, [e[1] for e in ents])) + break + if arrays and arrays[-1][0] == page: + break + return arrays + +def main(): + log("=== wall2_resolver2 start (wait_ring=%ss watch=%ss) ===" % (WAIT_RING_S, WATCH_S)) + ring_pa = None + ring_h = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + for ln in dmesg().splitlines(): + m = re.search(r"h=(0x[0-9a-f]+) .*ring rc=0 pa=0x([0-9a-f]+)", ln) + if m: + ring_h = m.group(1) + ring_pa = int(m.group(2), 16) + break + if ring_pa: + break + time.sleep(1) + if not ring_pa: + log("no ring line seen in %ss; exiting" % WAIT_RING_S) + return 1 + log("RING %s PA=0x%x" % (ring_h, ring_pa)) + + injected = set() + watch_deadline = time.time() + WATCH_S + last_pass = 0 + while time.time() < watch_deadline: + # ---- content pass + try: + cands = find_content_candidates() + except Exception as e: + log("content scan error:", e) + cands = [] + # prefer the canonical group offset 0x580, then highest payload + cands.sort(key=lambda c: (c[1] != REL_OFF, -c[3])) + if cands: + page, off, slot, pay = cands[0] + d = rd(page, 0x1000) + pok = bool(d) and prefix_ok(d[:0x100]) + log("content: %d candidate(s); best page=0x%x off=0x%x slot=0x%x pay=%u prefix_ok=%s" + % (len(cands), page, off, slot, pay, pok)) + key = ("C", PUSH_VA, page) + if key not in injected and pok: + if inject(PUSH_VA, page, 0x1000): + injected.add(key) + # also cover the following page (entry2 tail) if non-contiguous unknown; skip + else: + # ---- tuple fallback (only every ~10 s to save time) + if time.time() - last_pass > 10: + last_pass = time.time() + try: + arrs = find_tuple_arrays(ring_pa) + except Exception as e: + log("tuple scan error:", e) + arrs = [] + if arrs: + page, stride, start, idx, ents = arrs[0] + log("tuple: array page=0x%x stride=0x%x ring_index=%d entries=%d" + % (page, stride, idx, len(ents))) + for k in range(idx - DELTA, idx): + if k < 0 or k >= len(ents): + continue + va = PUSH_VA + (k - (idx - DELTA)) * 0x1000 + pa = ents[k] + key = ("T", va, pa) + if key in injected: + continue + d = rd(pa, 0x40) + if d and any(d): + inject(va, pa, 0x1000) + injected.add(key) + else: + log("tuple: no array with ring PA found") + # stop early when the service confirms our mapping + if len(injected) and "injected mapping va=0x%x" % PUSH_VA in dmesg(): + log("service confirmed the injected mapping; done") + break + time.sleep(1) + log("=== done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver3.py b/tools/guest-diag/wall2/wall2_resolver3.py new file mode 100644 index 0000000..4b16686 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver3.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# wall2_resolver3.py - HOST-SIDE wall#2 resolver v3: the RM window-memdesc recipe. +# +# Proven structure (r674/r675): the RM keeps per-window records +# {..., VA, SIZE, ...} (head/tail list nodes) +# and per-window memdescs in the same page: +# {SIZE, 0x03008000, FIRST_PA, ptr} +# where FIRST_PA is the PA of the window's FIRST VA page. Verified: the ring +# window {VA=0x44a0000, SIZE=0x40000} -> memdesc PA == the service's validated +# ring pa (0x1158ce000 r675). Therefore the push window +# {VA=0x4480000, SIZE=0x20000} -> memdesc PA == PA of VA 0x4480000 (the push +# page the sema service cannot translate). +# +# Recipe: wait for the ring line -> find the ring window memdesc +# {0x40000, 0x03008000, RING_PA} -> in the same page find the push window +# memdesc {0x20000, 0x03008000, PUSH_PA} -> inject 0x4480000 -> PUSH_PA. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver3.log" +PUSH_VA = 0x4480000 +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "150")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,%d\n" % (va, pa, ln, ap)) + log("INJECT va=0x%x -> pa=0x%x len=0x%x" % (va, pa, ln)) + return True + except OSError as e: + log("inject FAIL", e) + return False + +def find_memdescs(size): + """Find {size, 0x03008000, PA} patterns; returns [(page, off, pa)].""" + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", size, 0x03008000) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + pa = struct.unpack_from("<Q", d, i + 16)[0] + gpa = off2gpa(pos + i) + if 0x100000000 <= pa < 0x180000000: + out.append((gpa & ~0xfff, gpa - (gpa & ~0xfff), pa)) + pos += CH + return out + +def main(): + log("=== resolver3 start ===") + # count pre-existing ring lines; wait for a NEW one (fresh client kick) + def ring_lines(): + out = [] + for ln in dmesg().splitlines(): + m = re.search(r"ring rc=0 pa=0x([0-9a-f]+) ap=", ln) + if m: + out.append(int(m.group(1), 16)) + return out + base = ring_lines() + log("pre-existing ring lines: %d" % len(base)) + ring_pa = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + cur = ring_lines() + if len(cur) > len(base): + ring_pa = cur[-1] + break + time.sleep(1) + if not ring_pa: + log("no NEW ring line; exit"); return 1 + log("NEW RING PA = 0x%x" % ring_pa) + + injected = set() + watch = time.time() + WATCH_S + while time.time() < watch: + # ring window memdesc {0x40000, 0x03008000, ring_pa} + rd_hits = [x for x in find_memdescs(0x40000) if x[2] == ring_pa] + if not rd_hits: + time.sleep(0.5); continue + for (page, off, _pa) in rd_hits: + d = rd(page, 0x1000) + if not d: continue + # find sibling push memdesc in the same page + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if not (0x100000000 <= pa < 0x180000000): + continue + key = (page, pa) + if key in injected: continue + log("memdesc page=0x%x ring_desc@0x%x push_desc@0x%x push_pa=0x%x" + % (page, off, o, pa)) + # content sanity: read the push page + pd = rd(pa, 0x40) + log(" push page head: %s" % (pd.hex() if pd else "?")) + if inject(PUSH_VA, pa, 0x1000): + injected.add(key) + if injected and ("injected mapping va=0x%x" % PUSH_VA) in dmesg(): + log("service confirmed; keep watching") + time.sleep(0.5) + log("=== resolver3 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver4.py b/tools/guest-diag/wall2/wall2_resolver4.py new file mode 100644 index 0000000..140aaa7 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver4.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# wall2_resolver4.py - HOST-SIDE wall#2 resolver v4 (final recipe). +# +# Association proven on r675: the RM keeps, in ONE page, the client's channel +# window records and their memdescs: +# push window record {VA=0x4480000, SIZE=0x20000} +# ring window record {VA=0x44a0000, SIZE=0x40000} +# push memdesc {0x20000, 0x03008000, FIRST_PA, ptr} +# ring memdesc {0x40000, 0x03008000, FIRST_PA, ptr} +# where FIRST_PA is the PA of the window's FIRST VA page. Verified: the ring +# memdesc's PA == the service's validated ring pa. So the push memdesc's PA == +# the PA of VA 0x4480000 (the push page the service cannot translate). +# +# Recipe: wait for a NEW ring line -> find the page containing the ring window +# record {0x44a0000, 0x40000} -> require the sibling push window record +# {0x4480000, 0x20000} and the memdescs in the same page -> cross-check the +# ring memdesc PA == ring_pa -> inject 0x4480000 -> push memdesc PA (sudo). +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver4.log" +PUSH_VA = 0x4480000 +RING_VA = 0x44a0000 +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "180")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x len=0x%x OK" % (va, pa, ln)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + """Find qword pairs {a,b}; returns [(page, off)].""" + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + gpa = off2gpa(pos + i) + out.append((gpa & ~0xfff, gpa - (gpa & ~0xfff))) + pos += CH + return out + +def scan_page_descs(page): + d = rd(page, 0x1000) + if not d: return [] + out = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if fl != 0x03008000: continue + if sz not in (0x20000, 0x40000): continue + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + out.append((sz, o, pa)) + return out + +def main(): + log("=== resolver4 start ===") + def ring_lines(): + return re.findall(r"ring rc=0 pa=0x([0-9a-f]+) ap=", dmesg()) + base = ring_lines() + log("pre-existing ring lines: %d" % len(base)) + ring_pa = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + cur = ring_lines() + if len(cur) > len(base): + ring_pa = int(cur[-1], 16) + break + time.sleep(1) + if not ring_pa: + log("no NEW ring line; exit"); return 1 + log("NEW RING PA = 0x%x" % ring_pa) + + injected = set() + LEARNED = [0x101e39000, 0x102a62000] # RM window-record pages (stable across runs) + def try_page(page): + descs = scan_page_descs(page) + rdesc = [x for x in descs if x[0] == 0x40000] + pdesc = [x for x in descs if x[0] == 0x20000] + if not rdesc or not pdesc: + return False + if rdesc[0][2] != ring_pa: + return False + ok = False + for (_sz, o, pa) in pdesc: + key = (page, o) + if key in injected: continue + pd = rd(pa, 0x40) + log("page=0x%x ring_desc@0x%x(pa=0x%x OK) push_desc@0x%x pa=0x%x head=%s" + % (page, rdesc[0][1], rdesc[0][2], o, pa, + pd.hex() if pd else "?")) + if inject(PUSH_VA, pa, 0x1000): + injected.add(key); ok = True + return ok + log("learned pages: %s" % [hex(p) for p in LEARNED]) + watch = time.time() + WATCH_S + last_full = 0 + while time.time() < watch: + hit = False + for page in LEARNED: + if try_page(page): + hit = True + if not hit and time.time() - last_full > 5: + last_full = time.time() + for (page, _off) in find_qpair(RING_VA, 0x40000): + if try_page(page): + if page not in LEARNED: + LEARNED.append(page) + hit = True + time.sleep(0.25) + log("=== resolver4 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver5.py b/tools/guest-diag/wall2/wall2_resolver5.py new file mode 100644 index 0000000..faec902 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver5.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# wall2_resolver5.py - final: continuous scan, all-candidate injection. +# +# Structure (r675, verified): an RM page holds the client channel's window +# records and memdescs: +# {0x4480000, 0x20000} push window record +# {0x44a0000, 0x40000} ring window record +# {0x20000, 0x03008000, PUSH_PA} push window memdesc +# {0x40000, 0x03008000, RING_PA} ring window memdesc (PA == validated ring pa) +# The resolver scans CONTINUOUSLY (no ring-line gate), finds pages holding the +# pair of window records, and injects 0x4480000 -> every 0x20000 memdesc PA in +# the same page (last write wins in the service; content check arbitrates). +import subprocess, struct, time, sys, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver5.log" +PUSH_VA = 0x4480000 +RING_VA = 0x44a0000 +WATCH_S = int(os.environ.get("W2_WATCH", "600")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x" % (va, pa)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + gpa = off2gpa(pos + i) + out.append(gpa & ~0xfff) + pos += CH + return out + +def main(): + log("=== resolver5 start (watch=%ds) ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + try: + push_pages = set(find_qpair(PUSH_VA, 0x20000)) + ring_pages = set(find_qpair(RING_VA, 0x40000)) + except Exception as e: + log("scan error:", e); time.sleep(2); continue + for page in sorted(push_pages & ring_pages): + d = rd(page, 0x1000) + if not d: continue + # collect 0x20000 descs in this page + cands = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + cands.append((o, pa)) + for (o, pa) in cands: + key = (page, o) + if key in injected: continue + pd = rd(pa, 0x40) + nz = sum(1 for b in pd if b) if pd else -1 + log("page=0x%x desc@0x%x pa=0x%x live_nz=%d head=%s" + % (page, o, pa, nz, pd.hex()[:64] if pd else "?")) + if inject(PUSH_VA, pa): + injected.add(key) + time.sleep(1) + log("=== resolver5 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) |