summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--kernel/gsp_rm_emulation.c93
-rw-r--r--kernel/gsp_sema_service.c56
-rw-r--r--tools/guest-diag/wall2/README-session77.md580
-rw-r--r--tools/guest-diag/wall2/kc_dump_fn.py21
-rwxr-xr-xtools/guest-diag/wall2/nudge2.sh22
-rw-r--r--tools/guest-diag/wall2/t101.py55
-rw-r--r--tools/guest-diag/wall2/t102.py31
-rw-r--r--tools/guest-diag/wall2/t104.py56
-rw-r--r--tools/guest-diag/wall2/t106.py61
-rw-r--r--tools/guest-diag/wall2/t112c-gdbstub.py42
-rw-r--r--tools/guest-diag/wall2/t80r.py32
-rw-r--r--tools/guest-diag/wall2/t83.py25
-rw-r--r--tools/guest-diag/wall2/t89.py17
-rw-r--r--tools/guest-diag/wall2/t93.py57
-rw-r--r--tools/guest-diag/wall2/t94.py54
-rw-r--r--tools/guest-diag/wall2/t95.py19
-rw-r--r--tools/guest-diag/wall2/t96b.py40
-rw-r--r--tools/guest-diag/wall2/t97.py22
-rw-r--r--tools/guest-diag/wall2/t99.py79
-rw-r--r--tools/guest-diag/wall2/unwait.py48
-rw-r--r--tools/guest-diag/wall2/unwait2.py95
-rw-r--r--tools/vk/vk_frame.c29
22 files changed, 1527 insertions, 7 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index b8a3e37..03f861e 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -2665,6 +2665,20 @@ static void gsp_rm_handle_kgr_get_context_buffers_info(
* L2/LTC topology from here. Echo the requested indices and fill in the
* emulated TU104 values.
*/
+/*
+ * m113: FB_GET_INFO_V2 COMPRESSION_SIZE (index 0x01, bytes). The header:
+ * "A return value of 0 indicates the GPU does not support compression" -
+ * but the guest driver still attempts a COMPR-enabled, 64 KB-aligned
+ * comptag allocation (vendor VidHeap alloc, attr COMPR=2) and fails it
+ * NV_ERR_NO_MEMORY (0x51) when the comptag pool has no space, which
+ * libcuda maps to VK_ERROR_OUT_OF_DEVICE_MEMORY (the vkAllocateMemory -2
+ * gate). Report a comptag pool sized like the real Turing parts (~fb/16).
+ * 0 = the historic "no compression" reply.
+ */
+static uint gsp_rm_fb_compr_mb = 128;
+module_param_named(wall2_fb_compr_mb, gsp_rm_fb_compr_mb, uint, 0644);
+MODULE_PARM_DESC(wall2_fb_compr_mb, "FB_GET_INFO_V2 COMPRESSION_SIZE in MiB (default 128, 0 = none)");
+
static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
{
u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu);
@@ -2673,7 +2687,8 @@ static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
case TU104_FB_INFO_IDX_TILE_REGION_COUNT:
return 0;
case TU104_FB_INFO_IDX_COMPRESSION_SIZE:
- return 0;
+ /* m113: non-zero comptag pool (bytes) - see gsp_rm_fb_compr_mb. */
+ return gsp_rm_fb_compr_mb << 20;
case TU104_FB_INFO_IDX_PARTITION_COUNT:
return 1;
case TU104_FB_INFO_IDX_BAR1_SIZE:
@@ -3505,6 +3520,41 @@ MODULE_PARM_DESC(wall2_a097_rec6, "0x2080a097 reply word at params+0x1c (0xfffff
module_param_named(wall2_a097_rec7, gsp_rm_a097_rec7, uint, 0644);
MODULE_PARM_DESC(wall2_a097_rec7, "0x2080a097 reply word at params+0x20 (0xffffffff = echo)");
+/*
+ * m112b: the NV04_VID_HEAP_CONTROL enumeration pair 0x90960106/0x90960107
+ * (legacy NVOS32 VidHeap API, class 0x9096). libnvidia-eglcore (file offset
+ * 0x586d80) walks three heaps: per heap it queries 0x90960106 (12 bytes:
+ * {u32 firstIdx, u32 lastIdx}; the client then enumerates the heap's memory
+ * blocks by index via 0x90960107 (56 bytes) while firstIdx+1 <= idx <=
+ * lastIdx). The 56-byte block entry is read at +0x00/+0x08 (two u64s), +0x28
+ * (a u32 type compared against a table at eglcore 0x101b1c0) and +0x30 (a
+ * flag BYTE: when 0 the heap/block is skipped). The generic zero reply makes
+ * every heap look empty, the client concludes out-of-device-memory and
+ * vkAllocateMemory fails -2 WITHOUT any alloc reaching the emulation
+ * (dmesg-corroborated: only 0x9096* controls + a class 0xc5c0 alloc).
+ * Runtime-tunable so one boot can sweep reply variants (same pattern as the
+ * wall2_a097_* words).
+ */
+static uint gsp_rm_9096_lo; /* 0x90960106 firstIdx */
+static uint gsp_rm_9096_hi = 1; /* 0x90960106 lastIdx */
+module_param_named(wall2_9096_lo, gsp_rm_9096_lo, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_lo, "0x90960106 reply firstIdx (default 0)");
+module_param_named(wall2_9096_hi, gsp_rm_9096_hi, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_hi, "0x90960106 reply lastIdx (default 1)");
+static uint gsp_rm_9096_ent_addr; /* 0x90960107 entry +0x00 (u64) */
+module_param_named(wall2_9096_ent_addr, gsp_rm_9096_ent_addr, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_addr, "0x90960107 entry word at +0x00 (default 0)");
+static uint gsp_rm_9096_ent_size = 0x80000000u; /* entry +0x08 (u64) */
+module_param_named(wall2_9096_ent_size, gsp_rm_9096_ent_size, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_size, "0x90960107 entry word at +0x08 (default 0x80000000)");
+static uint gsp_rm_9096_ent_type = 3; /* entry +0x28 (u32) */
+module_param_named(wall2_9096_ent_type, gsp_rm_9096_ent_type, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_type, "0x90960107 entry word at +0x28 (default 3)");
+static uint gsp_rm_9096_ent_flag = 1; /* entry +0x30 (u8) */
+
+module_param_named(wall2_9096_ent_flag, gsp_rm_9096_ent_flag, uint, 0644);
+MODULE_PARM_DESC(wall2_9096_ent_flag, "0x90960107 entry flag byte at +0x30 (default 1)");
+
/*
* The live session's oracle client/object for the 0x20810107 experiment,
@@ -4348,6 +4398,47 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
break;
}
+ case 0x90960106: { /* NV04_VID_HEAP_CONTROL block-index range (m112b) */
+ u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE;
+ const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE;
+
+ if (req.paramsSize >= 12) {
+ memcpy(rp, pp, req.paramsSize);
+ *(u32 *)(rp + 0) = gsp_rm_9096_lo;
+ *(u32 *)(rp + 4) = gsp_rm_9096_hi;
+ if (gsp_sema_dbg)
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: ctrl 0x90960106 -> first=%u last=%u",
+ gsp_rm_9096_lo, gsp_rm_9096_hi);
+ }
+ out_status = 0;
+ out_params_size = req.paramsSize;
+ break;
+ }
+
+ case 0x90960107: { /* NV04_VID_HEAP_CONTROL block entry by index (m112b) */
+ u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE;
+ const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE;
+
+ if (req.paramsSize >= 56) {
+ memcpy(rp, pp, req.paramsSize);
+ *(u64 *)(rp + 0x00) = (u64)gsp_rm_9096_ent_addr;
+ *(u64 *)(rp + 0x08) = (u64)gsp_rm_9096_ent_size;
+ *(u32 *)(rp + 0x28) = gsp_rm_9096_ent_type;
+ *(u8 *)(rp + 0x30) = (u8)gsp_rm_9096_ent_flag;
+ if (gsp_sema_dbg)
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: ctrl 0x90960107 -> addr=0x%x size=0x%x type=%u flag=%u",
+ gsp_rm_9096_ent_addr,
+ gsp_rm_9096_ent_size,
+ gsp_rm_9096_ent_type,
+ gsp_rm_9096_ent_flag);
+ }
+ out_status = 0;
+ out_params_size = req.paramsSize;
+ break;
+ }
+
default:
/* Unknown control command - return success with zeroed params.
* Do NOT echo the request: pure queries pass uninitialized
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 993e507..b311c0d 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1586,12 +1586,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
* channel; the stashed geometry survives, so the scrub path
* does not depend on the channel-table identity at all.
*/
- if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va &&
+ /*
+ * m112b: the CeUtils scrubber is the ONLY exactly-4096-entry
+ * channel (NUM_COPY_BLOCKS). The old `>= 4096` test also
+ * matched every client channel (this driver version: 32768
+ * entries, not the 1024 the comment below assumed), so the
+ * single scrub slot bound to whichever client channel
+ * registered last (observed r701: the slot held h=0xbeee0102
+ * userd=0x8f400/entries=32768 and the pass authored put=0 for
+ * the wrong channel while the real scrubber's payload stayed
+ * zero - the vkAllocateMemory -2 gate).
+ */
+ if (desc->gpfifo_entries == 4096 && desc->gpfifo_va &&
desc->userd_pa) {
struct svc_scrub *s = svc_scrub_get(desc->vgpu, true);
if (!s)
goto scrub_done; /* table full: max guests */
+ pr_info("sema-svc: scrub stash bind vgpu=%px slot=%d h=0x%x entries=%u\n",
+ (void *)desc->vgpu, (int)(s - svc_scrubs),
+ desc->h_object, desc->gpfifo_entries);
/* Re-seed the counting on a fresh scrubber object or a
* new client session (m176: the counter/submitted state
* is per-session; a recycled scrubber handle after a
@@ -1616,6 +1630,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
desc->h_object, desc->gpfifo_va,
desc->gpfifo_entries, desc->userd_pa,
s->userd_as);
+ } else if (desc->gpfifo_entries >= 4096 &&
+ desc->gpfifo_entries != 32768) {
+ /* m112b: a non-32768 channel with >=4096 entries that
+ * fails the stash condition (missing VA or USERD PA)
+ * is a scrubber candidate the service cannot author
+ * for - log it so a silent no-authoring boot is
+ * diagnosable. */
+ pr_info_ratelimited("sema-svc: scrubber-stash MISS h=0x%x entries=%u va=0x%llx userd_pa=0x%llx\n",
+ desc->h_object, desc->gpfifo_entries,
+ desc->gpfifo_va, desc->userd_pa);
}
scrub_done:
pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
@@ -26992,19 +27016,43 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu)
u32 put = 0;
u64 delta;
- if (!s || !s->userd_ok || !s->pa_ok || !s->entries)
+ if (!s)
+ return;
+ if (!s->userd_ok || !s->pa_ok || !s->entries) {
+ /* m112b: silent-return diagnostic (bounded per boot). */
+ if (s->log_budget < 8) {
+ s->log_budget++;
+ pr_info("sema-svc: scrub pass idle vgpu=%px slot=%d userd_ok=%d pa_ok=%d entries=%u gen=%u\n",
+ (void *)vgpu, (int)(s - svc_scrubs),
+ s->userd_ok, s->pa_ok, s->entries,
+ s->session_gen);
+ }
return;
+ }
if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as,
&put, sizeof(put))) {
/* Retry through the alternate aperture (as registration did). */
if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT,
s->userd_as == SVC_APERTURE_FB ?
SVC_APERTURE_SYS : SVC_APERTURE_FB,
- &put, sizeof(put)))
+ &put, sizeof(put))) {
+ if (s->log_budget < 12) {
+ s->log_budget++;
+ pr_info("sema-svc: scrub pass read-fail userd=0x%llx as=%u\n",
+ s->userd_pa, s->userd_as);
+ }
return;
+ }
}
- if (put >= s->entries)
+ if (put >= s->entries) {
+ /* m112b: junk-translation diagnostic (bounded). */
+ if (s->log_budget < 12) {
+ s->log_budget++;
+ pr_info("sema-svc: scrub pass junk put=%u entries=%u userd=0x%llx\n",
+ put, s->entries, s->userd_pa);
+ }
return; /* junk translation; retry next pass */
+ }
if (!s->seeded) {
s->seeded = true;
diff --git a/tools/guest-diag/wall2/README-session77.md b/tools/guest-diag/wall2/README-session77.md
index 1c59e2d..1ae9a62 100644
--- a/tools/guest-diag/wall2/README-session77.md
+++ b/tools/guest-diag/wall2/README-session77.md
@@ -481,3 +481,583 @@ gr_forward=0 is the correct test configuration; the remaining suspect is
the service-global svc.op_tr_pending path in svc_opdef_release (it records
the release into an opdef slot and returns true silently when the slot
already contains it).
+
+**7. Session 90: the CURRENT-boot wait loop site (the old recipe no longer matches).**
+The client's long spin on this driver/boot sits INSIDE vkCreateInstance (bt:
+libEGL -> eglcore -> 0xB2852D helper), a bounded yield loop at
+eglcore+0xB2852D: `mov (%r11),%r12d / test $1,%r12b / jne <exit> /
+mov 0x4(%r11),%eax / cmp $1,%eax / jne <retry>` - the loop waits for an
+IN-PROCESS word pair at r11 (e.g. 0x7ffc2a1c5080, stack/heap; guest-written
+only, the host RAM file cannot feed it because it is not a GPU buffer).
+Evidence the old site moved: t91's breakpoint at eglcore+0xa85d24 never
+fires this boot (`clears=0`) and a read-watchpoint (t80r.py) on the +0x80
+words of the two 16 KB rw-s nvidiactl mappings does not fire either (the
+16K event-buffer wait of sessions 77/78 is not the live target now; the
+spin stack is +0xB2852D <- +0xA5325A <- +0xA539AD <- +0xA47491 <- +0x9FF515
+<- +0xA20D04 <- libEGL, NOT the 0xA85F1A/A8 path). A widened multi-word
+nudge (nudge2.sh/guest_words2.sh: all rw-s nvidia mappings <=256K, +0x80)
+DID fire on 20+ words but the client still returned -4 - the session-77
+caution stands: multi-word writes trip the error path; single-word only,
+after identifying the true target.
+NEXT: (a) rwatch WRITE on r11+4 to name the writer of the in-process word
+(the component that must run; if nothing ever writes it, disassemble the
+exit branch 0x...ebc9 vs retry 0x...eb93 to see the pass/fail contract);
+(b) the same for the 0xA53xxx caller chain - one of these callers is the
+A8/NvOgl GL-ucode wait of sessions 78/79 (its completion comes from the
+emulated GR engine path); (c) only then re-test the count-clear + single
+word nudge recipes.
+
+**8. Session 91: THE vkCreateInstance WALL BREACHED (the marker-clear recipe).**
+The client's retry loop (eglcore+0xB33D30) waits until NO event-slot entry has
+the pending marker 0xffff: per entry ptr = *(u64*)(obj+0x1428+i*8),
+cnt = *(u32*)(obj+0x22efc), the checked slot = ptr + n*0x10 (n = the call's
+esi), marker = u16@+0xe == 0xffff (or u32@+0 when (esi-0x43)<=4) via the
+check at +0xB1C750 (OR of all pending markers). The guest's event dispatch
+NEVER clears it, so the client burns its 10 s -4 timeout. PROVEN FIX:
+clear {u16@slot+0xe, u16@slot+0xc, u32@slot+0} to 0 at every hit of
++0xB33D30 (t93 single-shot; t94 auto-clearer, 4000 hits) -> the client
+passes vkCreateInstance AND device enumeration FOR THE FIRST TIME
+(`VKINSTANCE ok / VKDEVS 2 / VKPICK`, previously the 10 s -4 fired during
+this phase). t94 cleared 5+ distinct module objects (obj=0x5d67063a84a0,
+0x5d670661ad80, 0x5d670673eed0, 0x5d67067ebb50, ...) over the client's life.
+The client then sat in vkCreateDevice's poll(/dev/nvidia0, nfds=1,
+timeout=10 ms) loop at eglcore+0xA12DDF..+0xA12DE4 WITHOUT any -4 for 5+
+minutes (vs the old ~11 s death) - the longest the client has ever run.
+
+**9. The remaining gap (single): the event-fd signal.** The device-phase wait
+is a poll() on /dev/nvidia0 (fd 27) - nvidia_poll wakes only when the guest
+RM signals the client's event state; the guest consumes our POST_EVENT
+records but never performs the event signaling (session 78's "residual
+delivery gap"). Writing to /proc/PID/fd/<nvidia0> is not a signal. The
+session-84 count write (base+0x2070b7c, read 4, wrote 0, t97) did not flip
+this wait alone. NEXT: fix the host POST_EVENT -> guest dispatch association
+(gsp_rm_emulation.c r579 area: the payload fields hClient/hEvent/notifyIndex/
+data/info/status; compare against the guest's _kgspRpcPostEvent ->
+gpuNotifySubDeviceEvent expectations) so the guest signals the event and
+nvidia_poll returns POLLIN; then the same t94-pattern (or a service-side
+equivalent) durably unblocks both waits per boot.
+
+**10. Session 92: THE AUTOMATED PIPELINE - fresh boot to VKDEVICE with ZERO manual steps.**
+On a fresh r692 boot (cycle params unchanged, gr_forward=0) a single gdb
+script (t99.py) carrying two breakpoint handlers drove the client end-to-end:
+(a) the instance-phase marker cleaner at eglcore+0xB33D30 (t94 pattern: clear
+u16@slot+0xe / u16@slot+0xc / u32@slot to 0 per entry at ptr+n*0x10 for
+entries *(obj+0x1428+i*8), cnt *(obj+0x22efc)) - fired CLEAR#1..#5 across
+five module objects; (b) the device-phase wait-word chaser at
+eglcore+0xDCEBD9 (t79 chain: rbx -> *(rbx+0x70) -> *(sub+0x58)=obj2 ->
+ off=*(u32)(obj2+0x60), holder=*(obj2+0x58), base=*(holder+0x58),
+word=base+off) - found WORD addr=0x734781a86080 val=0x0 and wrote 1.
+Result: `VKINSTANCE ok / VKDEVS 2 / VKPICK / VKDEVICE ok family=0` with no
+manual intervention (r692 client 1725). The client then sat in the
+POST-DEVICE wait (the next phase - allocation/engine-init), which t99 does
+not cover. The live_walk PT-walk injector ran alongside and injected the
+window/ring VAs (0x4d20000, 0x4923000...; note 0x4b19b10 resolved ap=FB -
+the GL init release is vidmem-mapped and skipped by the SYS-only injector).
+
+**11. HARD LESSON (attach discipline, again).** A gdb attached to a client
+that never hits its breakpoints wedges: `continue` never returns, the
+timeout's SIGTERM does not tear it down, and the ptrace slot blocks all
+later attaches (hit 3x this session: 146084, the t91-on-1725 attach, both
+left holding). RULE: only attach with breakpoints that are KNOWN to fire
+for the client's current phase, keep windows <= ~60 s, and prefer
+self-terminating handlers (hit caps). A wedged attach leaves the client
+RUNNING (state R) but unattachable; cycle the VM for a clean environment
+when more attaches are needed.
+
+**12. Continuing the chain (next session).** (1) Identify the post-device
+wait of the VKDEVICE-ok client (bt sample once attachable; likely the
+allocation/UVM phase or the A8/NvOgl engine-init with its count gate at
+lib_base+0x2070b7c=4 -> 0 and/or the 0xdceb80 wait loop - add the t91
+count-clear BP to the t99 bundle if the A8 path is live). (2) Extend t99
+into the "durable harness bundle": marker-clean + word-chase + count-clear
++ (once VKFRAME) the release/alloc phases. (3) The bNotifyList=1
+(gsp_rm_ev_nlist) path stays armed for the event-signal experiment;
+post/EVPOST logging re-budgeted (gsp_rm_ev_log=2000 set).
+
+**13. Session 93: THE ZERO-PTRACE PIPELINE (r693) - VKDEVICE with no gdb-wedge risk.**
+Split the harness by risk: (a) ONE gdb (t101.py) for the INSTANCE-phase
+marker clears only (base+0xB33D30 handler, clears u16@+0xe/+0xc and u32@+0
+of each entry; self-disables after 24 clears; NO other BPs so it cannot
+fire constantly); (b) the WAIT WORDS handled by /tmp/unwait.py - a guest
+python poller with ZERO ptrace that scans the newest vk_frame's rw-s
+nvidia mappings of exactly 0x4000 bytes and writes 1 to any zero u32 at
+mapping+0x80 via /proc/PID/mem (the session-77 geometry, confirmed by t79
+chains: word = the +0x80 of a 16 KB rw-s nvidiactl mapping). Result on
+fresh r693 (mdev 261980af): t101 CLEAR#1..#5 then idle; unwait wrote FIVE
+wait words (0x7876ead57080, 0x7876f1816080, 0x7876ea47f080,
+0x7876ea81a080, 0x7876ead51080); client reached `VKINSTANCE ok / VKDEVS 2
+/ VKPICK / VKDEVICE ok family=0` with NO wedge exposure. live_walk (PT-walk
+injector) ran alongside. The client then sat in the post-device wait that
+the +0x80 poller does not cover (t102.py = the one-shot safe diagnostic:
+attach -> print PC/base/A8-count(base+0x2070b7c)/bt -> detach, NO continue
+and NO breakpoints - use it when the slot is free).
+
+**14. THE RM-LOCK WEDGE ROOT CAUSE + RULE.** A gdb killed (SIGTERM) while
+the tracee sits AT a breakpoint leaves the tracee STOPPED; if that stop is
+inside an RM api critical section (the rwlock held across an RM call!), the
+RM write lock stays held and EVERY other client blocks in
+os_acquire_rwlock_write (state D) - observed directly: old client 1725
+stopped by its wedged t99 gdb -> new client 7967 hung `state=D wchan=
+os_acquire_rwlock_write` on ioctl 0xc020462b; killing the gdb + `kill -CONT`
+resumed 1725 out of the ptrace stop but it resumed INSIDE the same RM wait,
+so the lock stayed held until the VM was cycled. RULES: never let a gdb
+timeout fire while a tracee is at a frequent-firing BP; prefer BPs that
+fire once or self-disable; prefer no-ptrace pollers for anything with a
+repeating pattern; the safe diagnostic is attach -> read -> detach without
+continue.
+
+**15. Session 94: THE INSTANCE CLEARS ARE REQUIRED (zero-gdb disproved); t104 is the safe form.**
+Pure zero-gdb run (fresh r694, client + unwait.py ONLY, no gdb): unwait
+wrote three +0x80 words and the client still ended `VKFAIL vkCreateDevice
+... -4` - WITHOUT the instance-phase marker clears the device gate fails
+even with the words. With t104.py (single BP at +0xB33D30; clears
+u16@+0xe/+0xc/u32@+0; self-exits on cleared>=8 OR chits>=40 OR 25 s since
+the last clear -> returns True so the script tail runs and batch gdb exits
+cleanly, no outer-timeout dependence) + unwait + live_walk: CLEAR#1..#5,
+two words written, client reached `VKINSTANCE ok / VKDEVS 2 / VKPICK /
+VKDEVICE ok family=0` (r694 client 3725). The client then sat in the
+post-device wait again; the t102 diagnostic could not attach because the
+t104 gdb still held the slot (its 25 s-idle exit never fired - no
+post-instance B33D30 hits - and its 150 s timeout likely fired at a stop
+= the known wedge risk). NEXT FORM: make t104 detach ITSELF from inside
+the handler once the clears are done (`gdb.execute("detach")` from the
+stop handler; the tracee resumes; the script ends and gdb exits) so NO
+gdb is ever held while the client runs - then t102 always has the slot
+and the wedge class disappears end-to-end.
+
+**16. The allocation/post-device phase is the remaining gate.** With
+VKDEVICE green the client waits in a site that is neither the +0x80 word
+pattern nor (likely) the A8 count path; the next session should (a) get
+the detach-first t104 live, (b) run t102 immediately after VKDEVICE to
+name the wait (read PC/bt/A8 count = base+0x2070b7c), and (c) inspect the
+service for this client (pushes decoded / releases applied / counter
+pub_calls) to see whether the allocation needs the live injections only
+or a service-side gap.
+
+**17. Session 95: THE GDB THREAD-STOP INTERFERENCE + the SIGKILL nuance.**
+The t105 attach (t=1.5 s, before the client's worker threads spawned) left
+four worker threads in `ptrace_stop.part.0` (futex) - the client's main
+thread span waiting on them, so the client froze at the post-device wait
+with the gdb holding the trace. TERM/INT/HUP on the wedged gdb were all
+ignored (batch gdb stuck in continue). SIGKILL on the gdb (1584) was tried
+with the reasoning that all stopped threads sat at futex (NO RM lock held):
+result - the tracer died, the tracees DETACHED+RESUMED (client 1518
+SURVIVED with its workers back in futex_wait_queue; contrast session 88
+where SIGKILLed-tracer tracees were KILLED mid-syscall). So the SIGKILL
+outcome depends on the tracee's stop context; the mission's never-SIGKILL
+rule stands, but this datapoint matters when judging recoverability.
+After the gdb died, 1518 still held the RM write lock from its own
+post-device wait (its wait word was no longer polled because unwait.py
+tracks the NEWEST vk_frame and a fresh test client 6555 was launched and
+blocked D-state on os_acquire_rwlock_write) - the unwait-newest coupling
+must be replaced by ALL-clients polling in the next form.
+
+**18. HARNESS DESIGN RULES (accumulated).** (1) A gdb attached across
+thread-spawn windows stops worker threads and freezes the client - attach
+only AFTER all threads exist, or ensure the gdb explicitly resumes new
+threads, or do the work with NO gdb. (2) Any gdb that must run through
+thread-spawn windows MUST self-detach quickly (the instance phase spans
+~10-20 s and the clears arrive in ~5 bursts) and NEVER rely on outer
+timeouts. (3) unwait.py should poll EVERY live vk_frame (not just the
+newest). (4) The durable answer for all of this is the service-side
+folding: the instance markers and the +0x80 words are guest-RAM writes a
+kernel-side service can perform on the same triggers (the marker semantics
+and the word semantics are understood) - that removes gdb and /proc
+helpers from the loop entirely.
+
+**19. Session 96: THE DETACH-FIRST CLEARER PROVEN (t106) + the orphan-tracer plague.**
+t106.py (single +0xB33D30 BP; clears markers; detaches FROM THE HANDLER as
+soon as cleared>=5 or chits>=14: `gdb.execute("detach")` + return True so
+the batch gdb exits) on fresh r696 (mdev 241af1c1): CLEAR#1..#5,
+`BC-DETACHED cleaned=5 chits=5`, `T106-DONE chits=5 cleared=5`,
+`TracerPid: 0` - THE SLOT IS LEFT FREE. unwait.py wrote five words and the
+client (1549) reached `VKINSTANCE ok / VKDEVS 2 / VKPICK / VKDEVICE ok
+family=0`. THE CLEAN FORM OF THE CLEARER EXISTS NOW.
+
+**20. THE ORPHAN-TRACER PLAGUE (root cause of every t102 failure).** Every
+INTERRUPTED gdb attempt leaves an attached orphan: (a) piping gdb to
+`head` closes the pipe early and the gdb stays attached (hit twice); (b) an
+ssh command dying/timing out mid-gdb-add leaves the gdb running (hit
+again). Symptoms: later attaches fail with `Could not attach ... already
+traced by process N` / `ptrace: Inappropriate ioctl for device` and the
+t102 log stays empty. RECOVERY (session-95 evidence): TERM is usually
+ignored (wedged in continue); SIGKILL the orphan + `kill -CONT <client>`;
+the tracee SURVIVES when it was running/at-futex and detaches cleanly
+(TracerPid 0). STRICT DISCIPLINE: never pipe gdb output (redirect to a
+file, read it after); never let an ssh command end while a gdb may be
+attached (generous timeouts); after every gdb run verify TracerPid 0.
+
+**21. THE NEXT FORM (t107, one gdb, one window).** Combine the proven
+clearer and the diagnostic in ONE batch gdb: attach -> clears -> detach ->
+`python time.sleep(...)` idle -> RE-ATTACH (gdb can attach again after
+detach) -> read PC / bt / A8 count (t102 content) -> detach -> print -
+one ssh session, one window, everything self-contained, no orphan risk
+(every path ends in an explicit detach; wrap each phase in try/except with
+a final detach). Expected: names the post-device wait (allocation/UVM vs
+A8 count base+0x2070b7c) so the allocation can be passed and the first
+VKFRAME reached.
+
+**22. Session 97: THE DEEP SUBMISSION STATE + THE COUNTER-SLOT WEB.**
+The r696 client (1549) reached the deepest state yet: the service reported
+`gpfifo done=11576 pushes=10932 counter pub_calls=1356` and climbing
+(gpfifo done was 0 in the pre-pipeline boots) - the client submits the full
+work stream and the service decodes/processes it (pairs=14395 map=14318,
+ipins=9240 from live_walk). The client then sits in the KNOWN userspace
+post-device spin (main thread `running` in all samples; one tid in
+os_acquire_rwlock_read). NEW: a counter-slot collision spam
+(`counter auto ... pa=0x5fedfff0 rejected (slot owned by another channel;
+not installed)`, 232+ callbacks suppressed) - the 0x20400f000 injection
+resolved the WHOLE counter VA family to ONE page, so both live channels
+collided; the manual bypass `echo <h>,<pa> > .../parameters/sema_counter`
+(h_object,pa syntax, main.c:2965) was applied for both handles
+(0xbeee0100,0xbeec0100 -> 0x5fedfff0) and pub_calls jumped 1356 -> 3739.
+The client gate (VKDEVICE -> frames) persists: its remaining wait is the
+userspace spin word NOT covered by the +0x80/16K poller (candidate: the
+tracker/counter the service now advances - watch whether pub traffic
+alone resolves the spin, else t107-GDB the spin site). ENVIRONMENT: one
+orphan gdb (10092 on 1549) + stale channels (beec0100) from prior
+attempts; the orphan-kill discipline applies (SIGKILL by PID + CONT +
+verify TracerPid).
+
+**23. THE IOCTL-ARG SAMPLING AXIS (no ptrace).** /proc/<tid>/syscall shows
+the running ioctl's ARGS (fd, cmd, arg) when caught - sampling the main
+thread repeatedly during an ioctl-loop phase names the exact stuck RM
+control without any gdb. Use it before ptrace-based diagnostics when the
+thread does syscalls at a catchable rate.
+
+**24. Session 98: THE POST-DEVICE SPIN NAMED - THE SEC2 SCRUBBER AUTH-TAG WAIT.**
+perf (no ptrace!) on the spun client: top symbol `channelReadChannelMemdesc`
+24.33% (then scheduler internals) - the guest's nvidia driver hot-loops
+reading the CHANNEL MEMDESC. Source decode (nvidia-570.144-reference,
+sec2_utils.c): the only reference caller is
+`_sec2utilsUpdateGetPtr` -> `channelReadChannelMemdesc(pChannel,
+authTagBufSemaOffset)` inside `_sec2utilsGetNextAuthTagSlot`, whose loop
+waits for the scrub auth-tag slots to free; `authTagBufSemaOffset =
+finishPayloadOffset + CHANNEL_ENGINE_SEMAPHORE_SIZE` (4). So the spin =
+the SEC2 MEMORY-SCRUBBER channel waiting for its auth-tag semaphore (a u32
+inside the channel memdesc buffer) to advance - the GPU never delivers it
+in our emulation, so the client waits forever (each poll = one full memdesc
+transfer = the 24% cost). THE FIX DIRECTION: the service must complete the
+scrubber's semaphore updates - the scrubber channel registers in the
+service (`scrubber channel 0x... detected (gpfifo=... userd=...)`) and its
+submits carry semaphore releases targeting the auth-tag VA
+(pbGpuVA + authTagBufSemaOffset) in the channel buffer; on decode, write
+the release value to that VA's page (the channel buffer is a guest-RAM
+allocation the service can resolve/pin; the pending queue holding 17
+entries is the suspect pool for the unresolvable ones - inspect
+svc_pending[]). Service-side counter to watch after the fix: the scrub
+loop exits, the client proceeds to the allocation/pipeline. Tool: perf
+record/report on the guest is the no-ptrace spin-pointer (`perf record -p
+<pid> -- sleep 3; perf report --stdio --sort=symbol,dso`).
+
+**25. Session 99: THE SCRUBBER GATE CONNECTED TO THE CE CHANNELS' ok=0.**
+chan_dump on r696 (client 1549 spun): six channels -
+`h=0xbeee0100 eng=0 gpfifo=0x44a0000 userd=0xfee00 ok=1 get=6 sub=6354 rel=0x4213020 crel=0x5fedfff0 notif=0x11cd51060`
+`h=0xbeec0100 eng=0 gpfifo=0x4cc0000 userd=0xfee00 ok=1 rel=0x4923020 crel=0x5fedfff0`
+`h=0xbeec0104 eng=0 gpfifo=0x4d20000 userd=0xfee00 ok=1 rel=0x4b17e30 crel=0x5fedffe0`
+and the CE-class channels `beee0101 (eng=11, gpfifo=0x4500000 userd=0x8f600 as=2 ok=0)`,
+`beee0102 (eng=12, gpfifo=0x4560000 userd=0x8f400 ok=0)`, `beed0101 (eng=12,
+gpfifo=0x49c0000 userd=0x8f200 ok=0)` - ALL ok=0: the CE userd discovery
+(failed probe: EAGAIN 0/0 or EINVAL) means no CE pump, no CE sema applies,
+so the SEC2 scrubber channel (a CE-class channel) never sees its auth-tag
+semaphore advance and the client spins in _sec2utilsGetNextAuthTagSlot.
+The service's candidates (registered PA, ring+0x10000 in both apertures,
+8-slot list) all miss for the CE channels. `find_userd` (FB scan for a
+live put@0x8c/get@0x88 pair) found ZERO candidates - the CE userds are not
+in FB with live puts (either the client's PUT writes go to a page the
+service's candidates miss, or the CE userds genuinely idle). A raw host-RAM
+shape scan (put in 1..65535, get<=put) is too noisy (24 hits in the first
+28KB of BIOS-era low RAM) - needs tighter validation. NEXT APPROACH: find
+the REAL userd page by DIFF-WATCH - sample the candidate pages (and the
+CE ring pages) twice around a scrub submit and diff for the page whose
++0x88/+0x8c pair changes; the guest kernel writes GP_PUT there, so the
+changing page is the true userd. Then fix the discovery (or add a
+sema_userd-style injection param) so the CE pump engages and the scrubber
+completes. NOTE: the GR channels' userd=0xfee00 (as=2, FB) and ok=1 with
+get=6/sub=6354 - the GR machinery runs fine; only the CE/scrubber path is
+dark.
+
+**26. Session 100: THE CE USERD HUNT EXHAUSTED THE STANDARD LAYOUTS; THE CIRCULAR WAIT.**
+Candidates examined for the CE userds this session: (a) the registered FB
+offsets (0x8f600/0x8f400/0x8f200 as=2) - ALL ZERO in the FB shadow
+(sema_vidmem_dump 0x8f100,0x600 twice); (b) the ring-PA + 0x10000 linear
+guess - random/zero/table garbage (the allocations are NON-CONTIGUOUS);
+(c) the PDB walks of gpfifo+0x10000 (0x4501000->pa 0x125e91000,
+0x4561000->pa 0x132032000, 0x49c1000->pa 0x11bddd000 via pdb 0x1152f4000)
+- ALL THREE PAGES ALL ZERO. The CE userds were never written - and the
+deeper reading: the SEC2 scrub flow is a CIRCULAR WAIT: the scrubber waits
+for its previous scrub pushes to be CONSUMED (the auth-tag sema advances
+only when the GPU runs the scrub) -> the consumption needs the CE pump ->
+the CE pump needs a live userd -> the userd is written BY SUBMITS -> the
+submits are held by the wait. In a working boot the guest's first scrub
+push happens BEFORE the wait loop (the loop only waits when the slots are
+full) - so in our boots the FIRST scrub push must have gone into the CE
+ring, the service failed to consume it (ok=0/userd dark), and the client
+now waits for a completion that will never come.
+
+**27. THE DIRECT BYPASS (next session, ranked).** (1) Write the auth-tag
+semaphore in the CHANNEL BUFFER (memdesc) page directly: the scrub loop's
+`_sec2utilsUpdateGetPtr` reads `channelReadChannelMemdesc(pChannel,
+authTagBufSemaOffset)` = a u32 at memdesc+finishPayloadOffset+4; advancing
+that value frees the slots and unblocks the client WITHOUT the CE pump.
+The memdesc = the channel buffer allocation (a VA in the client's space);
+resolve it via the PDB (the guest's channel struct gives the buffer VA;
+the service's channel registration/alloc params carry it) and write the
+incremented value from the service (or the host RAM file). (2) OR decode
+the scrub pushes the service already sees (check sema-push for the SCRUBBER
+channel - its submits carry the auth-tag VA; the release decode should
+target it; the service's release applies would write the sema if the VA
+resolves). (3) The registered FB userd offsets being zero suggests the
+client's kernel maps the userd through a VA whose PA the discovery misses;
+the diff-watch (two samples around activity) remains the fallback proof.
+
+**28. Session 101: THE LOOP SEMANTICS + perf trace (NO-PTRACE SYSCALL TRACING).**
+The wait loop decoded exactly (sec2_utils.c:420-441):
+`while (nextPut == pSec2Utils->authTagGetIndex) { gpuCheckTimeout();
+authTagGetIndex = _sec2utilsUpdateGetPtr(); osSpinLoop(); }` - it exits
+when the memdesc sema (u32 at pChannelBufferMemdesc +
+authTagBufSemaOffset; authTagBufSemaOffset = finishPayloadOffset + 4)
+advances past nextPut; it also has a GPU timeout that would error out.
+perf trace on the spun client (no ptrace!) revealed the live pattern:
+tid 1793 = a 10ms-timed futex loop (WAKE val=1 then
+WAIT_BITSET|CLOCK_REALTIME 10ms - the paced retry); tid 1788 = repeated
+`openat -> read(fd, 299 bytes) -> close -> futex-wait` - a transient-fd
+POLLING READER of a small blob (the fd is opened and closed each cycle, so
+fd 28 no longer exists; the client's persistent fds: /dev/nvidia-modeset
+(10), /dev/nvidia0 (11/12/14/15), GLCache .toc/.bin (16/17), memfd
+\"allocation fd\" (18)). The 299-byte read is the likely userland face of
+the memdesc read path (the 24% channelReadChannelMemdesc hot loop).
+NEXT: (a) catch the transient fd path by sampling `ls -l /proc/<pid>/fd`
+every ~0.2 s in a loop around the burst (or perf trace -s during a burst);
+(b) if it is a driver read() of the memdesc, resolve its buffer and write
+the auth-tag sema value (advance past nextPut) via the RAM file; (c) the
+loop timeout (gpuCheckTimeout) may eventually error the client - the write
+must land first.
+
+**29. Session 102: fd-sampler miss, no nvidia debugfs; the memdesc hunt narrows.**
+The 0.15 s /proc/1549/fd sampler over 20 s caught NO transient path (the
+open/read(299)/close reader is bursty or dormant between bursts; the static
+fds are /dev/nvidiactl, /dev/nvidia0, /dev/nvidia-modeset, a socket, the
+GLCache .toc/.bin pair, and a memfd "allocation fd"). No nvidia debugfs
+tree exists in the guest. Remaining ranked paths to the auth-tag sema
+(memdesc + finishPayloadOffset + 4): (1) content-locate the scrubber's
+channel buffer by its method stream (the C5B5/C56F shapes: SET_OBJECT
+0xc5c0/0xc5b5, headers 0x200121ed/0x200120a9 observed in the session-78
+window dumps) and read/write the sema inside; (2) the scrubber channel's
+own sema-push decodes (its pushes carry the auth-tag VA = pbGpuVA +
+authTagBufSemaOffset; grep the boot's sema-pushall for the scrubber
+channel id and read the rel= VA - then PDB-walk it); (3) sample the fd
+burst narrower (0.02 s) right after a client restart when the burst is
+hot. The service-side ideal: on detecting the scrubber channel, resolve
+its channel buffer and pre-advance the auth-tag sema (a durable fold).
+
+**30. Session 103: THE SCRUB STREAM FOUND; ONE RELEASE DELIVERED; THE SEMA PAGE STILL AMBIGUOUS.**
+Content-located the CE/scrub init method stream (headers 0x200121ed/
+0x200120a9 searched): full stream at `pa=0x114e98000` = SET_OBJECT
+0xC5C0/0xC5B5 (0x20012000/0x20012080 headers), engine-init methods
+0x200121ed(3)/0x200120a9(1)/0x2001209c(0x0c000001), bulk methods
+(0x8000xxxx), then TWO REPORT_SEM releases in one buffer:
+`0x200426c0 00000000 04b19b10 ...` (va=0x4b19b10) and
+`0x200426c0 00000000 04b17e30 ...` (va=0x4b17e30), tail 0x2001c00d
+70/68. Twins at pa=0x1026ad000+0x87c and pa=0x12afba000+0x87c (identical
+content at identical offsets - multi-mapped copies). Release resolution:
+`0x4b17e30 -> pa=0x114efbe30 (SYS) cur=1` - THAT release WAS delivered (the
+service applied it, cur=1). `0x4b19b10` resolves via the walker's big-half
+2MB branch to FB 0x660b10 holding garbage (0xD9F26F94) - poke_vidmem
+tests ([0x660b10]=1 then =0x40000000) did NOT flip the client, and the
+linear-block guesses (buffer VA 0x4b00000 <-> stream pa 0x114e98000, sema
+candidate 0x114eb1b10) read all zero. NEXT: the auth-tag sema page must be
+picked by VALIDATION (the existing value must be a small plausible
+GetPtr): walk 0x4b19000 returning ALL chains (small/64K/2MB both
+apertures), pick the page whose u32 at +0x1b10 (or the offset the memdesc
+actually uses) is a small number; or take the buffer aperture from the
+channel's alloc params (the service's registration) and walk with that
+aperture exclusively. The fix remains: advance that u32 (write via the
+host RAM file if SYS, poke_vidmem if FB).
+
+**31. Session 104: THE SEMA VA 0x4b19b10 IS NOT THE LOOP'S PAGE; THE KPROBE IS THE ANSWER.**
+The validated multi-chain walk (small-4KB and big chains BOTH) resolves
+VA 0x4b19b10 -> FB 0x660b10 (both chains converge; the guest's own PTs say
+the object lives in vidmem). poke_vidmem writes at 0x660b10 (0x1,
+0x40000000) and 0x660b14 (0x1, 0x40000000) LANDED (verified via
+sema_vidmem_dump: [0x660b10]=0x11112222) yet the client does not flip and
+perf still shows channelReadChannelMemdesc at 24.52%. CONCLUSION: the
+REPORT_SEM VA 0x4b19b10 (and 0x4b17e30) are NOT the auth-tag sema the loop
+reads - they are the GL-init release objects; the real auth-tag sema is at
+pChannelBufferMemdesc + finishPayloadOffset + 4 (a VA and page we do not
+yet know; finishPayloadOffset is NOT 0x19b10). THE DEFINITIVE INSTRUMENT:
+a guest kprobe on channelReadChannelMemdesc (the offset argument = %si =
+the authTagBufSemaOffset VALUE; and the pChannel pointer lets us read the
+channel struct -> pChannelBufferMemdesc): `echo 'p:crcd
+channelReadChannelMemdesc offset=%si' >> /sys/kernel/tracing/kprobe_events`
++ enable the event + read tracing pipe WHILE the loop runs. This names the
+offset and the memdesc in one shot, then the sema page = walk(memdesc+off)
+and the advance (RAM file if SYS; FB + poke if the converged mapping).
+
+**32. Session 105: KPROBE BLOCKED (notrace); THE BAR1-PATH ANALYSIS; THE CHANNEL-FIELD OFFSETS.**
+(1) The kprobe on channelReadChannelMemdesc is IMPOSSIBLE: the guest's
+dmesg says `trace_kprobe: Could not probe notrace function
+channelReadChannelMemdesc [nvidia]` - the function is annotated notrace
+(ftrace/perf-probe/BPF-kprobe all blocked). The tracefs filesystem had to
+be mounted first (mount -t tracefs tracefs /sys/kernel/tracing) and it
+already carries UVM kprobes from earlier sessions (uvmt/uvch/uvbp/uvtw/
+uvsv...). The static sec2utils functions (the callers) are absent from
+kallsyms (inlined) - no probe target on that path.
+(2) BAR1-path analysis: the guest driver's memdesc read uses
+TRANSFER_FLAGS_USE_BAR1|SHADOW_ALLOC|SHADOW_INIT_MEM - a CPU read of the
+GUEST'S BAR1 mapping, which the mdev mmap maps onto the HOST's vidmem
+shadow (the same buffer poke_vidmem writes). The pokes at FB 0x660b10/
+0x660b14 LANDED (verified) yet the loop did not advance, so the memdesc
+page is NOT the FB 0x660b10 the PT walk converged on (VA 0x4b19b10 belongs
+to the GL-init release object, not the channel-buffer memdesc).
+(3) The channel registration parse exposes the guest-visible channel
+fields and their alloc-param offsets (gsp_rm_emulation.c
+gsp_rm_register_channel: GSPRM_CHAN_GPFIFO_OFFSET/ENTRIES, ENGINE_TYPE,
+CID, INSTANCE_MEM, USERD_MEM(+APERTURE), NOTIFIER_MEM(+APERTURE)) - the
+channel-buffer/memdesc field (pChannelBufferMemdesc) is not among them;
+the next session should dump a CE channel's FULL alloc params (the
+service's rate-limited alloc dumps or a fresh boot) and look for the
+channel-buffer VA/PA field, then sema = buffer + finishPayloadOffset + 4
+(per-channel offset, e.g. the 0x19b10-style for the GL channel).
+(4) Fallback if the field is not in the params: per-VA content hunt for
+the auth-tag sema VALUE (the retprobe would have given it; without
+kprobes: the sema value = a small slot counter - scan the CE channel's
+allocation candidates for a plausible counter that CHANGES across two
+samples).
+
+**33. Session 106: THE CHANNEL BUFFER IS NOT A CLIENT PARAM; THE TAIL SWEEP AND HASH SCAN NEGATIVE.**
+NV_CHANNEL_ALLOC_PARAMS (alloc_channel.h:300-341) layout mapped to the
+service offsets: gpFifoOffset@8/entries@16, engineType@128, cid@132,
+instanceMem@144, userdMem@168, ramfcMem@192, mthdbufMem@216,
+hPhysChannelGroup/internalFlags, errorNotifierMem@248... - THE CHANNEL
+BUFFER (pChannelBufferMemdesc) IS NOT IN THE PARAMS: it is a
+driver-internal memdesc created by channelAlloc, so the alloc-params path
+cannot name it (the kprobe/debuginfo path was the right one; it is
+notrace-blocked). The finish-payload tail of the stream page
+(0x114e98000 +0xd4..+0x118) was swept with 0x40000000 (18 u32s; the
+pre-sweep values: +0xd8=0x68000000 finish marker, +0xe0=0x901025a6,
++0xe4..+0xec=0x8000xxxx methods) - no flip. A 16-page high-entropy scan of
+0x114e98000..0x114ea8000 for the auth-tag SHA-256 array: ZERO high-entropy
+32-byte blocks (the hash array is elsewhere / another allocation). NEXT:
+(1) the memmgrMemDescBeginTransfer SHADOW insight - the read happens from
+a FRESH kmalloc'd shadow copied from the memdesc each call; the shadow's
+source = the memdesc's backing page; the memdesc page must be found by
+TRANSFER-TIME properties (not the VA walk); (2) or a guest perf hardware
+watchpoint on the driver's BAR1 ioremap VA + the (unknown) memdesc offset;
+(3) or the service-side broad assist: increment a RANGE of plausible FB
+sema cells periodically (blind but bounded), watching for the flip.
+
+**34. Session 107: FB ASSIST SWEEPS NEGATIVE - THE READ IS NOT FB-SOURCED; THE TWINS ARE THE NEXT TARGET.**
+Two bounded FB "sema assist" sweeps (distinct values per cell,
+0x40000000|idx): (a) the walked page 0x660b00..0x661000 (320 cells) - no
+flip; (b) the CE-channel allocation cluster 0x8f000..0x90000 (1024 cells,
+where the registered userds live) - no flip; perf re-confirmed
+channelReadChannelMemdesc 23.13% (the loop still runs). CONCLUSION: the
+memdesc read does NOT come from the FB shadow the CPU can poke - the
+transfer's SHADOW_INIT must read the memdesc through a path that bypasses
+these bytes, or the memdesc backing is SYSTEM memory via an unknown VA.
+NEXT (one-minute test): the STREAM TWINS - pa 0x1026ad000 and
+0x12afba000 hold BYTE-IDENTICAL copies of the CE method stream (verified at
++0x7c0/+0x880: same 0x68000000 finish marker area, same 0x200120a9/9c
+methods) - one of them is very likely the memdesc BACKING in sysmem (the
+driver copies the same stream per channel/boot). Sweep BOTH twins'
+stream-area pages (reuse the assist code; 1 page each) and watch for the
+flip; if it flips, bisect the winning page to the sema cell and make it
+durable (host RAM file writes = zero-cost, no poke_vidmem needed for
+sysmem). Failing that: content-diff the twins across a driver transition
+(the memdesc copy updates during submits; the copy that CHANGES is the
+backing).
+
+**35. Session 108: THE CONTENT-DIFF - 9 ACTIVE PAGES + THE CONSTANT-READ DEDUCTION.**
+Page-hash diff over three +/-256KB windows (twins 0x1026ad000/0x12afba000,
+stream 0x114e98000) across 15 s: NINE pages change:
+0x102690000, 0x102698000 (440 word diffs - a buffer written continuously =
+the push copy!), 0x1026a7000 (+0xe8: 934 -> 940 - a slow clean counter),
+0x1026b3000 (+0x48: 32768 -> 1 transition), 0x1026b5000 (+0x244/+0x24c:
+24987 -> 32765, two u16s at ~650/s), 0x1026c2000 (+0xe8: 232 -> 3, a
+wrapping counter mod ~233), 0x1026e6000 (+0x18: 2764 -> 1930, +0x88/+0x90
+decreasing - countdowns), 0x114e6b000, 0x114e6e000. THE DEDUCTION: the
+stuck loop's read value must be CONSTANT (its page shows no writes; the
+GPU never advances the sema), so the auth-tag sema is a static cell -
+likely zero - and the fix is to make the read NONZERO (advance it) at the
+right page. None of the 9 changing pages is it (their cells move on their
+own). NEXT: (1) restrict the search to CONSTANT pages among the CE
+channel's candidate backing (the twins region pages that DID NOT change
+but sit adjacent to the active set); (2) target the sema cell: the cell
+that equals nextPut (small constant, e.g. 0) at the memdesc+off; sweep the
+CONSTANT pages' zero u32s in a bounded ring around the twins/active set
+with 1 (the sysmem write is free); (3) the durable service fold once
+found.
+
+**36. Session 109: THE BURST vs HEARTBEAT DISCOVERY + THE 200ms READER.**
+perf trace of the whole process over 10 s with -e ioctl: ZERO ioctls from
+ANY thread; meanwhile the process shows a strict 200 ms pattern on thread
+1788: `openat(filename ptr 0x5eaec740) -> read(fd 28, count 299) -> close`
+(the transient fd the fd-sampler missed!) and thread 1793's 10 ms
+futex re-arms. The perf-reported 24% channelReadChannelMemdesc (a kernel
+function only reachable via RM calls) CONTRADICTS the zero-ioctl window:
+the hot kernel work is BURSTY (periodic rounds between the 200 ms
+heartbeats), not continuous. The 299-byte blob read every 200 ms = the
+client's health poll (unnamed path; candidate: /proc/driver/nvidia/...
+informational node - find it by reading /proc/PID/io or by narrowing the
+sampler to the 200 ms phase). THE MODEL NOW: the client idles (futex
+waits) + wakes every 200 ms to poll a node + periodically runs RM bursts
+(the burst = a full retry round hitting channelReadChannelMemdesc). NEXT:
+(1) name the 299-byte node (it may BE the wait's condition - a status the
+service must change); (2) capture the BURST with perf (a long record +
+per-thread breakdown) to see which thread runs the memdesc reads (the RM
+burst thread) and correlate with the host-side RPC traffic in the same
+window (the host dmesg RPC lines during a burst = the exact RM call!).
+
+**37. Session 110: THE LOOP IS CONTINUOUS INSIDE ONE LONG IOCTL (THE LOCK MODEL CLOSED).**
+(1) The 299-byte/200ms reader: the filename pointer 0x5eaec740 is NEVER in
+the client's maps (30 Hz sampler over 15 s, zero hits) yet openat returns
+28 every cycle - a decode artifact/transience, and at 5 reads/s it is
+irrelevant to the 24% blocker. Treat it as a RED HERRING. (2) A 20-second
+perf record (80K samples): channelReadChannelMemdesc 24.65% + a scheduler
+family (pick_next_task_fair/update_curr/pick_eevdf/__schedule/
+schedule_timeout/os_schedule ~30% total) = the SEC2 wait loop running
+CONTINUOUSLY. The earlier zero-ioctl windows are explained: THE MAIN
+THREAD SITS INSIDE ONE LONG-LIVED ioctl (entered hours ago; the loop runs
+in the kernel inside it) - and that ioctl HOLDS THE RM WRITE LOCK, which
+closes the lock model: tid 1791 waits on the RM read lock, new clients
+block D-state (os_acquire_rwlock_write), all consistent. (3) PEBS/
+mem-loads events are unavailable in the guest perf (no memory-address
+sampling) - the read address cannot be sampled. Instruments for the sema
+address are now exhausted short of the notrace blocker or the blind
+service-side assist.
+NEXT: (1) the blind assist, refined: the loop's READ is CONTINUOUS and
+the sema is CONSTANT - sweep ONLY constant small-value cells in the CE
+candidate pages (see session 108's active set for the region), one value
+at a time, 2 s per batch, watching the flip; (2) or a graft: since the
+loop runs inside the client's blocked ioctl, unloading the CLIENT (kill)
+releases the RM lock and the fresh client restarts the same phase - use
+fresh-client runs for any A/B of service-side fixes; (3) the durable
+service-side fold remains the endgame once the cell is known.
+
+**38. Session 111: THE FB IS FULLY DISPROVED (WHOLE-FB FILL, NO FLIP); THE SYS-FILL WEDGED THE GUEST (CYCLE NEXT).**
+(1) scan_vidmem for the stream signatures (0x200121ed, 0x200120a9) and even
+0x68: ZERO hits across 256MB - THE FB SHADOW WAS ALL ZEROS (before any
+pokes): the GPU-side objects are never written in our emulation (the
+driver->GPU DMA never runs), so the driver's memdesc SHADOW_INIT copies
+zeros and the sec2 loop reads 0 forever (nextPut==authTagGetIndex==0 ->
+infinite wait; the complete model). (2) The refined assist: swept FB
+0x80000..0x90000 (16384 cells, distinct values; 60s/32KB via poke_vidmem)
+- no flip. (3) THE DEFINITIVE TEST: `fill_vidmem 0x40000000` filled the
+WHOLE 256MB FB shadow in-kernel (`vgsp: fill_vidmem value=0x40000000
+size=0x10000000 done`) - STILL NO FLIP: THE LOOP'S READ DOES NOT COME FROM
+THE FB SHADOW AT ALL; the channel-buffer memdesc is SYS-BACKED (or another
+backing) in our emulation, not the FB. (4) The follow-up SYS fill (128
+candidate pages near the active set overwritten with 0x40000000 via the
+RAM file) CLOBBERED LIVE DRIVER DATA and WEDGED THE GUEST (SSH banner
+timeout; QEMU alive) - the next session MUST CYCLE (fresh boot); lesson:
+never bulk-overwrite pages adjacent to the identified active set (0x102698000
+was a hot staging buffer).
+NEXT: (1) fresh cycle; (2) the sema is a SYS page - apply the full-page
+marker fill ONLY to genuinely-constant, non-active pages (re-run the
+content-diff first, then fill the constant set in small batches with a
+client check after each, NOT the active neighborhoods); (3) or the
+service-side durable route: have the SERVICE write the marker into the
+channel-buffer page once it can resolve it (the memdesc VA = the CE
+channel's buffer; the service's channel registration fields + the PT walk
+give the page candidates).
diff --git a/tools/guest-diag/wall2/kc_dump_fn.py b/tools/guest-diag/wall2/kc_dump_fn.py
new file mode 100644
index 0000000..e481d24
--- /dev/null
+++ b/tools/guest-diag/wall2/kc_dump_fn.py
@@ -0,0 +1,21 @@
+import struct, sys
+VA = int(sys.argv[1], 16)
+N = int(sys.argv[2], 16) if len(sys.argv) > 2 else 0x800
+f = open("/proc/kcore","rb")
+hdr = f.read(64)
+e_phoff = struct.unpack_from("<Q", hdr, 0x20)[0]
+e_phentsize = struct.unpack_from("<H", hdr, 0x36)[0]
+e_phnum = struct.unpack_from("<H", hdr, 0x38)[0]
+segs = []
+for i in range(e_phnum):
+ f.seek(e_phoff + i*e_phentsize)
+ ph = f.read(e_phentsize)
+ p_type, p_flags, p_offset, p_vaddr, p_paddr, p_filesz, p_memsz = struct.unpack_from("<IIQQQQQ", ph, 0)
+ if p_type == 1:
+ segs.append((p_vaddr, p_memsz, p_offset))
+for v, sz, off in segs:
+ if v <= VA < v+sz:
+ f.seek(off + (VA - v))
+ d = f.read(min(N, v+sz-VA))
+ sys.stdout.write(d.hex())
+ break
diff --git a/tools/guest-diag/wall2/nudge2.sh b/tools/guest-diag/wall2/nudge2.sh
new file mode 100755
index 0000000..16ebcd8
--- /dev/null
+++ b/tools/guest-diag/wall2/nudge2.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# nudge2.sh - widened readiness-word nudger: ALL rw-s nvidia mappings <=256K,
+# writes 1 to any +0x80 u32 that is 0.
+DUR=${1:-300}
+F=/dev/shm/vgpu-guest-ram
+END=$(( $(date +%s) + DUR ))
+sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/guest_words2.sh ubuntu@127.0.0.1:/tmp/guest_words2.sh >/dev/null 2>&1
+while [ "$(date +%s)" -lt "$END" ]; do
+ OUT=$(timeout 12 sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 "sudo bash /tmp/guest_words2.sh" 2>/dev/null)
+ [ -n "$OUT" ] && echo "[$(date +%H:%M:%S)] words:" && echo "$OUT"
+ echo "$OUT" | while read -r VA PA SZ; do
+ [ -z "$PA" ] && continue
+ PAD=$((PA)); FOD=$((PAD - 0x80000000)); [ $FOD -lt 0 ] && FOD=$((PAD))
+ CUR=$(sudo dd if=$F bs=4 skip=$((FOD/4)) count=1 2>/dev/null | xxd -p)
+ if [ "$CUR" = "00000000" ]; then
+ printf '\x01\x00\x00\x00' | sudo dd of=$F bs=1 seek=$FOD count=4 conv=notrunc 2>/dev/null
+ echo " NUDGE va=$VA pa=$PA -> 1"
+ fi
+ done
+ sleep 1.5
+done
+echo NUDGE2-DONE
diff --git a/tools/guest-diag/wall2/t101.py b/tools/guest-diag/wall2/t101.py
new file mode 100644
index 0000000..436c609
--- /dev/null
+++ b/tools/guest-diag/wall2/t101.py
@@ -0,0 +1,55 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x PID %d" % (base or 0, pid))
+def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a))
+def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a))
+state = {"chits":0, "cleared":0}
+class BC(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xB33D30), internal=False)
+ self.silent = True
+ def stop(self):
+ state["chits"] += 1
+ try:
+ obj = int(gdb.parse_and_eval("$rdi"))
+ n = int(gdb.parse_and_eval("$rsi"))
+ cnt = u32(obj + 0x22efc)
+ for i in range(0, min(cnt, 32)):
+ p = u64(obj + 0x1428 + i*8)
+ if not p:
+ continue
+ tgt = p + n * 0x10
+ try:
+ v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe)))
+ if v16 == 0xffff:
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xe))
+ gr("set {unsigned int}0x%x = 0" % tgt)
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xc))
+ state["cleared"] += 1
+ print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True)
+ except Exception:
+ pass
+ except Exception as e:
+ if state["chits"] <= 4:
+ print("chitfail %s" % e, flush=True)
+ if state["cleared"] >= 24:
+ self.enabled = False
+ print("BC-DISABLED", flush=True)
+ return False
+try:
+ BC(); print("BPS-SET", flush=True)
+ gr("continue")
+except Exception as e:
+ print("bpfail %s" % e, flush=True)
+print("T101-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True)
diff --git a/tools/guest-diag/wall2/t102.py b/tools/guest-diag/wall2/t102.py
new file mode 100644
index 0000000..759d974
--- /dev/null
+++ b/tools/guest-diag/wall2/t102.py
@@ -0,0 +1,31 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x PID %d" % (base or 0, pid))
+try:
+ pc = int(gdb.parse_and_eval("$pc"))
+ print("PC 0x%x off=0x%x" % (pc, pc - base if base else 0))
+except Exception as e:
+ print("pcfail %s" % e)
+try:
+ C = base + 0x2070B7C
+ print("A8COUNT %u" % int(gdb.parse_and_eval("*(unsigned int*)0x%x" % C)))
+except Exception as e:
+ print("countfail %s" % e)
+try:
+ print(gr("bt 6", to_string=True))
+except Exception as e:
+ print("btfail %s" % e)
+gr("detach")
+gr("quit")
+print("T102-DONE")
diff --git a/tools/guest-diag/wall2/t104.py b/tools/guest-diag/wall2/t104.py
new file mode 100644
index 0000000..5a4c059
--- /dev/null
+++ b/tools/guest-diag/wall2/t104.py
@@ -0,0 +1,56 @@
+import gdb, time
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x PID %d" % (base or 0, pid), flush=True)
+def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a))
+def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a))
+state = {"chits":0, "cleared":0, "t0":time.time(), "last":time.time()}
+class BC(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xB33D30), internal=False)
+ self.silent = True
+ def stop(self):
+ state["chits"] += 1
+ try:
+ obj = int(gdb.parse_and_eval("$rdi"))
+ n = int(gdb.parse_and_eval("$rsi"))
+ cnt = u32(obj + 0x22efc)
+ for i in range(0, min(cnt, 32)):
+ p = u64(obj + 0x1428 + i*8)
+ if not p:
+ continue
+ tgt = p + n * 0x10
+ try:
+ v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe)))
+ if v16 == 0xffff:
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xe))
+ gr("set {unsigned int}0x%x = 0" % tgt)
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xc))
+ state["cleared"] += 1
+ state["last"] = time.time()
+ print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True)
+ except Exception:
+ pass
+ except Exception:
+ pass
+ if state["cleared"] >= 8 or state["chits"] >= 40 or (time.time() - state["last"]) > 25.0:
+ self.enabled = False
+ print("BC-EXIT cleaned=%d chits=%d" % (state["cleared"], state["chits"]), flush=True)
+ return True
+ return False
+try:
+ BC(); print("BPS-SET", flush=True)
+ gr("continue")
+except Exception as e:
+ print("bpfail %s" % e, flush=True)
+print("T104-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True)
diff --git a/tools/guest-diag/wall2/t106.py b/tools/guest-diag/wall2/t106.py
new file mode 100644
index 0000000..ceaac44
--- /dev/null
+++ b/tools/guest-diag/wall2/t106.py
@@ -0,0 +1,61 @@
+import gdb, time
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x PID %d" % (base or 0, pid), flush=True)
+def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a))
+def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a))
+state = {"chits":0, "cleared":0, "done":False}
+class BC(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xB33D30), internal=False)
+ self.silent = True
+ def stop(self):
+ if state["done"]:
+ return False
+ state["chits"] += 1
+ try:
+ obj = int(gdb.parse_and_eval("$rdi"))
+ n = int(gdb.parse_and_eval("$rsi"))
+ cnt = u32(obj + 0x22efc)
+ for i in range(0, min(cnt, 32)):
+ p = u64(obj + 0x1428 + i*8)
+ if not p:
+ continue
+ tgt = p + n * 0x10
+ try:
+ v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe)))
+ if v16 == 0xffff:
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xe))
+ gr("set {unsigned int}0x%x = 0" % tgt)
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xc))
+ state["cleared"] += 1
+ print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True)
+ except Exception:
+ pass
+ except Exception:
+ pass
+ if state["cleared"] >= 5 or state["chits"] >= 14:
+ state["done"] = True
+ try:
+ gr("detach")
+ print("BC-DETACHED cleaned=%d chits=%d" % (state["cleared"], state["chits"]), flush=True)
+ except Exception as e:
+ print("detachfail %s" % e, flush=True)
+ return True
+ return False
+try:
+ BC(); print("BPS-SET", flush=True)
+ gr("continue")
+except Exception as e:
+ print("contend %s" % e, flush=True)
+print("T106-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True)
diff --git a/tools/guest-diag/wall2/t112c-gdbstub.py b/tools/guest-diag/wall2/t112c-gdbstub.py
new file mode 100644
index 0000000..27d0b9d
--- /dev/null
+++ b/tools/guest-diag/wall2/t112c-gdbstub.py
@@ -0,0 +1,42 @@
+import gdb
+
+gr = gdb.execute
+
+def u32(a):
+ return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a))
+def u64(a):
+ return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a))
+
+state = {"hits": {}}
+
+class HB(gdb.Breakpoint):
+ def __init__(self, pa, tag, quiet_after=8):
+ super().__init__("*0x%x" % pa, type=gdb.BP_BREAKPOINT, internal=False)
+ self.tag = tag
+ self.quiet_after = quiet_after
+ self.silent = True
+ def stop(self):
+ n = state["hits"].get(self.tag, 0) + 1
+ state["hits"][self.tag] = n
+ try:
+ rdi = int(gdb.parse_and_eval("$rdi"))
+ rsi = int(gdb.parse_and_eval("$rsi"))
+ rdx = int(gdb.parse_and_eval("$rdx"))
+ rcx = int(gdb.parse_and_eval("$rcx"))
+ r8 = int(gdb.parse_and_eval("$r8"))
+ r9 = int(gdb.parse_and_eval("$r9"))
+ if n <= self.quiet_after or self.tag in ("PRED", "COMPR"):
+ print("HIT %s #%d rdi=0x%x rsi=0x%x rdx=0x%x rcx=0x%x r8=0x%x r9=0x%x" %
+ (self.tag, n, rdi, rsi, rdx, rcx, r8, r9), flush=True)
+ except Exception as e:
+ print("err", e, flush=True)
+ return False
+
+HB(0xffffffffc0c403b0, "VIDH")
+HB(0xffffffffc0d667a0, "PMA", 4)
+HB(0xffffffffc0d6ae30, "PRED")
+HB(0xffffffffc0d57d60, "COMPR", 16)
+HB(0xffffffffc0d607f0, "SCRUB", 4)
+print("HB-SET", flush=True)
+gr("continue")
+print("T112B-DONE %s" % state["hits"], flush=True)
diff --git a/tools/guest-diag/wall2/t80r.py b/tools/guest-diag/wall2/t80r.py
new file mode 100644
index 0000000..ac2263b
--- /dev/null
+++ b/tools/guest-diag/wall2/t80r.py
@@ -0,0 +1,32 @@
+import gdb, sys
+inf = gdb.selected_inferior()
+pid = inf.pid
+cands = []
+for line in open("/proc/%d/maps" % pid):
+ if "nvidiactl" in line and "rw-s" in line:
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if hi - lo == 0x4000:
+ cands.append(lo + 0x80)
+print("CANDIDATES %s" % ["0x%x" % c for c in cands])
+class W(gdb.Breakpoint):
+ def __init__(self, addr):
+ super().__init__("*(unsigned int*)0x%x" % addr, gdb.BP_WATCHPOINT, gdb.WP_READ, internal=False)
+ self.addr = addr
+ self.silent = True
+ def stop(self):
+ print("READWORD 0x%x" % self.addr)
+ return True
+armed = 0
+for c in cands:
+ try:
+ W(c); armed += 1
+ except Exception as e:
+ print("watchfail 0x%x: %s" % (c, e))
+print("ARMED %d" % armed)
+if armed:
+ gdb.execute("continue")
+print("T80R-DONE")
diff --git a/tools/guest-diag/wall2/t83.py b/tools/guest-diag/wall2/t83.py
new file mode 100644
index 0000000..5e3cce2
--- /dev/null
+++ b/tools/guest-diag/wall2/t83.py
@@ -0,0 +1,25 @@
+import gdb
+from collections import Counter
+gr = gdb.execute
+pcs = Counter()
+inf = gdb.selected_inferior()
+for i in range(300):
+ try:
+ gr("stepi", to_string=True)
+ except Exception as e:
+ print("stepfail %s" % e); break
+ try:
+ pcs[int(gdb.parse_and_eval("$pc"))] += 1
+ except Exception:
+ break
+print("HOTPCS")
+top = pcs.most_common(4)
+for pc, n in top:
+ print("PC 0x%x hits=%d" % (pc, n))
+for pc, n in top[:1]:
+ try:
+ print(gr("x/10i 0x%x" % pc, to_string=True))
+ print(gr("info registers rax rbx rcx rdx rsi rdi rbp r8 r9 r10 r11 r12 r13 r14 r15", to_string=True))
+ except Exception as e:
+ print("fail %s" % e)
+print("T83-DONE")
diff --git a/tools/guest-diag/wall2/t89.py b/tools/guest-diag/wall2/t89.py
new file mode 100644
index 0000000..92c1de3
--- /dev/null
+++ b/tools/guest-diag/wall2/t89.py
@@ -0,0 +1,17 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x" % (base or 0))
+print(gr("x/70i 0x%x" % (base + 0xB28519), to_string=True))
+print(gr("x/40i 0x%x" % (base + 0xB28560), to_string=True))
+print("T89-DONE")
diff --git a/tools/guest-diag/wall2/t93.py b/tools/guest-diag/wall2/t93.py
new file mode 100644
index 0000000..6a0c23d
--- /dev/null
+++ b/tools/guest-diag/wall2/t93.py
@@ -0,0 +1,57 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x" % (base or 0))
+class B(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xB33D30), internal=False)
+ self.silent = True; self.done = False
+ def stop(self):
+ if self.done:
+ return True
+ self.done = True
+ try:
+ obj = int(gdb.parse_and_eval("$rdi"))
+ n = int(gdb.parse_and_eval("$rsi"))
+ print("RETRYFN obj=0x%x n=0x%x" % (obj, n))
+ cnt = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % (obj + 0x22efc)))
+ print("CNT %d" % cnt)
+ wrote = 0
+ for i in range(0, min(cnt, 32)):
+ p = int(gdb.parse_and_eval("*(unsigned long*)0x%x" % (obj + 0x1428 + i*8)))
+ if not p:
+ continue
+ tgt = p + n * 0x10
+ try:
+ v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe)))
+ v0 = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % tgt))
+ print("E%d ptr=0x%x tgt=0x%x w0=0x%x w16e=0x%x" % (i, p, tgt, v0, v16))
+ if v16 == 0xffff:
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xe))
+ gr("set {unsigned int}0x%x = 0" % tgt)
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xc))
+ wrote += 1
+ except Exception as e:
+ print("E%d fail %s" % (i, e))
+ print("CLEARED %d" % wrote)
+ gr("detach")
+ except Exception as e:
+ print("fail %s" % e)
+ return True
+try:
+ B()
+ print("BP-SET")
+ gr("continue")
+except Exception as e:
+ print("bpfail %s" % e)
+print("T92-DONE")
diff --git a/tools/guest-diag/wall2/t94.py b/tools/guest-diag/wall2/t94.py
new file mode 100644
index 0000000..0d2aa27
--- /dev/null
+++ b/tools/guest-diag/wall2/t94.py
@@ -0,0 +1,54 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x" % (base or 0))
+state = {"hits":0, "cleared":0}
+class B(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xB33D30), internal=False)
+ self.silent = True
+ def stop(self):
+ state["hits"] += 1
+ try:
+ obj = int(gdb.parse_and_eval("$rdi"))
+ n = int(gdb.parse_and_eval("$rsi"))
+ cnt = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % (obj + 0x22efc)))
+ if state["hits"] <= 6:
+ print("HIT %d obj=0x%x n=0x%x cnt=%d" % (state["hits"], obj, n, cnt))
+ for i in range(0, min(cnt, 32)):
+ p = int(gdb.parse_and_eval("*(unsigned long*)0x%x" % (obj + 0x1428 + i*8)))
+ if not p:
+ continue
+ tgt = p + n * 0x10
+ try:
+ v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe)))
+ if v16 == 0xffff:
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xe))
+ gr("set {unsigned int}0x%x = 0" % tgt)
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xc))
+ state["cleared"] += 1
+ if state["cleared"] <= 12:
+ print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt))
+ except Exception as e:
+ if state["hits"] <= 6:
+ print("E%d fail %s" % (i, e))
+ except Exception as e:
+ if state["hits"] <= 6:
+ print("hitfail %s" % e)
+ return state["hits"] >= 4000
+try:
+ B(); print("BP-SET")
+ gr("continue")
+except Exception as e:
+ print("bpfail %s" % e)
+print("T94-DONE hits=%d cleared=%d" % (state["hits"], state["cleared"]))
diff --git a/tools/guest-diag/wall2/t95.py b/tools/guest-diag/wall2/t95.py
new file mode 100644
index 0000000..8daf6ca
--- /dev/null
+++ b/tools/guest-diag/wall2/t95.py
@@ -0,0 +1,19 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x" % (base or 0))
+print("--- wait site +0xA12D90..:")
+print(gr("x/60i 0x%x" % (base + 0xA12D90), to_string=True))
+print("--- caller +0xD690D0..:")
+print(gr("x/40i 0x%x" % (base + 0xD690D0), to_string=True))
+print("T95-DONE")
diff --git a/tools/guest-diag/wall2/t96b.py b/tools/guest-diag/wall2/t96b.py
new file mode 100644
index 0000000..cd3b7ab
--- /dev/null
+++ b/tools/guest-diag/wall2/t96b.py
@@ -0,0 +1,40 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x" % (base or 0))
+class B(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xA12DDF), internal=False)
+ self.silent = True; self.done = False
+ def stop(self):
+ if self.done:
+ return True
+ self.done = True
+ try:
+ rdi = int(gdb.parse_and_eval("$rdi"))
+ rsi = int(gdb.parse_and_eval("$rsi"))
+ rdx = int(gdb.parse_and_eval("$rdx"))
+ print("POLL nfds=%d timeout=%d arr=0x%x" % (rsi, rdx, rdi))
+ fd = int(gdb.parse_and_eval("*(int*)0x%x" % rdi))
+ ev = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (rdi + 4)))
+ rv = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (rdi + 6)))
+ print("FDPOLL fd=%d events=0x%x revents=0x%x" % (fd, ev, rv))
+ except Exception as e:
+ print("fail %s" % e)
+ return True
+try:
+ B(); print("BP-SET")
+ gr("continue")
+except Exception as e:
+ print("bpfail %s" % e)
+print("T96-DONE")
diff --git a/tools/guest-diag/wall2/t97.py b/tools/guest-diag/wall2/t97.py
new file mode 100644
index 0000000..5a12348
--- /dev/null
+++ b/tools/guest-diag/wall2/t97.py
@@ -0,0 +1,22 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x" % (base or 0))
+C = base + 0x2070b7c
+v = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % C))
+print("COUNT %u" % v)
+print(gr("x/8wx 0x%x" % (C - 0x10), to_string=True))
+if v:
+ gr("set {unsigned int}0x%x = 0" % C)
+ print("WROTE 0")
+print("T97-DONE")
diff --git a/tools/guest-diag/wall2/t99.py b/tools/guest-diag/wall2/t99.py
new file mode 100644
index 0000000..ed665fb
--- /dev/null
+++ b/tools/guest-diag/wall2/t99.py
@@ -0,0 +1,79 @@
+import gdb
+gr = gdb.execute
+inf = gdb.selected_inferior()
+pid = inf.pid
+base = None
+for line in open("/proc/%d/maps" % pid):
+ parts = line.split()
+ try:
+ lo, hi = [int(x,16) for x in parts[0].split('-')]
+ except Exception:
+ continue
+ if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0:
+ base = lo
+print("BASE 0x%x PID %d" % (base or 0, pid))
+def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a))
+def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a))
+state = {"chits":0, "cleared":0, "whits":0, "wrote":0}
+class BC(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xB33D30), internal=False)
+ self.silent = True
+ def stop(self):
+ state["chits"] += 1
+ try:
+ obj = int(gdb.parse_and_eval("$rdi"))
+ n = int(gdb.parse_and_eval("$rsi"))
+ cnt = u32(obj + 0x22efc)
+ for i in range(0, min(cnt, 32)):
+ p = u64(obj + 0x1428 + i*8)
+ if not p:
+ continue
+ tgt = p + n * 0x10
+ try:
+ v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe)))
+ if v16 == 0xffff:
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xe))
+ gr("set {unsigned int}0x%x = 0" % tgt)
+ gr("set {unsigned short}0x%x = 0" % (tgt + 0xc))
+ state["cleared"] += 1
+ if state["cleared"] <= 10:
+ print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt))
+ except Exception:
+ pass
+ except Exception as e:
+ if state["chits"] <= 4:
+ print("chitfail %s" % e)
+ return state["chits"] >= 6000
+class BW(gdb.Breakpoint):
+ def __init__(self):
+ super().__init__("*0x%x" % (base + 0xDCEBD9), internal=False)
+ self.silent = True
+ def stop(self):
+ state["whits"] += 1
+ try:
+ rbx = int(gdb.parse_and_eval("$rbx"))
+ sub = u64(rbx + 0x70)
+ obj2 = u64(sub + 0x58)
+ off = u32(obj2 + 0x60)
+ holder = u64(obj2 + 0x58)
+ base2 = u64(holder + 0x58)
+ word = base2 + off
+ val = u32(word)
+ if state["whits"] <= 6 or val == 0:
+ print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val))
+ if val == 0:
+ gr("set {unsigned int}0x%x = 1" % word)
+ state["wrote"] += 1
+ if state["wrote"] <= 10:
+ print("WORDWROTE 0x%x" % word)
+ except Exception as e:
+ if state["whits"] <= 4:
+ print("whitfail %s" % e)
+ return False
+try:
+ BC(); BW(); print("BPS-SET")
+ gr("continue")
+except Exception as e:
+ print("bpfail %s" % e)
+print("T99-DONE chits=%d cleared=%d whits=%d wrote=%d" % (state["chits"], state["cleared"], state["whits"], state["wrote"]))
diff --git a/tools/guest-diag/wall2/unwait.py b/tools/guest-diag/wall2/unwait.py
new file mode 100644
index 0000000..94e677c
--- /dev/null
+++ b/tools/guest-diag/wall2/unwait.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# unwait.py - guest-side, zero-ptrace wait-word poller: writes 1 to the +0x80
+# u32 of every zero 16 KB rw-s nvidiactl mapping of the newest vk_frame.
+import os, re, subprocess, sys, time
+DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600
+def newest_pid():
+ try:
+ out = subprocess.run(["pgrep", "-n", "-x", "vk_frame"], capture_output=True).stdout
+ return int(out.decode().strip())
+ except Exception:
+ return None
+def maps(pid):
+ try:
+ return open("/proc/%d/maps" % pid).read()
+ except Exception:
+ return ""
+logged = set()
+end = time.time() + DUR
+while time.time() < end:
+ pid = newest_pid()
+ if pid:
+ for line in maps(pid).splitlines():
+ p = line.split()
+ if len(p) < 6 or "nvidia" not in p[-1] or "rw-s" not in p[1]:
+ continue
+ try:
+ lo, hi = [int(x, 16) for x in p[0].split("-")]
+ except Exception:
+ continue
+ if hi - lo != 0x4000:
+ continue
+ wa = lo + 0x80
+ try:
+ fd = os.open("/proc/%d/mem" % pid, os.O_RDWR)
+ os.lseek(fd, wa, 0)
+ v = os.read(fd, 4)
+ if len(v) == 4 and v == b"\x00\x00\x00\x00":
+ os.lseek(fd, wa, 0)
+ os.write(fd, b"\x01\x00\x00\x00")
+ key = (pid, wa)
+ if key not in logged:
+ logged.add(key)
+ print("WORDWROTE pid=%d va=0x%x" % (pid, wa), flush=True)
+ os.close(fd)
+ except Exception:
+ pass
+ time.sleep(0.4)
+print("UNWAIT-DONE", flush=True)
diff --git a/tools/guest-diag/wall2/unwait2.py b/tools/guest-diag/wall2/unwait2.py
new file mode 100644
index 0000000..cd4169b
--- /dev/null
+++ b/tools/guest-diag/wall2/unwait2.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# unwait2.py - no-ptrace guest-side harness:
+# (a) writes 1 to any zero u32 at +0x80 of every 0x4000-byte rw-s nvidia
+# mapping of EVERY live vk_frame;
+# (b) scans each live vk_frame's anonymous/heap rw-p mappings (16-byte
+# aligned) for marker slots {u32@0==0, u16@+0xe==0xffff} and clears
+# u16@+0xe to 0 (the instance-phase pending markers the t105 gdb used
+# to clear). No ptrace anywhere.
+import os, subprocess, sys, time
+DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600
+def pids():
+ try:
+ out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout
+ return [int(x) for x in out.split()]
+ except Exception:
+ return []
+def rd(fd, off, n):
+ try:
+ os.lseek(fd, off, 0)
+ return os.read(fd, n)
+ except Exception:
+ return b""
+def wr(fd, off, data):
+ try:
+ os.lseek(fd, off, 0)
+ os.write(fd, data)
+ return True
+ except Exception:
+ return False
+wlog = set(); mlog = set()
+end = time.time() + DUR
+last_scan = 0.0
+while time.time() < end:
+ for pid in pids():
+ try:
+ fd = os.open("/proc/%d/mem" % pid, os.O_RDWR)
+ except Exception:
+ continue
+ try:
+ for line in open("/proc/%d/maps" % pid):
+ p = line.split()
+ if len(p) < 6 or "nvidia" not in p[-1] or "rw-s" not in p[1]:
+ continue
+ lo, hi = [int(x, 16) for x in p[0].split("-")]
+ if hi - lo != 0x4000:
+ continue
+ wa = lo + 0x80
+ v = rd(fd, wa, 4)
+ if len(v) == 4 and v == b"\x00\x00\x00\x00":
+ if wr(fd, wa, b"\x01\x00\x00\x00"):
+ k = (pid, wa)
+ if k not in wlog:
+ wlog.add(k)
+ print("WORDWROTE pid=%d va=0x%x" % (pid, wa), flush=True)
+ except Exception:
+ pass
+ if time.time() - last_scan > 1.5:
+ budget = 64
+ try:
+ for line in open("/proc/%d/maps" % pid):
+ p = line.split()
+ if len(p) < 6 or p[1] != "rw-p":
+ continue
+ path = p[-1]
+ if path.startswith("/") and "nvidia" not in path and path != "[heap]":
+ continue
+ lo, hi = [int(x, 16) for x in p[0].split("-")]
+ if hi - lo > 0x10000000:
+ hi = lo + 0x10000000
+ off = lo
+ while off < hi:
+ chunk = rd(fd, off, 0x100000)
+ if len(chunk) < 32:
+ break
+ j = 0
+ while j + 16 <= len(chunk):
+ if chunk[j:j+4] == b"\x00\x00\x00\x00" and chunk[j+0xe:j+0x10] == b"\xff\xff":
+ wa = off + j + 0xe
+ if (pid, wa) not in mlog:
+ if wr(fd, wa, b"\x00\x00"):
+ mlog.add((pid, wa))
+ print("MARKCLR pid=%d va=0x%x" % (pid, wa), flush=True)
+ j += 16
+ off += 0x100000
+ budget -= 1
+ if budget <= 0:
+ break
+ if budget <= 0:
+ break
+ except Exception:
+ pass
+ last_scan = time.time()
+ os.close(fd)
+ time.sleep(0.4)
+print("UNWAIT2-DONE", flush=True)
diff --git a/tools/vk/vk_frame.c b/tools/vk/vk_frame.c
index 8b864ff..62548be 100644
--- a/tools/vk/vk_frame.c
+++ b/tools/vk/vk_frame.c
@@ -27,6 +27,8 @@
#define FRAME_OFFSET 0x100000UL
#define HDR_MAGIC 0x4b564d46u
+static VkImageTiling g_tiling = VK_IMAGE_TILING_OPTIMAL;
+
struct frame_hdr {
uint32_t magic;
uint32_t version;
@@ -59,6 +61,11 @@ static uint64_t now_ns(void)
int main(int argc, char **argv)
{
+ if (argc > 7 && atoi(argv[7]) == 1) {
+ g_tiling = VK_IMAGE_TILING_LINEAR;
+ printf("TILING linear\n");
+ }
+
int frames = argc > 1 ? atoi(argv[1]) : 60;
uint32_t W = argc > 2 ? (uint32_t)atoi(argv[2]) : 640;
uint32_t H = argc > 3 ? (uint32_t)atoi(argv[3]) : 480;
@@ -179,7 +186,7 @@ int main(int argc, char **argv)
ci.mipLevels = 1;
ci.arrayLayers = 1;
ci.samples = VK_SAMPLE_COUNT_1_BIT;
- ci.tiling = VK_IMAGE_TILING_OPTIMAL;
+ ci.tiling = g_tiling;
ci.usage = VK_IMAGE_USAGE_STORAGE_BIT |
VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
ci.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
@@ -244,7 +251,25 @@ int main(int argc, char **argv)
break;
}
}
- VKC(vkAllocateMemory(dev, &ma, NULL, &buf_mem));
+ {
+ int rc_ = vkAllocateMemory(dev, &ma, NULL, &buf_mem);
+ if (rc_ != VK_SUCCESS) {
+ VkPhysicalDeviceMemoryProperties mp2;
+ uint32_t k2;
+ vkGetPhysicalDeviceMemoryProperties(phys, &mp2);
+ for (k2 = 0; k2 < mp2.memoryTypeCount && rc_ != VK_SUCCESS; k2++) {
+ if (!(mr.memoryTypeBits & (1u << k2)))
+ continue;
+ ma.memoryTypeIndex = k2;
+ rc_ = vkAllocateMemory(dev, &ma, NULL, &buf_mem);
+ printf("BUF retry type=%u rc=%d\n", k2, rc_);
+ }
+ }
+ if (rc_ != VK_SUCCESS) {
+ printf("VKFAIL vkAllocateMemory(buf) -> %d (line %d)\n", rc_, __LINE__);
+ return 1;
+ }
+ }
VKC(vkBindBufferMemory(dev, buf, buf_mem, 0));
}
/* shader + pipeline */
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-25 21:14:34 +0000