diff options
74 files changed, 6045 insertions, 21 deletions
diff --git a/docs/04-implementation/04-real-render-walls.md b/docs/04-implementation/04-real-render-walls.md index 7792a2d..f6ea39d 100644 --- a/docs/04-implementation/04-real-render-walls.md +++ b/docs/04-implementation/04-real-render-walls.md @@ -151,4 +151,37 @@ display-engine MMIO/RPC backend the guest expects from its host (the equivalent of NVIDIA's vGPU display plugin), or the specific vGPU configuration that makes the kernel accept the display channel class. Until (b) exists the guest's NVIDIA display path is closed by design, and -GZDOOM-class workloads cannot render on the guest GPU.
\ No newline at end of file +GZDOOM-class workloads cannot render on the guest GPU. +## 7. UPDATE 2026-10-25 #2 (SESSION 66): the 0x56 pinned to the RS alloc path + +A new instrument closed the gap: the guest's `nvidia.ko` (DKMS build) is +**not stripped** - copy it out (`/lib/modules/<kver>/updates/dkms/nvidia.ko`), +`objdump -t`/`-dr` it, compute the module's `.text` runtime base from any +`/proc/kallsyms` symbol, and every guest-kernel function/offset becomes +addressable for live gdb breakpoints on the QEMU stub. With it: + +* `rpcAllocObject_v29_06`/`_rpcAllocObjectPrologue` are **exonerated** for + 0xc57e (their reject list is 0x73/0xc67*/0xc770/0xc77d only; the register + mapping `hClass = r8d` was verified live). +* The failing chain is pinned exactly: `rmapiAllocWithSecInfo` **enters** + with class 0xc57e, calls `serverAllocResourceUnderLock`, and that + function returns **0x56** - *before* the GSP-RM alloc RPC + (`rpcRmApiAlloc_GSP`, called from under-lock+0x447; the emulation never + sees an ALLOC for 0xc57e). The same probe shows 0xc57d/0xc570/0xc573 + return 0. +* So the last blocker for the window path is a **local RS-layer validation + inside `serverAllocResourceUnderLock`** (candidates: the inlined + privilege/class-support checks, `rmapiFixupAllocParams`' per-class fixup, + or `clientAllocResource`'s construct dispatch). The 0x56-producing + instruction is not yet pinned - a `stepi` loop over the QEMU stub is far + too slow and wedged the guest (recovered by a fresh cycle); the next + session should use conditional breakpoints on the function's call-return + sites (the approach in `gdbdiag19` works; the earlier run had truncated + addresses and needs the `0xffffffff00000000` base restored) or a + watchpoint on the status word. + +**gdb-stub field notes**: the stub is single-client; killing an attached +gdb leaves the VM stopped (recover with a detach-only connect); +`stepi` loops over the stub freeze the guest's userspace for minutes - +prefer breakpoints; every probe run with the caps-fill breakpoint changes +the driver's path, so keep the fill in each experiment. diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index ec6ca20..1855908 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -1913,7 +1913,20 @@ static void gsp_rm_handle_bus_get_pci_bar_info(struct gsp_emu_context *ctx, * 36 bytes (observed paramsSize): feHwSysCap(0) windowPresentMask(4) * bFbRemapperEnabled(8) numHeads(12) i2cPort(16) internalDispActiveMask(20) * embeddedDisplayPortMask(24) muxSupported(28) numDispChannels(32). - * Report a minimal headless display: 1 head, 1 (core) channel. + * + * r651: this reply is also the gate for the WINDOW-channel alloc path. + * Live gdb on the guest nvidia.ko pinned the display path 0x56 to + * kdispGetChannelNum_v03_00: for dispChnClass_Win it requires + * pStaticInfo->windowPresentMask & NVBIT32(instance) and returns + * NV_ERR_NOT_SUPPORTED (0x56) when the bit is clear; the Curs class reads + * HEAD_EXISTS bits from feHwSysCap the same way (0x49+instance channel num). + * The previous reply left both masks zero, so nvRmApiAlloc(0xc57e) died in + * the guest's local resource construct (clientAllocResource -> + * resservResourceFactory -> dispchnConstruct -> kdispGetChannelNum) before + * any GSP-RM RPC was issued. Report the same one-head / 8-window model the + * NV0073 shim and the caps notifier expose (feHwSysCap head-0 exists, + * windowPresentMask bits 0..7, numDispChannels covering the cursor channel + * numbers 73..80, i.e. 81 entries). */ static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx, u32 sequence, @@ -1925,15 +1938,205 @@ static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx, memset(out_params, 0, 36); + /* + * feHwSysCap: NV_PDISP_FE_HW_SYS_CAP_HEAD_EXISTS(i) occupies bit i + * (dev_disp.h v03_00); head 0 exists on the emulated board. + */ + val32 = 0x1; + memcpy(out_params + 0, &val32, 4); + + /* + * windowPresentMask: bit i = window instance i present. Mirrors the + * caps notifier's SYS_CAPB=0xff and the NV0073 window assignment + * (windows 0/1 flexible) - eight usable windows. + */ + val32 = 0xff; + memcpy(out_params + 4, &val32, 4); + val32 = 1; /* numHeads */ memcpy(out_params + 12, &val32, 4); - val32 = 1; /* numDispChannels (core channel) */ + + /* + * numDispChannels sizes pClientChannelTable[], indexed by the packed + * channel number kdispGetChannelNum returns: CORE=0, WIN(i)=1+i, + * WINIM(i)=33+i, CURS(i)=73+i. Cover CURS(7)=80 -> 81 entries so any + * instance the guest allocates stays in bounds. + */ + val32 = 81; memcpy(out_params + 32, &val32, 4); *out_params_size = 36; *out_status = 0; - nv_vgpu_info(ctx->vgpu, "vgsp-rm: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=1"); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=81 winMask=0xff feHwSysCap=0x1"); +} + +/* + * INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES (0x20800a5d) + * + * dispswConstruct_IMPL (disp_sw.c) asks for {displayMask, numHeads} and + * rejects every logicalHeadId >= numHeads with "invalid logical head + * number: N" + NV_ERR_INSUFFICIENT_RESOURCES; the generic all-zero reply + * made numHeads = 0, so nvkms' DISPLAY_SW objects could not be constructed + * on any head (observed r653: heads 0..3 all rejected -> vkCreateDevice -3). + * Report the same one-head model as DISPLAY_GET_STATIC_INFO: display id 0 + * active, one head. + * + * Params ABI (NV2080_CTRL_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES_PARAMS, + * 8 bytes): displayMask(0) numHeads(4). + */ +static void gsp_rm_handle_display_active_devices(struct gsp_emu_context *ctx, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 displayMask = 0x1; /* display id 0 */ + u32 numHeads = 1; + + memcpy(out_params + 0, &displayMask, 4); + memcpy(out_params + 4, &numHeads, 4); + + *out_params_size = 8; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DISPLAY_GET_ACTIVE_DISPLAY_DEVICES -> displayMask=0x1 numHeads=1"); +} + +/* + * NVC370_CTRL_CMD_GET_CHANNEL_INFO (0xc3700104) + * + * nvRMSyncEvoChannel() (nvkms-rm.c) polls this control through the display + * object after every EVO channel kickoff and waits up to 2 s for the channel + * to report NVC370_CTRL_GET_CHANNEL_INFO_STATE_IDLE, otherwise it logs + * "Idling display engine timed out: <class>:<instance>:<sd>:<token>" and + * fails the whole modeset init (observed on the r652 window path: 0xc57d and + * 0xc57e instances 0/1). The emulated display engine has no pushbuffer + * consumer, so the state would never leave BUSY; report IDLE immediately. + * + * Params ABI (NVC370_CTRL_CMD_GET_CHANNEL_INFO_PARAMS, 20 bytes): + * subdeviceIndex(0) channelClass(4) channelInstance(8) + * IsChannelInDebugMode(12) channelState(16 = NVBIT(0) IDLE). + * Echo the request so class/instance round-trip and set the state word. + */ +static void gsp_rm_handle_display_channel_info(struct gsp_emu_context *ctx, + const u8 *in_params, u32 in_size, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 idle = 1; /* NVC370_CTRL_GET_CHANNEL_INFO_STATE_IDLE */ + u32 class_id = 0, instance = 0; + u32 size = in_size; + + if (size > 64) + size = 64; /* defensive cap; the real params are 20 bytes */ + if (size >= 20) { + memcpy(out_params, in_params, size); + memcpy(&class_id, in_params + 4, 4); + memcpy(&instance, in_params + 8, 4); + memcpy(out_params + 16, &idle, 4); + } else { + memset(out_params, 0, 20); + memcpy(out_params + 16, &idle, 4); + size = 20; + } + + *out_params_size = size; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: GET_CHANNEL_INFO class=0x%x instance=%u -> IDLE", + class_id, instance); +} + +/* + * NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES (0x801707) + * + * The Vulkan/GL client queries the device for the size and alignment of an + * engine's context save region while building its GR channel context + * (observed: engineId 0x8 = GRAPHICS_ZCULL, one query at the end of every + * vkCreateDevice attempt). The generic all-zero reply handed back size = 0; + * the client's context-buffer allocation then failed (memdesc of size 0) + * and device creation aborted with VK_ERROR_INITIALIZATION_FAILED + * (observed r661: four create attempts, each ending at this control, then + * teardown and VKFAIL -3). + * + * The sizes below mirror the real 570.144 GSP-RM geometry captured on this + * TU104 (rm-call golden runs); entries marked "fill" were not covered by + * the capture and use nonzero page-aligned placeholders - revisit them when + * a real capture for the remaining engine ids exists. + * + * Params ABI (NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS, + * 12 bytes): engineId(0, in) alignment(4, out) size(8, out). + */ +static const struct gsp_rm_engine_ctx_geom { + u32 engine_id; + u32 alignment; + u32 size; +} gsp_rm_engine_ctx_geoms[] = { + { 0x00, 0x1000, 0xf2d00 }, /* GRAPHICS (captured) */ + { 0x08, 0x1000, 0x10000 }, /* GRAPHICS_ZCULL (fill) */ + { 0x09, 0x1000, 0xd100 }, /* GRAPHICS_PM (captured) */ + { 0x0a, 0x1000, 0x1000 }, /* COMPUTE_PREEMPT (fill) */ + { 0x0b, 0x1000, 0x1000 }, /* GRAPHICS_PREEMPT (fill) */ + { 0x0c, 0x1000, 0x1000 }, /* GRAPHICS_SPILL (fill) */ + { 0x0d, 0x100, 0x20000 }, /* GRAPHICS_PAGEPOOL (fill) */ + { 0x0e, 0x100, 0x8000 }, /* GRAPHICS_BETACB (fill) */ + { 0x0f, 0x100, 0x80000 }, /* GRAPHICS_RTV (fill) */ + { 0x10, 0x1000, 0x5000 }, /* GRAPHICS_PATCH (captured) */ + { 0x11, 0x100, 0x3000 }, /* GRAPHICS_BUNDLE_CB (captured) */ + { 0x12, 0x100, 0x20000 }, /* GRAPHICS_PAGEPOOL_GLOBAL (captured) */ + { 0x13, 0x1000, 0x7d7400 }, /* GRAPHICS_ATTRIBUTE_CB (captured) */ + { 0x14, 0x100, 0x80000 }, /* GRAPHICS_RTV_CB_GLOBAL (captured) */ + { 0x15, 0x100, 0x20000 }, /* GRAPHICS_GFXP_POOL (fill) */ + { 0x16, 0x100, 0x1000 }, /* GRAPHICS_GFXP_CTRL_BLK (fill) */ + { 0x17, 0x1000, 0x10000 }, /* GRAPHICS_FECS_EVENT (captured) */ + { 0x18, 0x1000, 0x80000 }, /* GRAPHICS_PRIV_ACCESS_MAP (captured) */ + { 0x19, 0x100, 0x1000 }, /* GRAPHICS_SETUP (fill) */ +}; + +static void gsp_rm_handle_fifo_engine_ctx_props(struct gsp_emu_context *ctx, + const u8 *in_params, u32 in_size, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 engine_id = ~0u; + u32 alignment = 0x1000, size = 0x1000; + u32 i; + bool found = false; + + if (in_size >= 4) + memcpy(&engine_id, in_params + 0, 4); + + for (i = 0; i < ARRAY_SIZE(gsp_rm_engine_ctx_geoms); i++) { + if (gsp_rm_engine_ctx_geoms[i].engine_id == engine_id) { + alignment = gsp_rm_engine_ctx_geoms[i].alignment; + size = gsp_rm_engine_ctx_geoms[i].size; + found = true; + break; + } + } + + if (engine_id > 0x19) { + /* Same rejection the real RM's VF handler uses for ids + * outside the engine context properties range. */ + *out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */ + *out_params_size = 0; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: FIFO_GET_ENGINE_CONTEXT_PROPERTIES engineId=0x%x -> NOT_SUPPORTED", + engine_id); + return; + } + + memcpy(out_params + 0, &engine_id, 4); + memcpy(out_params + 4, &alignment, 4); + memcpy(out_params + 8, &size, 4); + *out_params_size = 12; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: FIFO_GET_ENGINE_CONTEXT_PROPERTIES engineId=0x%x -> alignment=0x%x size=0x%x%s", + engine_id, alignment, size, found ? "" : " (default)"); } /* @@ -2956,11 +3159,22 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, gsp_rm_register_channel(ctx, ho, params, params_size); /* - * NV01_EVENT (allocated with class 0x79 in the guest ABI; params are - * NV0005_ALLOC_PARAMETERS {hParentClient, hSrcResource, hClass, - * notifyIndex, data}). The driver allocates one per channel and waits - * for the GPU to post the event before it considers submitted work - * complete - remember them so completed work can be signalled back. + * NV01_EVENT (params are NV0005_ALLOC_PARAMETERS {hParentClient, + * hSrcResource, hClass, notifyIndex, data}). The driver allocates one + * per channel and waits for the GPU to post the event before it + * considers submitted work complete - remember them so completed work + * can be signalled back. + * + * r682: the guest ABI uses THREE event classes and only the + * NV01_EVENT_OS_EVENT (0x79) shape was registered: libcuda's channel + * events arrive as 0x79, but the modern nvkms/Vulkan path allocates + * NV01_EVENT_KERNEL_CALLBACK_EX (0x7E, nvos.h) - observed live as + * `GSP_RM_ALLOC class=0x7e paramsSize=24 hParent=<subdevice> + * hObject=0xcaf00003` from the second RM client while its main + * thread sat in the vkCreateDevice wait loop (readiness word in a + * /dev/nvidiactl notification page, never signalled). Register 0x78 + * (KERNEL_CALLBACK) and 0x7E alongside 0x79 so the completion + * wake-ups reach those events too. * * The table is append+retire: FREE retires records (valid=false) but * never shrinks events_n, so long-lived boots with many client @@ -2973,7 +3187,8 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, * first retired slot for a new registration, and log loudly when the * table is genuinely full of live entries. */ - if (class_id == 0x79 && params_size >= 24) { + if ((class_id == 0x79 || class_id == 0x7e || class_id == 0x78) && + params_size >= 24) { u32 h_src = 0, notify_index = 0, hclass = 0; u64 data64 = 0; int hc_off = -1, i; @@ -2991,16 +3206,17 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, rm->events_n, ho); else { /* - * Locate hClass (0x79) inside the params instead of - * trusting a fixed offset: NV0005_ALLOC_PARAMETERS is - * {hParentClient, hSrcResource, hClass, notifyIndex, - * data}, with hSrcResource immediately before hClass. + * Locate hClass (0x78/0x79/0x7e) inside the params + * instead of trusting a fixed offset: + * NV0005_ALLOC_PARAMETERS is {hParentClient, + * hSrcResource, hClass, notifyIndex, data}, with + * hSrcResource immediately before hClass. */ for (i = 0; i + 4 <= (int)params_size; i += 4) { u32 v; memcpy(&v, params + i, 4); - if (v == 0x79) { + if (v == 0x79 || v == 0x7e || v == 0x78) { hc_off = i; break; } @@ -3237,6 +3453,94 @@ MODULE_PARM_DESC(gsp_rm_binapi07_fwd, "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)"); /* + * session 75: reply words for the client-pending control 0x2080a097 (the + * libnvidia-eglcore engine-readiness poll). Runtime-tunable so one boot can + * sweep reply variants: the client reads +0x204, +0x488 and +0x48c back + * (file offsets 0xecd45f..0xecd475) before its engine loop re-issues the + * control. Defaults: flag=1 (the request sets it to 0), the other words 0. + */ +static uint gsp_rm_a097_flag = 1; +module_param_named(wall2_a097_flag, gsp_rm_a097_flag, uint, 0644); +MODULE_PARM_DESC(wall2_a097_flag, "0x2080a097 reply word at params+0x204 (default 1)"); +static uint gsp_rm_a097_v488; +module_param_named(wall2_a097_v488, gsp_rm_a097_v488, uint, 0644); +MODULE_PARM_DESC(wall2_a097_v488, "0x2080a097 reply word at params+0x488 (default 0)"); +static uint gsp_rm_a097_v48c; +module_param_named(wall2_a097_v48c, gsp_rm_a097_v48c, uint, 0644); +MODULE_PARM_DESC(wall2_a097_v48c, "0x2080a097 reply word at params+0x48c (default 0)"); + +/* + * session 76: the 0x2080a097 reply's RECORD area (params+4..+0x23). The + * real host RM handler overwrites up to four {u32, u32} mode pairs there + * (per-mode engine status + a x1000 clock/percent value); the client + * (count=1) copies back the first 16 bytes into its per-engine query + * buffer, and the second word of the first pair is consumed as the + * "engine progress" word (libnvidia-eglcore d5e3e0: *(inner+4) -> + * session+0xe104, compared against per-boot config thresholds). + * Each word defaults to 0xffffffff = leave the echoed request word + * untouched; any other value replaces it, so one boot can sweep the + * record matrix without rebuilding. + */ +static uint gsp_rm_a097_count = ~0u; +module_param_named(wall2_a097_count, gsp_rm_a097_count, uint, 0644); +MODULE_PARM_DESC(wall2_a097_count, "0x2080a097 reply count word at params+0 (0xffffffff = echo)"); +static uint gsp_rm_a097_rec0 = ~0u, gsp_rm_a097_rec1 = ~0u; +static uint gsp_rm_a097_rec2 = ~0u, gsp_rm_a097_rec3 = ~0u; +static uint gsp_rm_a097_rec4 = ~0u, gsp_rm_a097_rec5 = ~0u; +static uint gsp_rm_a097_rec6 = ~0u, gsp_rm_a097_rec7 = ~0u; +module_param_named(wall2_a097_rec0, gsp_rm_a097_rec0, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec0, "0x2080a097 reply word at params+0x4 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec1, gsp_rm_a097_rec1, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec1, "0x2080a097 reply word at params+0x8 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec2, gsp_rm_a097_rec2, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec2, "0x2080a097 reply word at params+0xc (0xffffffff = echo)"); +module_param_named(wall2_a097_rec3, gsp_rm_a097_rec3, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec3, "0x2080a097 reply word at params+0x10 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec4, gsp_rm_a097_rec4, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec4, "0x2080a097 reply word at params+0x14 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec5, gsp_rm_a097_rec5, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec5, "0x2080a097 reply word at params+0x18 (0xffffffff = echo)"); +module_param_named(wall2_a097_rec6, gsp_rm_a097_rec6, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec6, "0x2080a097 reply word at params+0x1c (0xffffffff = echo)"); +module_param_named(wall2_a097_rec7, gsp_rm_a097_rec7, uint, 0644); +MODULE_PARM_DESC(wall2_a097_rec7, "0x2080a097 reply word at params+0x20 (0xffffffff = echo)"); + +/* + * m112b: the NV04_VID_HEAP_CONTROL enumeration pair 0x90960106/0x90960107 + * (legacy NVOS32 VidHeap API, class 0x9096). libnvidia-eglcore (file offset + * 0x586d80) walks three heaps: per heap it queries 0x90960106 (12 bytes: + * {u32 firstIdx, u32 lastIdx}; the client then enumerates the heap's memory + * blocks by index via 0x90960107 (56 bytes) while firstIdx+1 <= idx <= + * lastIdx). The 56-byte block entry is read at +0x00/+0x08 (two u64s), +0x28 + * (a u32 type compared against a table at eglcore 0x101b1c0) and +0x30 (a + * flag BYTE: when 0 the heap/block is skipped). The generic zero reply makes + * every heap look empty, the client concludes out-of-device-memory and + * vkAllocateMemory fails -2 WITHOUT any alloc reaching the emulation + * (dmesg-corroborated: only 0x9096* controls + a class 0xc5c0 alloc). + * Runtime-tunable so one boot can sweep reply variants (same pattern as the + * wall2_a097_* words). + */ +static uint gsp_rm_9096_lo; /* 0x90960106 firstIdx */ +static uint gsp_rm_9096_hi = 1; /* 0x90960106 lastIdx */ +module_param_named(wall2_9096_lo, gsp_rm_9096_lo, uint, 0644); +MODULE_PARM_DESC(wall2_9096_lo, "0x90960106 reply firstIdx (default 0)"); +module_param_named(wall2_9096_hi, gsp_rm_9096_hi, uint, 0644); +MODULE_PARM_DESC(wall2_9096_hi, "0x90960106 reply lastIdx (default 1)"); +static uint gsp_rm_9096_ent_addr; /* 0x90960107 entry +0x00 (u64) */ +module_param_named(wall2_9096_ent_addr, gsp_rm_9096_ent_addr, uint, 0644); +MODULE_PARM_DESC(wall2_9096_ent_addr, "0x90960107 entry word at +0x00 (default 0)"); +static uint gsp_rm_9096_ent_size = 0x80000000u; /* entry +0x08 (u64) */ +module_param_named(wall2_9096_ent_size, gsp_rm_9096_ent_size, uint, 0644); +MODULE_PARM_DESC(wall2_9096_ent_size, "0x90960107 entry word at +0x08 (default 0x80000000)"); +static uint gsp_rm_9096_ent_type = 3; /* entry +0x28 (u32) */ +module_param_named(wall2_9096_ent_type, gsp_rm_9096_ent_type, uint, 0644); +MODULE_PARM_DESC(wall2_9096_ent_type, "0x90960107 entry word at +0x28 (default 3)"); +static uint gsp_rm_9096_ent_flag = 1; /* entry +0x30 (u8) */ +module_param_named(wall2_9096_ent_flag, gsp_rm_9096_ent_flag, uint, 0644); +MODULE_PARM_DESC(wall2_9096_ent_flag, "0x90960107 entry flag byte at +0x30 (default 1)"); + + +/* * The live session's oracle client/object for the 0x20810107 experiment, * created on demand; 0 until the first query builds it. One device per module * load, so file-statics are enough; reset by unload. @@ -3777,6 +4081,26 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; + case 0xc3700104: /* NVC370_CTRL_CMD_GET_CHANNEL_INFO */ + gsp_rm_handle_display_channel_info(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case 0x20800a5d: /* NV2080_CTRL_CMD_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES */ + gsp_rm_handle_display_active_devices(ctx, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case 0x801707: /* NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES */ + gsp_rm_handle_fifo_engine_ctx_props(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + case 0x730102: /* NV0073_CTRL_CMD_SYSTEM_GET_NUM_HEADS */ case 0x730287: /* NV0073_CTRL_CMD_SPECIFIC_GET_ALL_HEAD_MASK */ case 0x7302ad: /* NV0073_CTRL_CMD_SPECIFIC_GET_VALID_HEAD_WINDOW_ASSIGNMENT */ @@ -3992,6 +4316,113 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; } + case 0x2080a097: { /* client-pending engine-context query (session 75) + * + * libnvidia-eglcore issues this via NVOS54 and then + * POLLS the reply's params words: the request sets + * +0x204 = 0, and the caller re-issues the whole + * control every ~1.8 s while its readiness check + * stays negative (observed: 32-36 polls against the + * generic all-zero reply, then the client gives up + * and vkCreateDevice returns -4). The params are an + * in/out record block: u32 count at +0, then count + * 16-byte records, then +0x204, and the client reads + * back +0, +0x204, +0x488 and +0x48c after the call + * (libnvidia-eglcore.so.570.144 file offsets + * 0xecd45f..0xecd475). The readiness wait itself + * lives in the client's engine loop + * (0xd6ef40..0xd6f0ab polling byte +0x1b038 of the + * engine object, which a91ea0 recomputes from many + * state fields), so the reply words below are + * runtime-tunable for a single-cycle sweep without + * rebuilding: the wall2_a097_* module parameters. + * + * Reply: verbatim echo (the request records are the + * client's own query data) with the tunable words + * applied. Defaults keep the completion-style flag. */ + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 0x490 && + req.paramsSize + GSP_RM_CTRL_HDR_SIZE <= + GSPRM_MAX_SINGLE_ELEM_DATA) { + u32 count = 0, flag; + static const u32 *const recp[8] = { + &gsp_rm_a097_rec0, &gsp_rm_a097_rec1, + &gsp_rm_a097_rec2, &gsp_rm_a097_rec3, + &gsp_rm_a097_rec4, &gsp_rm_a097_rec5, + &gsp_rm_a097_rec6, &gsp_rm_a097_rec7, + }; + int i; + + memcpy(rp, pp, req.paramsSize); + memcpy(&count, pp, 4); + memcpy(&flag, pp + 0x204, 4); + if (gsp_rm_a097_count != ~0u) + *(u32 *)(rp + 0) = gsp_rm_a097_count; + for (i = 0; i < 8; i++) { + if (*recp[i] != ~0u) + *(u32 *)(rp + 4 + i * 4) = *recp[i]; + } + *(u32 *)(rp + 0x204) = gsp_rm_a097_flag; + *(u32 *)(rp + 0x488) = gsp_rm_a097_v488; + *(u32 *)(rp + 0x48c) = gsp_rm_a097_v48c; + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: ctrl 0x2080a097: echoed %u bytes (count=%u, req flag=%u) -> flag=%u v488=0x%x v48c=0x%x", + req.paramsSize, count, flag, + gsp_rm_a097_flag, gsp_rm_a097_v488, + gsp_rm_a097_v48c); + out_status = 0; + out_params_size = req.paramsSize; + } else { + out_status = 0; + out_params_size = 0; + } + break; + } + + case 0x90960106: { /* NV04_VID_HEAP_CONTROL block-index range (m112b) */ + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 12) { + memcpy(rp, pp, req.paramsSize); + *(u32 *)(rp + 0) = gsp_rm_9096_lo; + *(u32 *)(rp + 4) = gsp_rm_9096_hi; + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: ctrl 0x90960106 -> first=%u last=%u", + gsp_rm_9096_lo, gsp_rm_9096_hi); + } + out_status = 0; + out_params_size = req.paramsSize; + break; + } + + case 0x90960107: { /* NV04_VID_HEAP_CONTROL block entry by index (m112b) */ + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 56) { + memcpy(rp, pp, req.paramsSize); + *(u64 *)(rp + 0x00) = (u64)gsp_rm_9096_ent_addr; + *(u64 *)(rp + 0x08) = (u64)gsp_rm_9096_ent_size; + *(u32 *)(rp + 0x28) = gsp_rm_9096_ent_type; + *(u8 *)(rp + 0x30) = (u8)gsp_rm_9096_ent_flag; + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: ctrl 0x90960107 -> addr=0x%x size=0x%x type=%u flag=%u", + gsp_rm_9096_ent_addr, + gsp_rm_9096_ent_size, + gsp_rm_9096_ent_type, + gsp_rm_9096_ent_flag); + } + out_status = 0; + out_params_size = req.paramsSize; + break; + } + default: /* Unknown control command - return success with zeroed params. * Do NOT echo the request: pure queries pass uninitialized @@ -4433,6 +4864,23 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) * For vGPU, we just acknowledge success. */ nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC UPDATE_BAR_PDE(%u) seq=%u len=%u -> SUCCESS", function, sequence, hdr->length); + { + const u8 *pl = (const u8 *)hdr + sizeof(*hdr); + u32 n = min_t(u32, hdr->length, 64); + char dump[256]; + int o = 0; + u32 i; + + for (i = 0; i < n; i += 4) { + u32 v = 0; + + memcpy(&v, pl + i, 4); + o += snprintf(dump + o, sizeof(dump) - o, + "%08x ", v); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: bar-pde payload: %s", dump); + } if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: UPDATE_BAR_PDE reply send failed"); else @@ -4461,6 +4909,31 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)", gsp_rm_rpc_name(function), function, sequence, hdr->length, rm->unknown_rpcs); + /* + * Bounded payload dump for the first unknown RPCs: the vGPU + * guest maps its client VA spaces through some RPC; the + * mapping data (va pages / PAs / memdesc handles) must be + * visible here to be captured. Never echo it back (pure + * queries would eat their own stack garbage). + */ + if (rm->unknown_rpcs <= 96 && hdr->length) { + const u8 *pl = (const u8 *)hdr + sizeof(*hdr); + u32 n = min_t(u32, hdr->length, 64); + char dump[256]; + int o = 0; + u32 i; + + for (i = 0; i < n; i += 4) { + u32 v = 0; + + memcpy(&v, pl + i, 4); + o += snprintf(dump + o, sizeof(dump) - o, + "%08x ", v); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: unknown fn=%u seq=%u len=%u payload: %s", + function, sequence, hdr->length, dump); + } if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function); else diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b6fa436..b311c0d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -88,6 +88,20 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu) #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL +/* + * session 79: the GR/GL ucode init releases target the RM's LOW aperture + * (the same 0x4000000-0x4ffffff VA family as the push/ring windows): the + * live GL init push carries `2005c017 04923060 ... 02000001` (0x5c-form, + * va=0x4923060, payload 1, RELEASE) and `200426c0 00000000 04b19b10 + * 00000001 10000000` (REPORT_SEM group, va=0x4b19b10, payload 1, + * RELEASE) - both rejected by the sysmem-only range check, so the GR + * completion never applied and the NvOgl engine-init wait spun forever. + * Accept the low aperture for release decode (the pushes are only + * decoded from trusted ring reads, so the false-positive risk the + * sysmem bound guarded against does not apply here). + */ +#define SVC_LOW_REL_VA_LO 0x4000000ULL + /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL @@ -1572,12 +1586,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * channel; the stashed geometry survives, so the scrub path * does not depend on the channel-table identity at all. */ - if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + /* + * m112b: the CeUtils scrubber is the ONLY exactly-4096-entry + * channel (NUM_COPY_BLOCKS). The old `>= 4096` test also + * matched every client channel (this driver version: 32768 + * entries, not the 1024 the comment below assumed), so the + * single scrub slot bound to whichever client channel + * registered last (observed r701: the slot held h=0xbeee0102 + * userd=0x8f400/entries=32768 and the pass authored put=0 for + * the wrong channel while the real scrubber's payload stayed + * zero - the vkAllocateMemory -2 gate). + */ + if (desc->gpfifo_entries == 4096 && desc->gpfifo_va && desc->userd_pa) { struct svc_scrub *s = svc_scrub_get(desc->vgpu, true); if (!s) goto scrub_done; /* table full: max guests */ + pr_info("sema-svc: scrub stash bind vgpu=%px slot=%d h=0x%x entries=%u\n", + (void *)desc->vgpu, (int)(s - svc_scrubs), + desc->h_object, desc->gpfifo_entries); /* Re-seed the counting on a fresh scrubber object or a * new client session (m176: the counter/submitted state * is per-session; a recycled scrubber handle after a @@ -1602,6 +1630,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, s->userd_as); + } else if (desc->gpfifo_entries >= 4096 && + desc->gpfifo_entries != 32768) { + /* m112b: a non-32768 channel with >=4096 entries that + * fails the stash condition (missing VA or USERD PA) + * is a scrubber candidate the service cannot author + * for - log it so a silent no-authoring boot is + * diagnosable. */ + pr_info_ratelimited("sema-svc: scrubber-stash MISS h=0x%x entries=%u va=0x%llx userd_pa=0x%llx\n", + desc->h_object, desc->gpfifo_entries, + desc->gpfifo_va, desc->userd_pa); } scrub_done: pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", @@ -10760,7 +10798,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && - va >= SEMA_VA_LO && va < SEMA_VA_HI) { + (va >= SEMA_VA_LO || va >= SVC_LOW_REL_VA_LO) && + va < SEMA_VA_HI) { if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; @@ -10953,7 +10992,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; va = ((u64)st.rs_upper << 32) | st.rs_lower; - if (va < SEMA_VA_LO || va >= SEMA_VA_HI) + if ((va < SEMA_VA_LO && + va < SVC_LOW_REL_VA_LO) || + va >= SEMA_VA_HI) break; if (page_pa) svc_mark_hot(page_pa); @@ -26975,19 +27016,43 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) u32 put = 0; u64 delta; - if (!s || !s->userd_ok || !s->pa_ok || !s->entries) + if (!s) return; + if (!s->userd_ok || !s->pa_ok || !s->entries) { + /* m112b: silent-return diagnostic (bounded per boot). */ + if (s->log_budget < 8) { + s->log_budget++; + pr_info("sema-svc: scrub pass idle vgpu=%px slot=%d userd_ok=%d pa_ok=%d entries=%u gen=%u\n", + (void *)vgpu, (int)(s - svc_scrubs), + s->userd_ok, s->pa_ok, s->entries, + s->session_gen); + } + return; + } if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, &put, sizeof(put))) { /* Retry through the alternate aperture (as registration did). */ if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB, - &put, sizeof(put))) + &put, sizeof(put))) { + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass read-fail userd=0x%llx as=%u\n", + s->userd_pa, s->userd_as); + } return; + } } - if (put >= s->entries) + if (put >= s->entries) { + /* m112b: junk-translation diagnostic (bounded). */ + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass junk put=%u entries=%u userd=0x%llx\n", + put, s->entries, s->userd_pa); + } return; /* junk translation; retry next pass */ + } if (!s->seeded) { s->seeded = true; diff --git a/tools/guest-diag/wall2/README-session74.md b/tools/guest-diag/wall2/README-session74.md new file mode 100644 index 0000000..58587eb --- /dev/null +++ b/tools/guest-diag/wall2/README-session74.md @@ -0,0 +1,66 @@ +# WALL #2 / device-init completion — session 74 findings (r675) + +## The client's "11 s stalls" fully root-caused (this is NOT the channel-init release) +strace (ioctl-only) shows THREE exact 11.00 s gaps with NO ioctl in flight; a +full strace shows the client doing a pure `sched_yield` spin (~57K/s) + +`clock_gettime` (perf) inside **libnvidia-eglcore.so.570.144** — i.e. a +userspace timeout loop. gdb backtraces place the spin inside the Vulkan ICD's +EGL/GL init (`vkCreateInstance` -> libGLX_nvidia vk_icdNegotiate... -> +libEGL -> eglcore wait loop). + +## The wait target: completion slots in /dev/nvidiactl mappings +- The wait loop (eglcore file offsets ~0xb284b0 delay, ~0xb33d00 wait loop, + ~0xb1c750 slot check) polls a slot table on the client's heap object: + table at object+0x1428, count at object+0x22efc. +- Slot check: `esi` is offset by 0x43; for esi-0x43 > 4 it reads a **u16 at + slot+0xe** and treats `0xffff` as "empty/not complete". +- gdb sampling during the stall: 13164 checks with `u16@slot+0xe = 0xffff`, + 1 with `0` — the client spins until the flag changes. +- The slot pointer = a **/dev/nvidiactl rw-s mapping** (page-sized; base+0x20 + = slot). These are the client's mappings of RM memory (the channel + notifier pages: service logs `notif=0x11627f060` per channel). + +## Filling the slots (guest-side) WORKS mechanically but is not the -4 cause +- `slotfill.sh` / `slotfill2.sh`: while vk_frame runs, write 0 to + `base+0x2e` of every small /dev/nvidiactl rw-s mapping when it reads 0xffff. +- Result: the 11 s gaps vanish (5 fills within ~2 s; the client proceeds + immediately) — **but vkCreateDevice still fails -4**. So the 11 s waits are + not the failure; they are an init-side slowdown the client tolerates. + +## The next wall: the polled RM control 0x2080a097 +- With the slots filled, the client settles into **NVOS54 (ioctl nr=0x2a) + polling every ~1.25 s** (strace), and the host dmesg shows + `GSP_RM_CONTROL cmd=0x2080a097 hClient=0xc1d000bb hObject=0xbeef0004 + paramsSize=1168 flags=0x0` -> `unknown cmd=0x2080a097 -> generic SUCCESS` + (all-zero reply), 36 polls in 45 s. +- The NVOS54 struct (gdb dump): {hClient=0xc1d000c0, hObject=0xbeef0004, + cmd=0x2080a097, flags=0, params=ptr, paramsSize=0x490, status=ptr, ...}. +- The params buffer = 1168 bytes of 32-byte records with a `0x25` field at + +0x10 (dump in /tmp/opencode evidence; gdbparam.sh captures it live). +- The client polls until the control's status/reply signals completion; our + generic all-zero reply never does. **This is the wall that must be solved to + get vkCreateDevice past -4** (either a real reply for 0x2080a097 or making + the polled status field change). + +## Other session-74 results +- The GR ring-entry writer was NOT found by disassembly-guessing: nvidia.ko + `channelFillGpFifo` and nvidia-modeset `nvWriteGpEntry` never fire for this + channel's kick (gdb hbreaks); nvidia/modeset kprobes are `notrace`; QEMU gdb + hbreak works. +- `sema_push_raw` dumps (module param) captured the service's reads of the + push: entry0 decodes only 2 methods then zeros (`consumed=0x288/1432`), + entry2 starts `SET_OBJECT 0xc597` then zeros; **no release in the push** + (`have_rel=0` for all three entries) — the channel-init release theory for + these waits is dead. +- `find_ring_va.py` / `vmap_watch2.py`: the ring entries are NOT in any + vmallocinfo area during the run; the nv_vm_map_pages areas hold the CE + pushes; the GR push content is not found by content searches. + +## Files +- gdbpoll6.gdb, gdbu16.gdb, gdbfill.gdb, gdbwatch.gdb, gdbparam.sh, + gdbgap3.sh, gdbspin.sh — the gdb probes (slot check, u16 flag, fill, + NVOS54 param dump, backtraces). +- slotfill.sh / slotfill2.sh — the guest-side slot-fill watcher. +- stracevk.sh, fastrun.sh, fastrun2.sh — the strace/perf recipes. +- perfstall5.sh — perf sampling with map-relative offsets. +- find_ring_va.py, vmap_watch2.py — the vmap/kcore watchers. diff --git a/tools/guest-diag/wall2/README-session76.md b/tools/guest-diag/wall2/README-session76.md new file mode 100644 index 0000000..828b3b9 --- /dev/null +++ b/tools/guest-diag/wall2/README-session76.md @@ -0,0 +1,134 @@ +# session 76 notes (r680) — the 0x2080a097 engine-monitor re-examined + +## What the firmware says (ground truth for the a097 family) + +The shipped GSP-RM 570.144 image embeds the host-side control +descriptor table (file offset 0x41df8..0x42450, entry = 0x20 bytes +{u32 cmd, u32 params_size, u32 0x4049580, u32 0, u32 handler_ptr, +u32 0, u32 out_size, u32 0}). Entries seen: + + 0x2080a07f size 0x408 handler vaddr 0x551b2c0 + 0x2080a079 size 0x14804 + 0x2080a097 size 0x490 handler vaddr 0x55166f4 + 0x2080a0d1 size 0x7e8 + ... + +The real 0x2080a097 handler (0x55166f4) writes up to FOUR {u32,u32} +mode pairs into params+4..+0x23 (params[0] low byte = the request's +count flag): for i=1..4 it calls an engine-status helper +(0x5516334 / 0x55d7f04 / 0x55d7c1c / 0x55d7d60) with mode=i and emits +{pair} = {engine value, 1000 * u16(engine+0x1bc0)}; the loop stops at +the first helper error. So the real reply overwrites the client's +request record {1,0,0,0}. + +## What the client does with the reply + +libnvidia-eglcore's local handler (file offset 0xecd2e0, the userspace +implementation of the internal cmd 0x2080a040) copies back: + + result+0 = reply+0 (count) + result+0x10 = reply+0x204 (flag) + result+0x20 = reply+0x488 + result+0x24 = reply+0x48c + count*16 bytes from reply+4 -> *(result+8) + +For the engine tick (file offset 0xd5e3e0, called from the wait loop +0xd6ef40 per iteration) the result struct IS the caller's request at +rsp+0x10, and *(result+8) points at a 16-byte "inner" buffer +(rsp+0). The tick then reads inner+4 and stores it to session+0xe104 +(the per-engine "progress" cache; poll interval ~1.25 s from the same +function). A second consumer of the tick's return value compares it +against per-boot config at lib+0x2019320: + + 0xfc8 = 1000, 0xffc = 95 (progress threshold), + 0xfec = 10, 0xfed = 20 (bit-history cutoffs), + 0x1000 = 500 + +and ORs the comparison bit into a round-robin bitfield (+0xb4) of the +state machine object. + +## Live evidence on r680 (fresh boot, full-param recipe) + +* The 0x2080a097 polls (NVOS54 0x2a, 1.25 s cadence) come from a + WORKER thread (gdb bt: start_thread -> eglcore a11b2a -> wait loop + 0xd6ef40 -> tick 0xd5e3e0). The engine object E (0x604101365e90) + sits at E+0x150 = 0x00200000 (bit 21 only), E+0x1b038 = 1, + E+0x1b039 = 1, *(E+0x1b030)+8 != 0 — all static for 5+ minutes. +* MAIN thread (vkCreateDevice) is in a poll(fd=/dev/nvidia0, events + POLLIN|POLLPRI, timeout=10s) loop inside eglcore (bt: poll <- + eglcore+0xa12de4 <- 0xd69159 <- 0xd6a02b <- 0xdcebc6 <- ... <- + libvulkan vkCreateDevice). No -4 is ever printed in this state: the + client waits indefinitely (killed by its own timeout in runs). +* GR channel kicks continue concurrently (sema-entry lines for + h=0xbeec0100 va=0x4ca0000 len=1432 etc.) and end as + "skipping stale entry ... after N failed rescues" (gr_wedge>0). +* Reply sweep on r680 (module params) — flag=0/2/1/1..0xffffffff, + v488, v48c AND the new record matrix count/rec0..rec7 = + 0/{1,0,0,0}/100/1000/100000/1000000/0xdeadbeef — produced ZERO + observable change: same a097 cadence, no writers to + E+0x150/E+0x1b038/E+0x1b039 ever caught, no client progress. The + hw watchpoints on those words fired zero times over ~5 min. + +## Tools + +* t75g.py — in-guest gdb python: dispatcher-type log, a097 issuer + dump, tick (d5e449) state + cfg dump, E/S capture, hw watchpoints on + E+0x150 / E+0x1b038 / E+0x1b039 / S+0x120, POST a040 consumption + dump. +* diag76.py — in-guest gdb python: thread-apply bt + map-relative + address resolution (used to find the main-thread poll wait). +* slots_probe.sh — read the u16 completion-slot words of every small + /dev/nvidiactl rw-s mapping (SESSION 74's slotfill ABI). +* run75g.sh — trace + record-word sweep driver. + +## Verdict + +The a097 monitor is a background worker, not an obviously blocking +gate: every knob the client reads from it can be varied with no +observable state change, while the main thread blocks elsewhere +(/dev/nvidia0 poll loop) and the GR channel-init pushes still cannot +be read by the sema service. Next lever is the main thread's wait +(what would make the /dev/nvidia0 poll return readable) or the GR +channel-init completion — not the a097 reply words. + +## ADDENDUM (r681) — THE MAIN-THREAD vkCreateDevice WAIT DECODED + +Live gdb on the main thread (r681, fresh boot) resolved the loop at +eglcore 0xdceb80 (frame chain: poll <- a12de4 <- d69159 <- d6a02b <- +dcebc6 <- d3838a <- ... <- libvulkan vkCreateDevice): + +* rbx (waitable, vt 0x...3f58f8): vt+0x18 = d69ee0 = "should wait" + (returns `(*0x48(*(rbx+0x70)) != 1)`); vt+0x28 = d69f50 = "wait" + (builds a pollfd array from the object's fd holder and poll()s up to + 16 fds with the 10 s timeout). +* rbp (loop object, vt 0x...3f71f8): vt+0x38 = dcee20 = the give-up + predicate. dcee20 returns nonzero (-> return -4 = the client's + VKFAIL -4 origin) when either object at rbp+0x110 / rbp+0x120 shows + an error (b29d60(...) != 0 or *(obj+0x70cc) != 0); otherwise 0 and + the loop repeats. It also throttles itself at 1e6 (units) via + rbp+0x1b8. +* The readiness value: sub = *(rbx+0x70) (0x...f7f0c90); sub+0x58 -> + obj2; obj2's vt+0x10 = 0xce71f0 = `read u32 at (*(obj2+0x58)) + + *(u32*)(obj2+0x60)` — i.e. a u32 word at holder_base + 0x80 (the + live object had obj2+0x60 = 0x80, obj2+0x64 = 0x1e). Observed: 0x0 + every poll (39/39) -> should-wait = 1 forever. +* The word lives in a **/dev/nvidiactl rw-s mapping** + (0x7623d0480000-0x7623d0484000) — a VM_IO driver page, not plain + guest RAM: /proc/PID/mem reads of it return zeros, and writing the + guest-RAM file at the pagemap-derived PAs (0x1036dd080 and both + hole-shifted candidates) did NOT change the client's behaviour. + The writer is therefore the guest kernel driver/RM when the awaited + operation completes (the same class as the SESSION-74 completion + slots) — not a host RAM poke. +* All 20 sampled polls of the wait helper returned 0 (timeout): the + fd (/dev/nvidia0) never becomes readable (nv_poll reports + POLLPRI|POLLIN only when an nv_event is pending on the file). + +So vkCreateDevice waits for (a) the readiness word == 1, and the wait +ends only when the driver/RM signals that word (or an nv_event arrives +on /dev/nvidia0). The give-up path (-4) fires only if the RM error +flags appear. Next: identify the operation whose completion writes the +word (the object chain sub -> obj2 -> holder is a "signal/semaphore" +object; dump the holder object and its creating control), then make +the emulation drive the guest kernel to complete it (the engine/ +channel-init completion the sema service already tries to deliver). diff --git a/tools/guest-diag/wall2/README-session77.md b/tools/guest-diag/wall2/README-session77.md new file mode 100644 index 0000000..1ae9a62 --- /dev/null +++ b/tools/guest-diag/wall2/README-session77.md @@ -0,0 +1,1063 @@ +# session 77 (r683/r684) — vkCreateDevice PASSES: the 0x7E event validation and the readiness-word completion + +## The two-part unblock + +1. **r682 fix validated** (`e3c1cdc5`, loaded on r683/r684): the + nvkms/Vulkan path allocates its completion event as + NV01_EVENT_KERNEL_CALLBACK_EX (class 0x7E); the emulation now + registers it alongside 0x79. Live proof (r683, sema_dbg=1): + + GSP_RM_ALLOC class=0x7e paramsSize=24 hClient=0xc1e00011 + hParent=0xcaf00001 hObject=0xcaf00003 + event registered hEvent=0xcaf00003 hSrc=0x0 + notifyIndex=1006632972 (off 8, slot 13) + params=1100e0c1000000007e0000000c00003c0000000000000000 + EVPOST all ev[13] cl=0xc1e00011 ev=0xcaf00003 nidx=0x3c00000c rc=0 + + (notifyIndex 0x3c00000c = subdevice 0x3c, index 0xc = + NV2080_NOTIFIERS_GRAPHICS). The guest consumes the records + (`guest_read == our_write` in the rmstate dump), but the guest's + event dispatch does NOT write the event's notification word — the + residual delivery gap. + +2. **The completion word found and driven (the actual unblock)**. The + main-thread vkCreateDevice wait (eglcore 0xdceb80, session 76) reads + a u32 readiness word: `should-wait = (word != 1)`; the word lives at + `+0x80` of a **16 KB rw-s /dev/nvidiactl mapping** (the event + notification buffer; holder chain sub -> obj2 -> holder, resolved + live by t79.py). The host can write it directly in the guest RAM + file: + + PA = pagemap PFN << 12 (+0x80); file offset = PA - 0x80000000 + (above-4G RAM; below-4G would be PA itself per the cycle's + guest-RAM registration: gpa 0x0/0x100000000 -> off 0x0/0x80000000) + + Writing 1 flipped the wait and **vkCreateDevice returned success** + (`VKDEVICE ok family=0`) — reproduced three times: + * r683: manual write after a t79 probe; + * r684: nudger (all 16 KB mapping words) + slotfill restart; + * r684d: probe -> single-word write (the reliable recipe). + +## The reliable recipe (per run) + +1. r682+ module; `sema_dbg=1 gsp_rm_ev_log=300` optional. +2. Launch `vk_frame` + `slotfill2.sh` (EGL completion-slot fills; the + slots re-arm, so re-run slotfill if the client is stuck in the + sched_yield spin before the word-wait). +3. Wait for the main thread in `do_poll` (the word-wait). +4. `t79.py` (gdb) -> `WAIT word=0x...`; guest pagemap -> PFN -> PA; + host write `01 00 00 00` at `PA-0x80000000`. +5. `VKDEVICE ok` follows within ~10 s. The client then proceeds to + image/memory/pipeline setup; its next wait is NOT the same word + (the word stays 1) — next session: probe the follow-on state + (attach fails while the client spins in the post-device phase; + use the /proc-only checks + a longer observation window). + +## Cautions learned + +* Writing ALL 16 KB mapping words (the multi-page nudger) can trip the + -4 give-up path (an RM error flag word gets written): restrict nudges + to the single probed word. +* Attaching gdb to the client while it is in the tight post-device spin + can hang the attach (leave a `timeout`-wrapped gdb, then kill that + gdb by PID and `kill -CONT` the client — the pending SIGSTOP from a + dead tracer otherwise wedges it). +* zsh truncates 64-bit hex literals in `$(( ))`: compute PFN/PA in + python, not zsh. + +## Tools + +* `t79.py` — resolves the wait chain (sub -> obj2 -> holder -> word VA) + and prints the client's nvidiactl rw-s mapping sizes. +* `guest_words.sh` — prints VA/PA of the +0x80 word for every 16 KB + nvidiactl mapping (pagemap; run in the guest as root). +* `nudge.sh` — host-side loop: poll guest_words.sh and write 1 to any + zero word (multi-page variant; prefer the single-word recipe). + +## ADDENDUM (r685, session 78) — POST-DEVICE WAIT IDENTIFIED: the NvOgl (GL ucode) engine init; push windows now READABLE (resolver6) + +* The post-device spin is the **GL module bring-up**: the main thread's + A8-layer retry loop (eglcore 0xa85d24) works on a work item whose + embedded name is ASCII **"NvOgl_m0"** (+0xf04: "NvOgl_m40"); the D6 + submit layer (0xd624e0) ORs **0x200000** into its pending word (the + same engine-pending bit SESSION 75/76 watched on E+0x150), then the + A8 loop yields via the B284B0 spin helper (sched_yield) waiting for + the GL ucode init to complete on the GR engine. +* **The push windows ARE now readable** (first time): the resolver6 + recipe (window-record pair {push_va,0x20000}+{ring_va,0x40000} -> + same-page 0x20000 memdesc PA -> `sema_map_add`) injected + `0x4480000 -> 0x120cfb000` (content `2001c00d 70000000 2001c00d + 68000000` = the 2-method semaphore-like push) and `0x4ca0000 -> + 0x12e51e000` (the 1432-byte GR init stream: SET_OBJECT 0xc5c0/0xc5b5, + `0x200121ed`, `0x200120a9`, bulk methods, tail `2001c00d + 70/68`). The service now decodes the streams (`sema-push ... tail=0dc0 + 012000000070...`) where SESSION 73-74 saw "unread". + CAUTION: the resolver injects every candidate; a zero-page candidate + injected last replaces the good mapping for that VA (re-inject the + good page after the scan). +* **Remaining gate**: the service still reports `have_rel=0` for every + GR push (the CE-shaped release matcher) and — decisively — the + GR-forward machinery is NOT engaging: `drain held=4 starved=149 + forwarded=0 ... gr_nocap=4`. The GL ucode init work is never + forwarded/executed, so the completion (and the 0x200000 bit clear) + never arrives. Next: the GR forward/capture path for these channels + (operand capture + release decode for the GR shape), then the client + proceeds to VKFRAME. + +## ADDENDUM (r686, session 79) — the low-VA release fix and the corrected injection recipe + +* The GR/GL init releases decode only with the low-aperture acceptance + (`SEMA_VA_LO` bound relaxed to >= 0x4000000 in both release forms, + commit cb8b75c5): live `rel=0x4b19b10 pay=1` (REPORT_SEM group + `200426c0 00000000 04b19b10 00000001 10000000`), `rel=0x4b17bb0 + pay=3` and `rel=0x4b17e30 pay=1` (0x5c-form `2005c017 04923060 ... + 02000001`); `have_rel=1`, `applied` rising, un-translated releases + queue in svc_pending (417 observed). +* Corrected pair-specific push injections for r686: + `0x4480000 -> 0x125692000` (record page 0x1163c9000, live desc), + `0x4ca0000 -> 0x131649000` (page 0x122f20000), plus + `0x4b16000 -> 0x12a76a000` (the {0x4b16000,0x1000} window). + CAUTION: resolver7 injects each page's descs for BOTH pairs (union + loop) - fix it to keep per-pair pages or inject manually from the + per-pair scan. The window-record layout is {tag, VA, SIZE, 0, + "tail", ...} stride 0x40 (search the VA u64 with size at +8 and + "tail" at +0x10). +* Remaining: the semaphore VAs (0x4b17bb0/0x4b17e30/0x4b19b10) have no + window records in the known record pages; the service cannot resolve + them (targeted rm_mem miss) so the releases stay pending and the + NvOgl wait spins. Find their PAs (client mapping pages / RM + VA-tracker / live poll-watch), inject, and the pending releases + should apply. + +## ADDENDUM (r686, session 80) — ALL low-aperture window records found (the "liat" tag fix); the semaphore windows identified and their releases applying + +* **The record scan fix**: the RM window record is 0x40 bytes + {u64 tag; u64 VA; u64 SIZE; u64 0; u32 0x7461696c; u32 0}. The tag is + stored as the u32 0x7461696c - on disk the BYTES are "liat", NOT + "tail" (the hex display `7461696c` reads as "tail" only as a number). + Searching for b'tail' finds nothing; `wall2_recscan.py` searches for + the byte pattern 6c 69 61 74 and decodes {tag,VA,SIZE} at + -0x20/-0x18/-0x10. It found **all 75 records** in one pass. +* **The GR init semaphore windows** (r686): + `va=0x4b17000 size=0x2000 -> pa=0x12a76f000` (verified by content: + +0xbb0=3, +0xe30=1 - exactly the rel=0x4b17bb0 pay=3 / + rel=0x4b17e30 pay=1 payloads) and + `va=0x4b19000 size=0x1000 -> pa=0x122f9b000` (after injection the + release applied there: +0xb10 became 2). Injections: + `echo 0x4b17000,0x12a76f000,0x2000,1 > sema_map_add` and + `echo 0x4b19000,0x122f9b000,0x1000,1 > sema_map_add`. +* **State after the injections**: the low-VA releases decode + (`have_rel=1`) and apply to the semaphore pages; `pending` hovers at + ~418 (the client re-kicks continuously); the NvOgl wait still spins + (the A8 loop compares per-entry counter tables r14+0x5e8 vs the + wanted set - the completion is not just the semaphore value). Next: + the A8 counter tables' source (what fills r14+0x5e8 / the "wanted" + table - likely the GR channel's own progress counters) or the GR + forward/ucode execution (`forwarded=0 gr_nocap=7` still). + +## ADDENDUM (r686, session 81) — the counter-autoreg gap analysed: the counter page 0x20400f000 is unresolvable; the manual `sema_counter` registration is the bypass + +* `svc_counter_autoreg` (per-pass for eng==0 channels, n_gr=20) needs a + translation for the rank slot VA `0x20400fec0 + (19-r)*0x10` + (r = gr_index = rank by ascending handle among the guest's GR + channels): either an ipin pin (`svc_ipin_lookup`, 9275 entries live + from the guest pin stream) or `svc_counter_derive` (the + vaspaces/discovered/targeted walk chain, `sema_counter_derive=1` + default). Both miss for the counter page -> no "counter auto" log, + `pub_calls=0` -> the A8 loop's per-entry counter tables never fill and + the NvOgl wait spins. +* The client does NOT map the counter pool: `/proc/<pid>/pagemap` for + 0x20400f000/0x20400ffd0/0x20400ffc0/0x20400ffb0 is all-zero (not + present) and no VMA covers it - it is a UVM kernel-side object, so the + pin stream cannot resolve it through the client's page tables. The + stream's log shows `sem=0` (zero semaphore resolutions) even though + the want-list (`/tmp/guest_want_vas.txt`, host-written) contains 44 + `20400f` entries; the stream's "channel-anchored expansion" for the + UVM tracking sems (guest_pin_stream.py ~line 950-1010) is the + machinery that must find them (it is written for the CUDA second-client + stall - "the only live uvmt event is the spinning sem"). +* Bypass available now: the `sema_counter` parameter registers a channel + slot manually - `echo <h_object>,<pa> > .../parameters/sema_counter`. + Once the counter page's PA is known (via the guest kcore/UVM walk), + this wires the publish path without a kernel rebuild. +* The rank slots for the GL client's channels (handles 0xbeec0100, + 0xbeec0104, 0xbeee0100 -> ranks 0,1,2): 0x20400ffd0 / 0x20400ffc0 / + 0x20400ffb0. + +## ADDENDUM (r686, session 82) — the counter-page PA hunt: pagemap route confirmed dead, vidmem-window hypothesis tested and not supported + +* The client's own pagemap does NOT contain the counter pool: the + semfix_uvmt tool's resolver (`guest_pmap_win.py <pid> 0x20400f000 + 0x204010000 64`) returns NOTHING, as does the UVM pool range + (`0x121000000 0x121100000`). The tool's `pin_counter_page` has + therefore never succeeded for these clients (the STALL WATCHDOG's + "re-pin counter" attempts across sessions produced no "counter: + pinned" line). +* VIDMEM-window hypothesis tested: the counter VA 0x20400f000 is in the + emulated vidmem window (>= SVC_VIDMEM_VA_BASE 0x200080000), so its + window offset is 0x3F87000 (0x20400f000 - 0x200080000). The shadow + there is reachable and all-zero (`sema_vidmem_dump 0x3f87000,64`); + poking the rank slots (+0xfd0/+0xfc0/+0xfb0 = ranks 0/1/2) with 6 and + then 0x40 via `poke_vidmem` changed the shadow (verified by dump) but + did NOT move the client's wait. The vidmem route is not (or not only) + the completion path; the A8 loop's tables have another source. +* gdb-attach hazard (again): attaches to the client in its post-device + spin hang with no output; two gdb chains had to be killed by PID + (never a pattern) and the client SIGCONT'd. Probe only while the + client is in do_poll. + +## ADDENDUM (r687, session 83) — THE A8 COUNTER SOURCE DECODED: a client readback window the host can write (first live read-back confirmed) + +* The A8 loop calls `a82f30(obj)` per entry; a82f30 calls the object's + method pointer at ***(obj+0x19b8)** = eglcore+0xb250f0, which returns + `(hi<<32)|count`. The count comes from **b1bf00**: + - fast path (obj+0x224 != 0): `*(u32*)(obj+0x1078) - 1`; + - slow path (observed: obj+0x224 == 0): **b27c90(obj, 0x800000bb/0xbc)** + which reads `*(u32*)(*(*(obj+0x1480)+0x58) + idx*16)` - a **client + READBACK WINDOW** (idx = 0xbb..0xbb+n, n = *(u32*)(obj+0x22efc)); + the loop takes the MIN of those reads. +* Live (r687, object handle 0xbeec0100): the window = VA + 0x7827e6c12000 (a /dev/nvidiactl mapping), all zeros; obj+0x224=0 + (slow path active), obj+0x1078=7 (fast-path field unused). +* **The host CAN write it**: resolving the window's PA via the client's + pagemap and writing 0x100 into the window (offsets 0xbb0..0x1ff0, via + the RAM file) made the client read it - **obj+0x1070 = 0x100** (the + method stores the returned count there) and the A8 "actual" table + entry became `0xffffffff00000100`. FIRST LIVE PROOF that the client's + completion counter can be driven from the host. +* The wait still spins: the A8 loop's first scan (compare + a85e1d: actual vs want, want observed 0) passes with our value, but + the loop continues into a second scan (compare a85ec1, driven by + b29dc0) and the `*(rsp+0x38)` branch (a85e60). NEXT: decode those + (the second scan's actual source and the rsp+0x38 flag), and whether + the other objects (entries 0-2 of the A8 table) need the same window + writes. +* The other objects' windows were not observed (only 0xbeec0100 called + a82f30 in the sampled window); the nvidia0-side window (obj+0x1478) + is NOT present in the pagemap (VM_IO; host cannot write it via the + RAM file). + +## ADDENDUM (r687, session 84) — the a86038 want-scan mapped: ALL WANTS PASS with the driven window value; the residual gate is the A8 loop's internal callback/retry machinery + +* Live dump of the a86038 scan (BP at 0xa8608e, per-index): + i=0 want=0x0 actual=0x0 pend=no + i=1 want=0x0 actual=0x0 pend=no + i=2 want=0x0 actual=0x0 pend=no + i=3 want=0x6 actual=0xffffffff00000100 pend=no + (want from a82fd0(job_obj, i) = the job object's +0x98 array maxed + against global+0x600; actual from r14+0x5e8 = our driven counter). + NO pending entries -> the scan completes. +* Writing 0x100 to the ENTIRE readback window (offsets 0x0..0x1ff0 of VA + 0x7827e6c12000) did not clear the wait either. +* The loop still cycles: the bt stays at the a85f08-a85f1a yield call + (b284b0(*(rsp+0x10), 0xb) - a bounded pause/yield spin of ~0x14 + units, hence the slow cadence), and the exit-attempt path a85f30 + decrements *(rbp) (w0=1) while firing the global callbacks + (0x206f790/0x206f7a0 vtables: *(%rax), *0x10, *0x18) with args from + rbp+0x30/0x38/0x40. The remaining gate is this callback/retry + machinery (or another wait object), not the readback values. +* NEXT: resolve the global callback targets (0x206f790/0x206f7a0 + vtables at runtime) and trace the a85f30 exit path (what the + callbacks do and what makes *(rbp) reach 0); or find how the real GPU + writes the readback window (the op-4 methods 0x800000bb/0xbc the + service skips) and emulate that write so the client's own machinery + completes. + +## ADDENDUM (r687, session 85) — **THE NvOgl WAIT CLEARED** (the session-count gate) and the client advanced to vkAllocateMemory (new wall: -2 OUT_OF_DEVICE_MEMORY) + +* The A8 loop's exit requires the session count `*(u32*)(r14+0x5bc)` + (= lib_base + 0x2070b7c, the global at +0x20705c0+0x5bc) to reach 0; + the loop head increments `*(rbp)` and the entry scans pass with the + driven window value, but with count=4 the exit checks + (a85b48/a85e78: `if (r12d == 0)`) are never taken - the loop waits for + the engine sessions to complete (count -> 0). +* **DIRECT TEST (succeeded)**: `set {unsigned int}0x<base+0x2070b7c> = 0` + via gdb -> the loop exited, the NvOgl init completed, and the client + ADVANCED past the post-device wait for the first time: + + VKDEVICE ok family=0 + VKFAIL vkAllocateMemory(dev, &ma, NULL, &img_mem) -> -2 (line 204) + +* **NEW WALL**: `vkAllocateMemory` for the image (DEVICE_LOCAL) returns + -2 (VK_ERROR_OUT_OF_DEVICE_MEMORY) at vk_frame.c:204 - the guest + driver's heap/type selection fails before any alloc RPC (no + GSP_RM_ALLOC in dmesg). Next: the memory-heap reporting + (FB_GET_INFO_V2 0x20801303 is handled; check the heap-size/type + queries the driver uses for the DEVICE_LOCAL type, and the emulated + VRAM size the driver sees). +* Recipe for the next run: slotfill + word nudge -> VKDEVICE ok; then + write 0 to lib_base+0x2070b7c (gdb; the harness) once the NvOgl wait + starts (the A8 loop's count gate); then the memory-alloc wall. + +## ADDENDUM (r689, session 86) — **THE DEVICE_LOCAL -2 WALL CRACKED**: the 512 MiB emulated VRAM was the gate; the "nvidia-nvidia-vgpu-turing-2g" mdev type (2 GiB) lets the allocation proceed + +* Evidence chain: the ICD hooks (t91.py) captured the client's allocation + `ALLOC sType=5 size=0x4000 typeIndex=1` (a valid 16 KB DEVICE_LOCAL + request); the standalone memory-properties dump (vk_memprops.py) + showed device-local heap[0] = 512 MiB with a separate 2936 MiB + non-local heap. With 512 MiB the driver failed the allocation + (VK_ERROR_OUT_OF_DEVICE_MEMORY, no RM RPC). +* **FIX TESTED**: the cycle with `VGPU_MDEV_TYPE=nvidia-nvidia-vgpu-turing-2g` + (start_vm_keep.sh honours the env var) -> `heap[0] size=0x80000000 + (2048 MiB)` (vk_memprops.py) -> the SAME client with the SAME recipe + (slotfill + word nudge + count auto-clear) reaches `VKDEVICE ok` and + the allocation NO LONGER fails -2: the client proceeds into + `os_schedule` (a deeper wait). +* REMAINING in the new state: the client's later pushes (va=0x44852d4 / + 0x4ca52d4, then +0x586c) extend beyond the first window page and read + `pa=0x0`/zeros (the sema injections covered page 1 only; the window's + page list is scattered - the memdesc carries only the FIRST page). + NEXT: extend the window injections to the full scattered page list + (from the memdesc's page array) or accept 1-page pushes; then the + allocation completes and the client proceeds to the buffer/pipeline. +* Tools: vk_memprops.py (standalone memory-properties dump; safe + alongside other clients), vk_alloc_test.py (standalone allocation + reproducer - NOTE: it hits vkCreateDevice -4 without the harness + (word nudge + count clear), so it cannot replace the vk_frame run), + t91.py (auto-clear + ICD memprops/alloc hooks). + +## ADDENDUM (r689, session 87) — the window page-list hunt: the pages are scattered; content matches found but the VA<->page relation is unresolved + +* The 2 GiB cycle's window records (recscan89): `0x4480000` descs = + {0x107c47000 (live init stream), 0x103c06000, 0x12ecc2000}; + `0x4ca0000` descs = {0x11dbfa000, 0x12f398000 (live init stream)}. + The window's later offsets (va+0x52d4, +0x586c, +0x9000) at + first_PA+offset are ALL ZEROS -> the windows are NOT contiguous + (scattered pages). +* The memdesc record (page 0x106128000) is {size=0x20000, flags + 0x03008000, PA=0x107c47000, kernel pointers 0xffff8d2e5e21ec50 / + 0x4fe7ec50 / 0x461287e8 ... , "head"/"tail"} — the page list hangs off + the kernel pointers. A guest kcore walk of those pointers returned + "unreadable" (stale addresses by the time of the walk; the walk must + happen on FRESH pointers in the same pass as the dump). +* Content search for the distinctive init-stream prefix (`00 80 01 20 + b5 c5 00 00 04 00 04 20`) found the window pages: + 0x107c47000+0x0 (the injected init page), 0x103c06000+0x858, + 0x11dbfa000+0x858, 0x126d69000+0x59c — the later pages hold the same + engine-init streams at arbitrary offsets; the VA<->page relation is + NOT linear (the +0x59c push appears in pages whose +0x0 is zero), so + the mapping cannot be reconstructed by offset alone. +* NEXT: (a) dump the memdesc + kcore-walk its page-list pointers in ONE + guest pass (fresh addresses); or (b) inject the content-matched pages + at trial VAs and watch the service's reads; or (c) the service-side + decode of the window page lists. The client (2632) blocks in + os_schedule at the allocation — the unread later pushes are the + suspected gate. + +## Session 88 addendum - the push-page fix, the gdb-kill wedge, the ring-mapping gap + +**1. PATH-5 RULE (the push pages).** A GR window's pushes do NOT live at +the window base page: the r689-family windows carry their method streams in +WINDOW PAGE 5 (va offsets 0x52d4/0x559c/0x586c/0x596c all live in +0x5000-0x5fff). Standing injections at the base page resolve pa=0x0 for the +pushes (the service's `sema-pushall ... pa=0x0` lines) and the channel +holds/skips entries as "GR no-capture". The fix: page PA = +match_filoff - (va & 0xfff) + 0x80000000 for a content match of the push's +first8 bytes constrained to the same in-page offset; then +`echo 0x<va&~0xfff>,0x<pa>,0x1000,1 > .../sema_map_add`. Proven on r689: +channel beec0104's pushes at va=0x4d00000 (page -> PA 0x12f398000) went +from first8=00000000 to first8=20012000 0000c5c0, and `fast sweep` +applied 3281 -> 3809 (+30/sweep sustained). + +**2. host_resolve.py** automates 1: it polls dmesg for sema-pushall lines +with pa=0x0 and non-zero first8, searches /dev/shm/vgpu-guest-ram +(offset-constrained), and injects the page. The service's 6 s no-capture +hold is the race window - run it BEFORE the client and keep it polling. + +**3. NEVER SIGKILL AN ATTACHED GDB.** On r689, killing the stale t91.py +gdb (kill -9 on PID 5334) killed its tracees (client threads +2883/2884/2885/3048); one held the guest RM API rwsem write lock; the lock +leaked and every later RM call blocked (`[vkps] Update blocked for more +than 1228 seconds` in rmapiLockAcquire -> RmGssLegacyRpcCmd -> +_nv04ControlWithSecInfo). The t91 harness design is correct: `timeout 500` +SIGTERMs the gdb and it detaches cleanly. If a gdb must go, use SIGTERM +and wait; SIGKILL only when the guest is about to be cycled anyway. A +leaked kernel rwsem cannot be repaired live. + +**4. The r690 gap: the GR channel ring mapping.** Fresh cycle r690 (mdev +f5a796b0): every RM RPC of the client init is answered (GR ctx commit, +channel allocs class 0x9067 hObject 0xcef00000/0xcef00080, EVPOSTs), but +the service reports `gpfifo done=0 nomap=938 pushes=0` - it cannot read +any GPFIFO ring, so no entry decodes, no pushall line appears, and the +guest's completion poll loops on RM control 0x20800a70 until the ~10 s RM +timeout -> vkCreateDevice returns -4 (VK_ERROR_DEVICE_LOST). r689's +service had done=19108 (rings mapped) - so the ring mapping is the +missing ingredient for a fresh client in r690; candidate paths: the FIFO +alloc params (12 bytes, GPFIFO VA inside) are not logged; the scanner saw +the CE window (ipins 4592) but no GR ring this boot. Next: log the 0x9067 +alloc params (or scan the guest for the ring VA) and inject the ring page. + +**5. gr_walk.py / gr_walk2.py** are the guest-side kcore walkers (fresh-pass +memdesc discovery; 12 records found, kptrs readable in-pass) kept as the +alternate grounding path. + +## Session 89 addendum - THE PT WALK WORKS (live); the completion-word gap + +**1. GMMU PT walk resolved LIVE (the mission's PT-binding step: DONE).** +The service's own walker math (svc_gmmu_walk: root_bits=ilog2(entries), +shift=49-root_bits; L1 shift38, L2 shift29, PD0 idx=(va>>21)&0xff with 16-byte +dual big/small entries, PT idx=(va>>12)&0x1ff; PDE bit0=0, PTE bit0=1; +pte_addr=(e>>8)<<12) was replicated host-side over /dev/shm/vgpu-guest-ram +(walk.py / live_walk.py). The LIVE published pdb (dmesg: "page directory +... pdb=0x... entries=4" - a NEW pdb per client, e.g. 0x105f0a000 / +0x12a359000 / 0x107012000) resolves every client VA while the client runs: +ring 0x44a0000 -> PA, windows 0x4480000/0x4485000/0x4ca0000..., releases +0x4213000/0x45a0bb0, and the injection via sema_map_add (ap=1 sysmem) makes +the service read them. VALIDATED BY CONTENT: the ring page held exactly the +GPFIFO entries the service logged (entry[1] raw=8000040004480598 va=0x4480598 +len=4), and the push page at va=0x4480598 held the method stream +(...0x20020047...). Post-mortem reads fail: the guest frees/zeros the +structures and pdb pages when the client dies - ALL resolution must run +LIVE during the client's ~11 s life (live_walk.py polls dmesg every 0.4 s, +parses pdb + VAs, walks, injects; static family list pre-seeds the +deterministic VAs). + +**2. Progress with the walk+inject live: gpfifo done=0 -> 6-9.** +With the live injections the service decoded and CONSUMED the whole GR ring +(entries raw logged, releases decoded: rel=0x4213000 pay=1, rel=0x45a0bb0 +pay=3, have_rel=1) and submissions reached 6. The client STILL ends +vkCreateDevice -4 (~11 s, RM 10 s poll of control 0x20800a70) because the +releases are decoded but NOT APPLIED and the client's completion counter +page is unresolvable. + +**3. Why releases do not apply (two gates found).** +(a) svc_fwd_defer_arm(): with gsp_phys_gr_forward=1 a GR ring candidate that +is valid but not "offered" keeps defer armed - every GR release goes to the +forwarded-hardware path (which never runs) instead of the inline software +apply. Cycling with gsp_phys_gr_forward=0 removes the arm, but the release +still does not apply - the decode's apply path is additionally gated by +svc.op_tr_pending (a global "operand targeted scan in progress" state) via +svc_opdef_release: while op_tr_pending=1 every release is recorded into an +opdef slot and withheld, silently when the slot already contains it. The +live symptom: `counter page 0x20400f000 unreadable via +vaspaces/discovered/targeted; slot stays pending` repeated at the client's +poll rate plus repeated `targeted resolve va=0x20400f000: full RAM scan +found nothing`. + +**4. The counter/notifier page record tables (host-side resolve).** +Searching the RAM for the counter-page VA as u64 finds 16-byte {pa,va} +record arrays, e.g. {0x5FEDC000,0x20400D800},{0x5FEDD000,0x20400E000}, +{0x5FEDE000,0x20400E800},{0x5FEDF000,0x20400F000},... (VA step 0x800, +PA step 0x1000) and a second variant ({0x5FEEE000,0x20400E000}, +{0x5FEEF000,0x20400F000}). These are how the counter page (and probably +the notifier, notif=0x1xxxxxxx0 per registration) can be resolved WITHOUT +the VA-space walk: search the VA u64, parse the neighbouring u64 as PA, +inject. Must be done live (records vanish post-mortem). NOTE: the service's +own targeted scan reports "found nothing" for this VA - the record layout +the host parses is not what that scan looks for. + +**5. Next steps (ranked) for session 90.** +1. Extend live_walk.py with a record-resolve fallback: for any VA the + service logs as unreadable/pending/stalled (regex set already widens), + search the RAM for its u64, parse {pa,va} records (both orders), inject + candidates; feed it counter-page (0x20400f000 family), notifier + (notif=... per registration) and release VAs BEFORE the decode (static + list already covers the deterministic ones: add 0x20400f000 with the + record-resolve because the pdb walk cannot reach the UVM/counter space). +2. Or fix the two release gates kernel-side: (a) don't arm the GR deferral + when the forwarding engine has no proven channel (offered flag); (b) + treat op_tr_pending as per-request state (clear when no scan is actually + in progress for the entry) so a stuck global does not withhold every + release. Both are small, testable changes (gsp_sema_service.c). +3. Watch for the first live sign: `rel-apply ... written` / `pub-adv` / + `counter auto/derived` lines on the client's channel; then VKDEVICE + should pass and the flow proceeds to vkAllocateMemory. + +**6. Counter-page record resolve PROVEN LIVE (session 89 end).** +`echo 0x20400f000,0x5FEDF000,0x1000,1 > sema_map_add` (the {pa,va} record +found by the u64 search) made the service learn the slot: +`sema-chan: counter auto h=0xbeef0100 rank=0 pa=0x5fedfff0 (slot +va=0x20400fff0)` and the completion publish went from ZERO to 172 calls +(`counter pub_calls=72 -> 172` over the client's life; previously 0 on every +boot of this session). The client still ends vkCreateDevice -4, so the +remaining unmet wait is the GR ctx-init release word (rel=0x4213000 pay=1, +rel=0x45a0bb0 pay=3 - decoded, have_rel=1, never applied: no rel-apply / +pub-adv line for either). With the counter now advancing, the next +instrumented step is naming the exact GR release-handling branch that +swallows the apply (rebuild with a bounded print at each branch of +svc_rel_apply_906f / svc_opdef_release / svc_rel_apply_launch), then fix +that gate. The gr_forward defer-arm gate is already understood (see 3a) and +gr_forward=0 is the correct test configuration; the remaining suspect is +the service-global svc.op_tr_pending path in svc_opdef_release (it records +the release into an opdef slot and returns true silently when the slot +already contains it). + +**7. Session 90: the CURRENT-boot wait loop site (the old recipe no longer matches).** +The client's long spin on this driver/boot sits INSIDE vkCreateInstance (bt: +libEGL -> eglcore -> 0xB2852D helper), a bounded yield loop at +eglcore+0xB2852D: `mov (%r11),%r12d / test $1,%r12b / jne <exit> / +mov 0x4(%r11),%eax / cmp $1,%eax / jne <retry>` - the loop waits for an +IN-PROCESS word pair at r11 (e.g. 0x7ffc2a1c5080, stack/heap; guest-written +only, the host RAM file cannot feed it because it is not a GPU buffer). +Evidence the old site moved: t91's breakpoint at eglcore+0xa85d24 never +fires this boot (`clears=0`) and a read-watchpoint (t80r.py) on the +0x80 +words of the two 16 KB rw-s nvidiactl mappings does not fire either (the +16K event-buffer wait of sessions 77/78 is not the live target now; the +spin stack is +0xB2852D <- +0xA5325A <- +0xA539AD <- +0xA47491 <- +0x9FF515 +<- +0xA20D04 <- libEGL, NOT the 0xA85F1A/A8 path). A widened multi-word +nudge (nudge2.sh/guest_words2.sh: all rw-s nvidia mappings <=256K, +0x80) +DID fire on 20+ words but the client still returned -4 - the session-77 +caution stands: multi-word writes trip the error path; single-word only, +after identifying the true target. +NEXT: (a) rwatch WRITE on r11+4 to name the writer of the in-process word +(the component that must run; if nothing ever writes it, disassemble the +exit branch 0x...ebc9 vs retry 0x...eb93 to see the pass/fail contract); +(b) the same for the 0xA53xxx caller chain - one of these callers is the +A8/NvOgl GL-ucode wait of sessions 78/79 (its completion comes from the +emulated GR engine path); (c) only then re-test the count-clear + single +word nudge recipes. + +**8. Session 91: THE vkCreateInstance WALL BREACHED (the marker-clear recipe).** +The client's retry loop (eglcore+0xB33D30) waits until NO event-slot entry has +the pending marker 0xffff: per entry ptr = *(u64*)(obj+0x1428+i*8), +cnt = *(u32*)(obj+0x22efc), the checked slot = ptr + n*0x10 (n = the call's +esi), marker = u16@+0xe == 0xffff (or u32@+0 when (esi-0x43)<=4) via the +check at +0xB1C750 (OR of all pending markers). The guest's event dispatch +NEVER clears it, so the client burns its 10 s -4 timeout. PROVEN FIX: +clear {u16@slot+0xe, u16@slot+0xc, u32@slot+0} to 0 at every hit of ++0xB33D30 (t93 single-shot; t94 auto-clearer, 4000 hits) -> the client +passes vkCreateInstance AND device enumeration FOR THE FIRST TIME +(`VKINSTANCE ok / VKDEVS 2 / VKPICK`, previously the 10 s -4 fired during +this phase). t94 cleared 5+ distinct module objects (obj=0x5d67063a84a0, +0x5d670661ad80, 0x5d670673eed0, 0x5d67067ebb50, ...) over the client's life. +The client then sat in vkCreateDevice's poll(/dev/nvidia0, nfds=1, +timeout=10 ms) loop at eglcore+0xA12DDF..+0xA12DE4 WITHOUT any -4 for 5+ +minutes (vs the old ~11 s death) - the longest the client has ever run. + +**9. The remaining gap (single): the event-fd signal.** The device-phase wait +is a poll() on /dev/nvidia0 (fd 27) - nvidia_poll wakes only when the guest +RM signals the client's event state; the guest consumes our POST_EVENT +records but never performs the event signaling (session 78's "residual +delivery gap"). Writing to /proc/PID/fd/<nvidia0> is not a signal. The +session-84 count write (base+0x2070b7c, read 4, wrote 0, t97) did not flip +this wait alone. NEXT: fix the host POST_EVENT -> guest dispatch association +(gsp_rm_emulation.c r579 area: the payload fields hClient/hEvent/notifyIndex/ +data/info/status; compare against the guest's _kgspRpcPostEvent -> +gpuNotifySubDeviceEvent expectations) so the guest signals the event and +nvidia_poll returns POLLIN; then the same t94-pattern (or a service-side +equivalent) durably unblocks both waits per boot. + +**10. Session 92: THE AUTOMATED PIPELINE - fresh boot to VKDEVICE with ZERO manual steps.** +On a fresh r692 boot (cycle params unchanged, gr_forward=0) a single gdb +script (t99.py) carrying two breakpoint handlers drove the client end-to-end: +(a) the instance-phase marker cleaner at eglcore+0xB33D30 (t94 pattern: clear +u16@slot+0xe / u16@slot+0xc / u32@slot to 0 per entry at ptr+n*0x10 for +entries *(obj+0x1428+i*8), cnt *(obj+0x22efc)) - fired CLEAR#1..#5 across +five module objects; (b) the device-phase wait-word chaser at +eglcore+0xDCEBD9 (t79 chain: rbx -> *(rbx+0x70) -> *(sub+0x58)=obj2 -> + off=*(u32)(obj2+0x60), holder=*(obj2+0x58), base=*(holder+0x58), +word=base+off) - found WORD addr=0x734781a86080 val=0x0 and wrote 1. +Result: `VKINSTANCE ok / VKDEVS 2 / VKPICK / VKDEVICE ok family=0` with no +manual intervention (r692 client 1725). The client then sat in the +POST-DEVICE wait (the next phase - allocation/engine-init), which t99 does +not cover. The live_walk PT-walk injector ran alongside and injected the +window/ring VAs (0x4d20000, 0x4923000...; note 0x4b19b10 resolved ap=FB - +the GL init release is vidmem-mapped and skipped by the SYS-only injector). + +**11. HARD LESSON (attach discipline, again).** A gdb attached to a client +that never hits its breakpoints wedges: `continue` never returns, the +timeout's SIGTERM does not tear it down, and the ptrace slot blocks all +later attaches (hit 3x this session: 146084, the t91-on-1725 attach, both +left holding). RULE: only attach with breakpoints that are KNOWN to fire +for the client's current phase, keep windows <= ~60 s, and prefer +self-terminating handlers (hit caps). A wedged attach leaves the client +RUNNING (state R) but unattachable; cycle the VM for a clean environment +when more attaches are needed. + +**12. Continuing the chain (next session).** (1) Identify the post-device +wait of the VKDEVICE-ok client (bt sample once attachable; likely the +allocation/UVM phase or the A8/NvOgl engine-init with its count gate at +lib_base+0x2070b7c=4 -> 0 and/or the 0xdceb80 wait loop - add the t91 +count-clear BP to the t99 bundle if the A8 path is live). (2) Extend t99 +into the "durable harness bundle": marker-clean + word-chase + count-clear ++ (once VKFRAME) the release/alloc phases. (3) The bNotifyList=1 +(gsp_rm_ev_nlist) path stays armed for the event-signal experiment; +post/EVPOST logging re-budgeted (gsp_rm_ev_log=2000 set). + +**13. Session 93: THE ZERO-PTRACE PIPELINE (r693) - VKDEVICE with no gdb-wedge risk.** +Split the harness by risk: (a) ONE gdb (t101.py) for the INSTANCE-phase +marker clears only (base+0xB33D30 handler, clears u16@+0xe/+0xc and u32@+0 +of each entry; self-disables after 24 clears; NO other BPs so it cannot +fire constantly); (b) the WAIT WORDS handled by /tmp/unwait.py - a guest +python poller with ZERO ptrace that scans the newest vk_frame's rw-s +nvidia mappings of exactly 0x4000 bytes and writes 1 to any zero u32 at +mapping+0x80 via /proc/PID/mem (the session-77 geometry, confirmed by t79 +chains: word = the +0x80 of a 16 KB rw-s nvidiactl mapping). Result on +fresh r693 (mdev 261980af): t101 CLEAR#1..#5 then idle; unwait wrote FIVE +wait words (0x7876ead57080, 0x7876f1816080, 0x7876ea47f080, +0x7876ea81a080, 0x7876ead51080); client reached `VKINSTANCE ok / VKDEVS 2 +/ VKPICK / VKDEVICE ok family=0` with NO wedge exposure. live_walk (PT-walk +injector) ran alongside. The client then sat in the post-device wait that +the +0x80 poller does not cover (t102.py = the one-shot safe diagnostic: +attach -> print PC/base/A8-count(base+0x2070b7c)/bt -> detach, NO continue +and NO breakpoints - use it when the slot is free). + +**14. THE RM-LOCK WEDGE ROOT CAUSE + RULE.** A gdb killed (SIGTERM) while +the tracee sits AT a breakpoint leaves the tracee STOPPED; if that stop is +inside an RM api critical section (the rwlock held across an RM call!), the +RM write lock stays held and EVERY other client blocks in +os_acquire_rwlock_write (state D) - observed directly: old client 1725 +stopped by its wedged t99 gdb -> new client 7967 hung `state=D wchan= +os_acquire_rwlock_write` on ioctl 0xc020462b; killing the gdb + `kill -CONT` +resumed 1725 out of the ptrace stop but it resumed INSIDE the same RM wait, +so the lock stayed held until the VM was cycled. RULES: never let a gdb +timeout fire while a tracee is at a frequent-firing BP; prefer BPs that +fire once or self-disable; prefer no-ptrace pollers for anything with a +repeating pattern; the safe diagnostic is attach -> read -> detach without +continue. + +**15. Session 94: THE INSTANCE CLEARS ARE REQUIRED (zero-gdb disproved); t104 is the safe form.** +Pure zero-gdb run (fresh r694, client + unwait.py ONLY, no gdb): unwait +wrote three +0x80 words and the client still ended `VKFAIL vkCreateDevice +... -4` - WITHOUT the instance-phase marker clears the device gate fails +even with the words. With t104.py (single BP at +0xB33D30; clears +u16@+0xe/+0xc/u32@+0; self-exits on cleared>=8 OR chits>=40 OR 25 s since +the last clear -> returns True so the script tail runs and batch gdb exits +cleanly, no outer-timeout dependence) + unwait + live_walk: CLEAR#1..#5, +two words written, client reached `VKINSTANCE ok / VKDEVS 2 / VKPICK / +VKDEVICE ok family=0` (r694 client 3725). The client then sat in the +post-device wait again; the t102 diagnostic could not attach because the +t104 gdb still held the slot (its 25 s-idle exit never fired - no +post-instance B33D30 hits - and its 150 s timeout likely fired at a stop += the known wedge risk). NEXT FORM: make t104 detach ITSELF from inside +the handler once the clears are done (`gdb.execute("detach")` from the +stop handler; the tracee resumes; the script ends and gdb exits) so NO +gdb is ever held while the client runs - then t102 always has the slot +and the wedge class disappears end-to-end. + +**16. The allocation/post-device phase is the remaining gate.** With +VKDEVICE green the client waits in a site that is neither the +0x80 word +pattern nor (likely) the A8 count path; the next session should (a) get +the detach-first t104 live, (b) run t102 immediately after VKDEVICE to +name the wait (read PC/bt/A8 count = base+0x2070b7c), and (c) inspect the +service for this client (pushes decoded / releases applied / counter +pub_calls) to see whether the allocation needs the live injections only +or a service-side gap. + +**17. Session 95: THE GDB THREAD-STOP INTERFERENCE + the SIGKILL nuance.** +The t105 attach (t=1.5 s, before the client's worker threads spawned) left +four worker threads in `ptrace_stop.part.0` (futex) - the client's main +thread span waiting on them, so the client froze at the post-device wait +with the gdb holding the trace. TERM/INT/HUP on the wedged gdb were all +ignored (batch gdb stuck in continue). SIGKILL on the gdb (1584) was tried +with the reasoning that all stopped threads sat at futex (NO RM lock held): +result - the tracer died, the tracees DETACHED+RESUMED (client 1518 +SURVIVED with its workers back in futex_wait_queue; contrast session 88 +where SIGKILLed-tracer tracees were KILLED mid-syscall). So the SIGKILL +outcome depends on the tracee's stop context; the mission's never-SIGKILL +rule stands, but this datapoint matters when judging recoverability. +After the gdb died, 1518 still held the RM write lock from its own +post-device wait (its wait word was no longer polled because unwait.py +tracks the NEWEST vk_frame and a fresh test client 6555 was launched and +blocked D-state on os_acquire_rwlock_write) - the unwait-newest coupling +must be replaced by ALL-clients polling in the next form. + +**18. HARNESS DESIGN RULES (accumulated).** (1) A gdb attached across +thread-spawn windows stops worker threads and freezes the client - attach +only AFTER all threads exist, or ensure the gdb explicitly resumes new +threads, or do the work with NO gdb. (2) Any gdb that must run through +thread-spawn windows MUST self-detach quickly (the instance phase spans +~10-20 s and the clears arrive in ~5 bursts) and NEVER rely on outer +timeouts. (3) unwait.py should poll EVERY live vk_frame (not just the +newest). (4) The durable answer for all of this is the service-side +folding: the instance markers and the +0x80 words are guest-RAM writes a +kernel-side service can perform on the same triggers (the marker semantics +and the word semantics are understood) - that removes gdb and /proc +helpers from the loop entirely. + +**19. Session 96: THE DETACH-FIRST CLEARER PROVEN (t106) + the orphan-tracer plague.** +t106.py (single +0xB33D30 BP; clears markers; detaches FROM THE HANDLER as +soon as cleared>=5 or chits>=14: `gdb.execute("detach")` + return True so +the batch gdb exits) on fresh r696 (mdev 241af1c1): CLEAR#1..#5, +`BC-DETACHED cleaned=5 chits=5`, `T106-DONE chits=5 cleared=5`, +`TracerPid: 0` - THE SLOT IS LEFT FREE. unwait.py wrote five words and the +client (1549) reached `VKINSTANCE ok / VKDEVS 2 / VKPICK / VKDEVICE ok +family=0`. THE CLEAN FORM OF THE CLEARER EXISTS NOW. + +**20. THE ORPHAN-TRACER PLAGUE (root cause of every t102 failure).** Every +INTERRUPTED gdb attempt leaves an attached orphan: (a) piping gdb to +`head` closes the pipe early and the gdb stays attached (hit twice); (b) an +ssh command dying/timing out mid-gdb-add leaves the gdb running (hit +again). Symptoms: later attaches fail with `Could not attach ... already +traced by process N` / `ptrace: Inappropriate ioctl for device` and the +t102 log stays empty. RECOVERY (session-95 evidence): TERM is usually +ignored (wedged in continue); SIGKILL the orphan + `kill -CONT <client>`; +the tracee SURVIVES when it was running/at-futex and detaches cleanly +(TracerPid 0). STRICT DISCIPLINE: never pipe gdb output (redirect to a +file, read it after); never let an ssh command end while a gdb may be +attached (generous timeouts); after every gdb run verify TracerPid 0. + +**21. THE NEXT FORM (t107, one gdb, one window).** Combine the proven +clearer and the diagnostic in ONE batch gdb: attach -> clears -> detach -> +`python time.sleep(...)` idle -> RE-ATTACH (gdb can attach again after +detach) -> read PC / bt / A8 count (t102 content) -> detach -> print - +one ssh session, one window, everything self-contained, no orphan risk +(every path ends in an explicit detach; wrap each phase in try/except with +a final detach). Expected: names the post-device wait (allocation/UVM vs +A8 count base+0x2070b7c) so the allocation can be passed and the first +VKFRAME reached. + +**22. Session 97: THE DEEP SUBMISSION STATE + THE COUNTER-SLOT WEB.** +The r696 client (1549) reached the deepest state yet: the service reported +`gpfifo done=11576 pushes=10932 counter pub_calls=1356` and climbing +(gpfifo done was 0 in the pre-pipeline boots) - the client submits the full +work stream and the service decodes/processes it (pairs=14395 map=14318, +ipins=9240 from live_walk). The client then sits in the KNOWN userspace +post-device spin (main thread `running` in all samples; one tid in +os_acquire_rwlock_read). NEW: a counter-slot collision spam +(`counter auto ... pa=0x5fedfff0 rejected (slot owned by another channel; +not installed)`, 232+ callbacks suppressed) - the 0x20400f000 injection +resolved the WHOLE counter VA family to ONE page, so both live channels +collided; the manual bypass `echo <h>,<pa> > .../parameters/sema_counter` +(h_object,pa syntax, main.c:2965) was applied for both handles +(0xbeee0100,0xbeec0100 -> 0x5fedfff0) and pub_calls jumped 1356 -> 3739. +The client gate (VKDEVICE -> frames) persists: its remaining wait is the +userspace spin word NOT covered by the +0x80/16K poller (candidate: the +tracker/counter the service now advances - watch whether pub traffic +alone resolves the spin, else t107-GDB the spin site). ENVIRONMENT: one +orphan gdb (10092 on 1549) + stale channels (beec0100) from prior +attempts; the orphan-kill discipline applies (SIGKILL by PID + CONT + +verify TracerPid). + +**23. THE IOCTL-ARG SAMPLING AXIS (no ptrace).** /proc/<tid>/syscall shows +the running ioctl's ARGS (fd, cmd, arg) when caught - sampling the main +thread repeatedly during an ioctl-loop phase names the exact stuck RM +control without any gdb. Use it before ptrace-based diagnostics when the +thread does syscalls at a catchable rate. + +**24. Session 98: THE POST-DEVICE SPIN NAMED - THE SEC2 SCRUBBER AUTH-TAG WAIT.** +perf (no ptrace!) on the spun client: top symbol `channelReadChannelMemdesc` +24.33% (then scheduler internals) - the guest's nvidia driver hot-loops +reading the CHANNEL MEMDESC. Source decode (nvidia-570.144-reference, +sec2_utils.c): the only reference caller is +`_sec2utilsUpdateGetPtr` -> `channelReadChannelMemdesc(pChannel, +authTagBufSemaOffset)` inside `_sec2utilsGetNextAuthTagSlot`, whose loop +waits for the scrub auth-tag slots to free; `authTagBufSemaOffset = +finishPayloadOffset + CHANNEL_ENGINE_SEMAPHORE_SIZE` (4). So the spin = +the SEC2 MEMORY-SCRUBBER channel waiting for its auth-tag semaphore (a u32 +inside the channel memdesc buffer) to advance - the GPU never delivers it +in our emulation, so the client waits forever (each poll = one full memdesc +transfer = the 24% cost). THE FIX DIRECTION: the service must complete the +scrubber's semaphore updates - the scrubber channel registers in the +service (`scrubber channel 0x... detected (gpfifo=... userd=...)`) and its +submits carry semaphore releases targeting the auth-tag VA +(pbGpuVA + authTagBufSemaOffset) in the channel buffer; on decode, write +the release value to that VA's page (the channel buffer is a guest-RAM +allocation the service can resolve/pin; the pending queue holding 17 +entries is the suspect pool for the unresolvable ones - inspect +svc_pending[]). Service-side counter to watch after the fix: the scrub +loop exits, the client proceeds to the allocation/pipeline. Tool: perf +record/report on the guest is the no-ptrace spin-pointer (`perf record -p +<pid> -- sleep 3; perf report --stdio --sort=symbol,dso`). + +**25. Session 99: THE SCRUBBER GATE CONNECTED TO THE CE CHANNELS' ok=0.** +chan_dump on r696 (client 1549 spun): six channels - +`h=0xbeee0100 eng=0 gpfifo=0x44a0000 userd=0xfee00 ok=1 get=6 sub=6354 rel=0x4213020 crel=0x5fedfff0 notif=0x11cd51060` +`h=0xbeec0100 eng=0 gpfifo=0x4cc0000 userd=0xfee00 ok=1 rel=0x4923020 crel=0x5fedfff0` +`h=0xbeec0104 eng=0 gpfifo=0x4d20000 userd=0xfee00 ok=1 rel=0x4b17e30 crel=0x5fedffe0` +and the CE-class channels `beee0101 (eng=11, gpfifo=0x4500000 userd=0x8f600 as=2 ok=0)`, +`beee0102 (eng=12, gpfifo=0x4560000 userd=0x8f400 ok=0)`, `beed0101 (eng=12, +gpfifo=0x49c0000 userd=0x8f200 ok=0)` - ALL ok=0: the CE userd discovery +(failed probe: EAGAIN 0/0 or EINVAL) means no CE pump, no CE sema applies, +so the SEC2 scrubber channel (a CE-class channel) never sees its auth-tag +semaphore advance and the client spins in _sec2utilsGetNextAuthTagSlot. +The service's candidates (registered PA, ring+0x10000 in both apertures, +8-slot list) all miss for the CE channels. `find_userd` (FB scan for a +live put@0x8c/get@0x88 pair) found ZERO candidates - the CE userds are not +in FB with live puts (either the client's PUT writes go to a page the +service's candidates miss, or the CE userds genuinely idle). A raw host-RAM +shape scan (put in 1..65535, get<=put) is too noisy (24 hits in the first +28KB of BIOS-era low RAM) - needs tighter validation. NEXT APPROACH: find +the REAL userd page by DIFF-WATCH - sample the candidate pages (and the +CE ring pages) twice around a scrub submit and diff for the page whose ++0x88/+0x8c pair changes; the guest kernel writes GP_PUT there, so the +changing page is the true userd. Then fix the discovery (or add a +sema_userd-style injection param) so the CE pump engages and the scrubber +completes. NOTE: the GR channels' userd=0xfee00 (as=2, FB) and ok=1 with +get=6/sub=6354 - the GR machinery runs fine; only the CE/scrubber path is +dark. + +**26. Session 100: THE CE USERD HUNT EXHAUSTED THE STANDARD LAYOUTS; THE CIRCULAR WAIT.** +Candidates examined for the CE userds this session: (a) the registered FB +offsets (0x8f600/0x8f400/0x8f200 as=2) - ALL ZERO in the FB shadow +(sema_vidmem_dump 0x8f100,0x600 twice); (b) the ring-PA + 0x10000 linear +guess - random/zero/table garbage (the allocations are NON-CONTIGUOUS); +(c) the PDB walks of gpfifo+0x10000 (0x4501000->pa 0x125e91000, +0x4561000->pa 0x132032000, 0x49c1000->pa 0x11bddd000 via pdb 0x1152f4000) +- ALL THREE PAGES ALL ZERO. The CE userds were never written - and the +deeper reading: the SEC2 scrub flow is a CIRCULAR WAIT: the scrubber waits +for its previous scrub pushes to be CONSUMED (the auth-tag sema advances +only when the GPU runs the scrub) -> the consumption needs the CE pump -> +the CE pump needs a live userd -> the userd is written BY SUBMITS -> the +submits are held by the wait. In a working boot the guest's first scrub +push happens BEFORE the wait loop (the loop only waits when the slots are +full) - so in our boots the FIRST scrub push must have gone into the CE +ring, the service failed to consume it (ok=0/userd dark), and the client +now waits for a completion that will never come. + +**27. THE DIRECT BYPASS (next session, ranked).** (1) Write the auth-tag +semaphore in the CHANNEL BUFFER (memdesc) page directly: the scrub loop's +`_sec2utilsUpdateGetPtr` reads `channelReadChannelMemdesc(pChannel, +authTagBufSemaOffset)` = a u32 at memdesc+finishPayloadOffset+4; advancing +that value frees the slots and unblocks the client WITHOUT the CE pump. +The memdesc = the channel buffer allocation (a VA in the client's space); +resolve it via the PDB (the guest's channel struct gives the buffer VA; +the service's channel registration/alloc params carry it) and write the +incremented value from the service (or the host RAM file). (2) OR decode +the scrub pushes the service already sees (check sema-push for the SCRUBBER +channel - its submits carry the auth-tag VA; the release decode should +target it; the service's release applies would write the sema if the VA +resolves). (3) The registered FB userd offsets being zero suggests the +client's kernel maps the userd through a VA whose PA the discovery misses; +the diff-watch (two samples around activity) remains the fallback proof. + +**28. Session 101: THE LOOP SEMANTICS + perf trace (NO-PTRACE SYSCALL TRACING).** +The wait loop decoded exactly (sec2_utils.c:420-441): +`while (nextPut == pSec2Utils->authTagGetIndex) { gpuCheckTimeout(); +authTagGetIndex = _sec2utilsUpdateGetPtr(); osSpinLoop(); }` - it exits +when the memdesc sema (u32 at pChannelBufferMemdesc + +authTagBufSemaOffset; authTagBufSemaOffset = finishPayloadOffset + 4) +advances past nextPut; it also has a GPU timeout that would error out. +perf trace on the spun client (no ptrace!) revealed the live pattern: +tid 1793 = a 10ms-timed futex loop (WAKE val=1 then +WAIT_BITSET|CLOCK_REALTIME 10ms - the paced retry); tid 1788 = repeated +`openat -> read(fd, 299 bytes) -> close -> futex-wait` - a transient-fd +POLLING READER of a small blob (the fd is opened and closed each cycle, so +fd 28 no longer exists; the client's persistent fds: /dev/nvidia-modeset +(10), /dev/nvidia0 (11/12/14/15), GLCache .toc/.bin (16/17), memfd +\"allocation fd\" (18)). The 299-byte read is the likely userland face of +the memdesc read path (the 24% channelReadChannelMemdesc hot loop). +NEXT: (a) catch the transient fd path by sampling `ls -l /proc/<pid>/fd` +every ~0.2 s in a loop around the burst (or perf trace -s during a burst); +(b) if it is a driver read() of the memdesc, resolve its buffer and write +the auth-tag sema value (advance past nextPut) via the RAM file; (c) the +loop timeout (gpuCheckTimeout) may eventually error the client - the write +must land first. + +**29. Session 102: fd-sampler miss, no nvidia debugfs; the memdesc hunt narrows.** +The 0.15 s /proc/1549/fd sampler over 20 s caught NO transient path (the +open/read(299)/close reader is bursty or dormant between bursts; the static +fds are /dev/nvidiactl, /dev/nvidia0, /dev/nvidia-modeset, a socket, the +GLCache .toc/.bin pair, and a memfd "allocation fd"). No nvidia debugfs +tree exists in the guest. Remaining ranked paths to the auth-tag sema +(memdesc + finishPayloadOffset + 4): (1) content-locate the scrubber's +channel buffer by its method stream (the C5B5/C56F shapes: SET_OBJECT +0xc5c0/0xc5b5, headers 0x200121ed/0x200120a9 observed in the session-78 +window dumps) and read/write the sema inside; (2) the scrubber channel's +own sema-push decodes (its pushes carry the auth-tag VA = pbGpuVA + +authTagBufSemaOffset; grep the boot's sema-pushall for the scrubber +channel id and read the rel= VA - then PDB-walk it); (3) sample the fd +burst narrower (0.02 s) right after a client restart when the burst is +hot. The service-side ideal: on detecting the scrubber channel, resolve +its channel buffer and pre-advance the auth-tag sema (a durable fold). + +**30. Session 103: THE SCRUB STREAM FOUND; ONE RELEASE DELIVERED; THE SEMA PAGE STILL AMBIGUOUS.** +Content-located the CE/scrub init method stream (headers 0x200121ed/ +0x200120a9 searched): full stream at `pa=0x114e98000` = SET_OBJECT +0xC5C0/0xC5B5 (0x20012000/0x20012080 headers), engine-init methods +0x200121ed(3)/0x200120a9(1)/0x2001209c(0x0c000001), bulk methods +(0x8000xxxx), then TWO REPORT_SEM releases in one buffer: +`0x200426c0 00000000 04b19b10 ...` (va=0x4b19b10) and +`0x200426c0 00000000 04b17e30 ...` (va=0x4b17e30), tail 0x2001c00d +70/68. Twins at pa=0x1026ad000+0x87c and pa=0x12afba000+0x87c (identical +content at identical offsets - multi-mapped copies). Release resolution: +`0x4b17e30 -> pa=0x114efbe30 (SYS) cur=1` - THAT release WAS delivered (the +service applied it, cur=1). `0x4b19b10` resolves via the walker's big-half +2MB branch to FB 0x660b10 holding garbage (0xD9F26F94) - poke_vidmem +tests ([0x660b10]=1 then =0x40000000) did NOT flip the client, and the +linear-block guesses (buffer VA 0x4b00000 <-> stream pa 0x114e98000, sema +candidate 0x114eb1b10) read all zero. NEXT: the auth-tag sema page must be +picked by VALIDATION (the existing value must be a small plausible +GetPtr): walk 0x4b19000 returning ALL chains (small/64K/2MB both +apertures), pick the page whose u32 at +0x1b10 (or the offset the memdesc +actually uses) is a small number; or take the buffer aperture from the +channel's alloc params (the service's registration) and walk with that +aperture exclusively. The fix remains: advance that u32 (write via the +host RAM file if SYS, poke_vidmem if FB). + +**31. Session 104: THE SEMA VA 0x4b19b10 IS NOT THE LOOP'S PAGE; THE KPROBE IS THE ANSWER.** +The validated multi-chain walk (small-4KB and big chains BOTH) resolves +VA 0x4b19b10 -> FB 0x660b10 (both chains converge; the guest's own PTs say +the object lives in vidmem). poke_vidmem writes at 0x660b10 (0x1, +0x40000000) and 0x660b14 (0x1, 0x40000000) LANDED (verified via +sema_vidmem_dump: [0x660b10]=0x11112222) yet the client does not flip and +perf still shows channelReadChannelMemdesc at 24.52%. CONCLUSION: the +REPORT_SEM VA 0x4b19b10 (and 0x4b17e30) are NOT the auth-tag sema the loop +reads - they are the GL-init release objects; the real auth-tag sema is at +pChannelBufferMemdesc + finishPayloadOffset + 4 (a VA and page we do not +yet know; finishPayloadOffset is NOT 0x19b10). THE DEFINITIVE INSTRUMENT: +a guest kprobe on channelReadChannelMemdesc (the offset argument = %si = +the authTagBufSemaOffset VALUE; and the pChannel pointer lets us read the +channel struct -> pChannelBufferMemdesc): `echo 'p:crcd +channelReadChannelMemdesc offset=%si' >> /sys/kernel/tracing/kprobe_events` ++ enable the event + read tracing pipe WHILE the loop runs. This names the +offset and the memdesc in one shot, then the sema page = walk(memdesc+off) +and the advance (RAM file if SYS; FB + poke if the converged mapping). + +**32. Session 105: KPROBE BLOCKED (notrace); THE BAR1-PATH ANALYSIS; THE CHANNEL-FIELD OFFSETS.** +(1) The kprobe on channelReadChannelMemdesc is IMPOSSIBLE: the guest's +dmesg says `trace_kprobe: Could not probe notrace function +channelReadChannelMemdesc [nvidia]` - the function is annotated notrace +(ftrace/perf-probe/BPF-kprobe all blocked). The tracefs filesystem had to +be mounted first (mount -t tracefs tracefs /sys/kernel/tracing) and it +already carries UVM kprobes from earlier sessions (uvmt/uvch/uvbp/uvtw/ +uvsv...). The static sec2utils functions (the callers) are absent from +kallsyms (inlined) - no probe target on that path. +(2) BAR1-path analysis: the guest driver's memdesc read uses +TRANSFER_FLAGS_USE_BAR1|SHADOW_ALLOC|SHADOW_INIT_MEM - a CPU read of the +GUEST'S BAR1 mapping, which the mdev mmap maps onto the HOST's vidmem +shadow (the same buffer poke_vidmem writes). The pokes at FB 0x660b10/ +0x660b14 LANDED (verified) yet the loop did not advance, so the memdesc +page is NOT the FB 0x660b10 the PT walk converged on (VA 0x4b19b10 belongs +to the GL-init release object, not the channel-buffer memdesc). +(3) The channel registration parse exposes the guest-visible channel +fields and their alloc-param offsets (gsp_rm_emulation.c +gsp_rm_register_channel: GSPRM_CHAN_GPFIFO_OFFSET/ENTRIES, ENGINE_TYPE, +CID, INSTANCE_MEM, USERD_MEM(+APERTURE), NOTIFIER_MEM(+APERTURE)) - the +channel-buffer/memdesc field (pChannelBufferMemdesc) is not among them; +the next session should dump a CE channel's FULL alloc params (the +service's rate-limited alloc dumps or a fresh boot) and look for the +channel-buffer VA/PA field, then sema = buffer + finishPayloadOffset + 4 +(per-channel offset, e.g. the 0x19b10-style for the GL channel). +(4) Fallback if the field is not in the params: per-VA content hunt for +the auth-tag sema VALUE (the retprobe would have given it; without +kprobes: the sema value = a small slot counter - scan the CE channel's +allocation candidates for a plausible counter that CHANGES across two +samples). + +**33. Session 106: THE CHANNEL BUFFER IS NOT A CLIENT PARAM; THE TAIL SWEEP AND HASH SCAN NEGATIVE.** +NV_CHANNEL_ALLOC_PARAMS (alloc_channel.h:300-341) layout mapped to the +service offsets: gpFifoOffset@8/entries@16, engineType@128, cid@132, +instanceMem@144, userdMem@168, ramfcMem@192, mthdbufMem@216, +hPhysChannelGroup/internalFlags, errorNotifierMem@248... - THE CHANNEL +BUFFER (pChannelBufferMemdesc) IS NOT IN THE PARAMS: it is a +driver-internal memdesc created by channelAlloc, so the alloc-params path +cannot name it (the kprobe/debuginfo path was the right one; it is +notrace-blocked). The finish-payload tail of the stream page +(0x114e98000 +0xd4..+0x118) was swept with 0x40000000 (18 u32s; the +pre-sweep values: +0xd8=0x68000000 finish marker, +0xe0=0x901025a6, ++0xe4..+0xec=0x8000xxxx methods) - no flip. A 16-page high-entropy scan of +0x114e98000..0x114ea8000 for the auth-tag SHA-256 array: ZERO high-entropy +32-byte blocks (the hash array is elsewhere / another allocation). NEXT: +(1) the memmgrMemDescBeginTransfer SHADOW insight - the read happens from +a FRESH kmalloc'd shadow copied from the memdesc each call; the shadow's +source = the memdesc's backing page; the memdesc page must be found by +TRANSFER-TIME properties (not the VA walk); (2) or a guest perf hardware +watchpoint on the driver's BAR1 ioremap VA + the (unknown) memdesc offset; +(3) or the service-side broad assist: increment a RANGE of plausible FB +sema cells periodically (blind but bounded), watching for the flip. + +**34. Session 107: FB ASSIST SWEEPS NEGATIVE - THE READ IS NOT FB-SOURCED; THE TWINS ARE THE NEXT TARGET.** +Two bounded FB "sema assist" sweeps (distinct values per cell, +0x40000000|idx): (a) the walked page 0x660b00..0x661000 (320 cells) - no +flip; (b) the CE-channel allocation cluster 0x8f000..0x90000 (1024 cells, +where the registered userds live) - no flip; perf re-confirmed +channelReadChannelMemdesc 23.13% (the loop still runs). CONCLUSION: the +memdesc read does NOT come from the FB shadow the CPU can poke - the +transfer's SHADOW_INIT must read the memdesc through a path that bypasses +these bytes, or the memdesc backing is SYSTEM memory via an unknown VA. +NEXT (one-minute test): the STREAM TWINS - pa 0x1026ad000 and +0x12afba000 hold BYTE-IDENTICAL copies of the CE method stream (verified at ++0x7c0/+0x880: same 0x68000000 finish marker area, same 0x200120a9/9c +methods) - one of them is very likely the memdesc BACKING in sysmem (the +driver copies the same stream per channel/boot). Sweep BOTH twins' +stream-area pages (reuse the assist code; 1 page each) and watch for the +flip; if it flips, bisect the winning page to the sema cell and make it +durable (host RAM file writes = zero-cost, no poke_vidmem needed for +sysmem). Failing that: content-diff the twins across a driver transition +(the memdesc copy updates during submits; the copy that CHANGES is the +backing). + +**35. Session 108: THE CONTENT-DIFF - 9 ACTIVE PAGES + THE CONSTANT-READ DEDUCTION.** +Page-hash diff over three +/-256KB windows (twins 0x1026ad000/0x12afba000, +stream 0x114e98000) across 15 s: NINE pages change: +0x102690000, 0x102698000 (440 word diffs - a buffer written continuously = +the push copy!), 0x1026a7000 (+0xe8: 934 -> 940 - a slow clean counter), +0x1026b3000 (+0x48: 32768 -> 1 transition), 0x1026b5000 (+0x244/+0x24c: +24987 -> 32765, two u16s at ~650/s), 0x1026c2000 (+0xe8: 232 -> 3, a +wrapping counter mod ~233), 0x1026e6000 (+0x18: 2764 -> 1930, +0x88/+0x90 +decreasing - countdowns), 0x114e6b000, 0x114e6e000. THE DEDUCTION: the +stuck loop's read value must be CONSTANT (its page shows no writes; the +GPU never advances the sema), so the auth-tag sema is a static cell - +likely zero - and the fix is to make the read NONZERO (advance it) at the +right page. None of the 9 changing pages is it (their cells move on their +own). NEXT: (1) restrict the search to CONSTANT pages among the CE +channel's candidate backing (the twins region pages that DID NOT change +but sit adjacent to the active set); (2) target the sema cell: the cell +that equals nextPut (small constant, e.g. 0) at the memdesc+off; sweep the +CONSTANT pages' zero u32s in a bounded ring around the twins/active set +with 1 (the sysmem write is free); (3) the durable service fold once +found. + +**36. Session 109: THE BURST vs HEARTBEAT DISCOVERY + THE 200ms READER.** +perf trace of the whole process over 10 s with -e ioctl: ZERO ioctls from +ANY thread; meanwhile the process shows a strict 200 ms pattern on thread +1788: `openat(filename ptr 0x5eaec740) -> read(fd 28, count 299) -> close` +(the transient fd the fd-sampler missed!) and thread 1793's 10 ms +futex re-arms. The perf-reported 24% channelReadChannelMemdesc (a kernel +function only reachable via RM calls) CONTRADICTS the zero-ioctl window: +the hot kernel work is BURSTY (periodic rounds between the 200 ms +heartbeats), not continuous. The 299-byte blob read every 200 ms = the +client's health poll (unnamed path; candidate: /proc/driver/nvidia/... +informational node - find it by reading /proc/PID/io or by narrowing the +sampler to the 200 ms phase). THE MODEL NOW: the client idles (futex +waits) + wakes every 200 ms to poll a node + periodically runs RM bursts +(the burst = a full retry round hitting channelReadChannelMemdesc). NEXT: +(1) name the 299-byte node (it may BE the wait's condition - a status the +service must change); (2) capture the BURST with perf (a long record + +per-thread breakdown) to see which thread runs the memdesc reads (the RM +burst thread) and correlate with the host-side RPC traffic in the same +window (the host dmesg RPC lines during a burst = the exact RM call!). + +**37. Session 110: THE LOOP IS CONTINUOUS INSIDE ONE LONG IOCTL (THE LOCK MODEL CLOSED).** +(1) The 299-byte/200ms reader: the filename pointer 0x5eaec740 is NEVER in +the client's maps (30 Hz sampler over 15 s, zero hits) yet openat returns +28 every cycle - a decode artifact/transience, and at 5 reads/s it is +irrelevant to the 24% blocker. Treat it as a RED HERRING. (2) A 20-second +perf record (80K samples): channelReadChannelMemdesc 24.65% + a scheduler +family (pick_next_task_fair/update_curr/pick_eevdf/__schedule/ +schedule_timeout/os_schedule ~30% total) = the SEC2 wait loop running +CONTINUOUSLY. The earlier zero-ioctl windows are explained: THE MAIN +THREAD SITS INSIDE ONE LONG-LIVED ioctl (entered hours ago; the loop runs +in the kernel inside it) - and that ioctl HOLDS THE RM WRITE LOCK, which +closes the lock model: tid 1791 waits on the RM read lock, new clients +block D-state (os_acquire_rwlock_write), all consistent. (3) PEBS/ +mem-loads events are unavailable in the guest perf (no memory-address +sampling) - the read address cannot be sampled. Instruments for the sema +address are now exhausted short of the notrace blocker or the blind +service-side assist. +NEXT: (1) the blind assist, refined: the loop's READ is CONTINUOUS and +the sema is CONSTANT - sweep ONLY constant small-value cells in the CE +candidate pages (see session 108's active set for the region), one value +at a time, 2 s per batch, watching the flip; (2) or a graft: since the +loop runs inside the client's blocked ioctl, unloading the CLIENT (kill) +releases the RM lock and the fresh client restarts the same phase - use +fresh-client runs for any A/B of service-side fixes; (3) the durable +service-side fold remains the endgame once the cell is known. + +**38. Session 111: THE FB IS FULLY DISPROVED (WHOLE-FB FILL, NO FLIP); THE SYS-FILL WEDGED THE GUEST (CYCLE NEXT).** +(1) scan_vidmem for the stream signatures (0x200121ed, 0x200120a9) and even +0x68: ZERO hits across 256MB - THE FB SHADOW WAS ALL ZEROS (before any +pokes): the GPU-side objects are never written in our emulation (the +driver->GPU DMA never runs), so the driver's memdesc SHADOW_INIT copies +zeros and the sec2 loop reads 0 forever (nextPut==authTagGetIndex==0 -> +infinite wait; the complete model). (2) The refined assist: swept FB +0x80000..0x90000 (16384 cells, distinct values; 60s/32KB via poke_vidmem) +- no flip. (3) THE DEFINITIVE TEST: `fill_vidmem 0x40000000` filled the +WHOLE 256MB FB shadow in-kernel (`vgsp: fill_vidmem value=0x40000000 +size=0x10000000 done`) - STILL NO FLIP: THE LOOP'S READ DOES NOT COME FROM +THE FB SHADOW AT ALL; the channel-buffer memdesc is SYS-BACKED (or another +backing) in our emulation, not the FB. (4) The follow-up SYS fill (128 +candidate pages near the active set overwritten with 0x40000000 via the +RAM file) CLOBBERED LIVE DRIVER DATA and WEDGED THE GUEST (SSH banner +timeout; QEMU alive) - the next session MUST CYCLE (fresh boot); lesson: +never bulk-overwrite pages adjacent to the identified active set (0x102698000 +was a hot staging buffer). +NEXT: (1) fresh cycle; (2) the sema is a SYS page - apply the full-page +marker fill ONLY to genuinely-constant, non-active pages (re-run the +content-diff first, then fill the constant set in small batches with a +client check after each, NOT the active neighborhoods); (3) or the +service-side durable route: have the SERVICE write the marker into the +channel-buffer page once it can resolve it (the memdesc VA = the CE +channel's buffer; the service's channel registration fields + the PT walk +give the page candidates). diff --git a/tools/guest-diag/wall2/README.md b/tools/guest-diag/wall2/README.md new file mode 100644 index 0000000..1ef9558 --- /dev/null +++ b/tools/guest-diag/wall2/README.md @@ -0,0 +1,85 @@ +# WALL #2 (GR channel-init release) — session 73 findings + tools + +## Goal +The vk_frame Vulkan client fails `vkCreateDevice -> VKFAIL -4` because the +kernel RM kicks its GR channel (h=0xbeee0100, ring VA 0x44a0000, push VA +0x4480000) and waits ~11 s per attempt for a completion the sema service +cannot apply: it cannot translate the push VA ("false translation for push +va=0x4480000 ... rescuing"), so no release is decoded and the wait times out. + +## Findings (r674/r675, 8+ client runs) +1. The push is RM-built (nvidia.ko), NOT UVM: a kprobe on + `uvm_hal_turing_host_semaphore_release` never fired during a client run. +2. nvidia.ko / nvidia-modeset.ko functions are `notrace` -> `kprobe_events` + refuses them ("Could not probe notrace function ... [nvidia]"). UVM module + probes DO work. QEMU gdbstub hardware breakpoints (hbreak) work fine. +3. The service's rescue matcher is CE-shaped only (hdr 0x20038090, hi==1, + lo&0xfff==slot, w5/v5 shape) -> it can never match a GR push. Entries 0/1/2 + are consumed as "skipping stale entry ... after ~150 failed rescues"; + gr_wedge=1. +4. Host-side content searches of /dev/shm/vgpu-guest-ram during the stall found + NO push data: neither the CE release group (0x20038090) nor the host-class + release header (0x20050017) appears outside driver code pages. +5. Ring entry encoding (from `channelFillGpFifo` in nvidia.ko and + `nvWriteGpEntry` in nvidia-modeset.ko): [0] = va & ~3, [1] = + ((va>>32)&0xff) | ((len<<8)&0x7ffffc00) | 0x80000000 (flag bit seen on + entry[1]). +6. Neither `channelFillGpFifo` (nvidia.ko) nor `nvWriteGpEntry` + (nvidia-modeset.ko) is called during the client run (gdb hbreaks, 0 hits + for the GR channel) -> the GR channel's ring entries are written by an + unidentified writer (possibly the emulation/GSP path); the push content + was never located in guest RAM. +7. RM window records + memdescs (host-readable, stable layout): + - a page holds the client channel's window records + {VA=0x4480000, SIZE=0x20000} (push) and {VA=0x44a0000, SIZE=0x40000} + (ring), each inside head/tail list nodes; + - memdescs `{SIZE, 0x03008000, FIRST_PA, ptr}` live in the same page: + the ring memdesc's FIRST_PA == the service's validated ring pa + (verified twice: 0x1158ce000, 0x129415000) -> memdesc PA == the PA of the + window's first VA page; + - the push memdesc `{0x20000, 0x03008000, PA}` was injected as + 0x4480000 -> PA (resolver4, r675: PA 0x11595c000; "injected mapping + va=0x4480000 ... (sysmem)" in dmesg) and the service then read the page + WITHOUT the "false translation" line - but decoded no release (the page + content does not look like the method stream; later candidates read as + zero). +8. The kernel-side stall shows no GSP RPCs and no ioctls; the wait function + `channelWaitForFinishPayload` (polls `channelReadChannelMemdesc(channel, + channel->0x50)`, which reads `*(channel+0xe8)+off`) was NEVER called during + a client run (gdb hbreak) -> the RM's wait path for this kick is elsewhere. + +## Tools (this directory) +- wall2_resolver2.py — v2: CE release-group content search + tuple arrays + (pa&~0xfff, strides 0x20/0x40/0x60). No candidates live. +- wall2_resolver3.py — v3: ring memdesc -> same-page push memdesc recipe. +- wall2_resolver4.py — v4: window-record pair + ring memdesc cross-check + (this one injected successfully; rdesc[0] bug fixed in v5). +- wall2_resolver5.py — v5: continuous scan (no ring-line gate), injects every + 0x20000 memdesc PA found in pages holding both window records. +- wall2_livecap{,2,3}.py — live stall-time scanners for the CE group and the + host release header (both negative; useful for re-checks). +- scan_r673*.py — RAM structure analysis (ring-PA refs, record pages, memdesc + scans). +- trace_push674.py — VA-reference tracer with direct-map pointer following. +- gdb675*.gdb — QEMU gdbstub scripts (channelFillGpFifo / nvWriteGpEntry + probes, channel struct dumps). + +## Next steps (ranked) +1. Find the GR ring-entry writer (gdb watchpoint on the ring page's CPU VA; + get the VA from the channel struct; find the channel struct via the ring + memdesc page's head/tail pointers). The writer's caller holds the push's + CPU mapping -> PA -> inject -> service decodes the release. +2. Or: gdb-break at the stall on the RM's wait (sample the sleeping task's + kernel stack; the current candidates never fired) and read the semaphore + value it polls; bump it host-side via the ring-page PA (the finish payload + may live at ring_cpu + channel->0x50). +3. Or: fix the service's rescue matcher to accept GR-shaped releases once the + push content is captured (see item 1). + +## Ops +- pkill -f self-matches: the bracket trick fails when the pattern text appears + elsewhere in the agent's own command line; kill by explicit PID from + `ps -eo pid,comm,args | awk '$2=="python3" && /<script>/ {print $1}'`. +- r674 wedged after a gdb-interrupted run; fresh cycle r675 used (no host + reboot). gdb674d/gdb675d sessions left attached (never kill gdbs). +- Never reload the guest nvidia modules; never kill clients mid-flight. diff --git a/tools/guest-diag/wall2/diag76.py b/tools/guest-diag/wall2/diag76.py new file mode 100644 index 0000000..989abd3 --- /dev/null +++ b/tools/guest-diag/wall2/diag76.py @@ -0,0 +1,47 @@ +import gdb + +inf = gdb.selected_inferior() +pid = inf.pid +maps = [] +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + maps.append((lo, hi, off, parts[5] if len(parts) > 5 else "")) + except Exception: + pass + +def resolve(addr): + for lo, hi, off, name in maps: + if lo <= addr < hi: + return "%s+0x%x" % (name.split('/')[-1], off + (addr - lo)) + return "?" + +gdb.execute("set pagination off") +print("=== THREADS ===") +for t in inf.threads(): + try: + t.switch() + except Exception as e: + print("switch err", e) + continue + pc = int(gdb.parse_and_eval("$pc")) + print("TID %s pc=%s" % (t.ptid, resolve(pc))) + try: + bt = gdb.execute("bt 12", to_string=True) + except Exception as e: + bt = "bt err %s" % e + for ln in bt.splitlines(): + ln = ln.strip() + if ln.startswith("#"): + try: + a = int(ln.split()[1], 16) + print(" ", ln, "=>", resolve(a)) + except Exception: + print(" ", ln) + else: + print(" ", ln) +print("DIAG-DONE") +gdb.execute("detach") +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/fastrun.sh b/tools/guest-diag/wall2/fastrun.sh new file mode 100644 index 0000000..73b9d80 --- /dev/null +++ b/tools/guest-diag/wall2/fastrun.sh @@ -0,0 +1,10 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /tmp/slotfill2.log 2>&1 & ) +sleep 0.5 +sudo timeout 60 strace -f -T -tt -e trace=ioctl -o /tmp/vk_strace3.out /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk77.out 2>&1 +echo "=== client ===" +cat /tmp/vk77.out +echo "=== last ioctls ===" +tail -12 /tmp/vk_strace3.out diff --git a/tools/guest-diag/wall2/fastrun2.sh b/tools/guest-diag/wall2/fastrun2.sh new file mode 100644 index 0000000..3c7b943 --- /dev/null +++ b/tools/guest-diag/wall2/fastrun2.sh @@ -0,0 +1,12 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /dev/null 2>&1 & ) +sleep 0.5 +sudo timeout 45 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk78.out 2>&1 +echo "=== client ===" +cat /tmp/vk78.out +echo "=== repeated controls (top) ===" +sudo dmesg -T | grep -oE "GSP_RM_CONTROL cmd=0x[0-9a-f]+" | sort | uniq -c | sort -rn | head -12 +echo "=== last controls ===" +sudo dmesg -T | grep "GSP_RM_CONTROL cmd" | tail -8 diff --git a/tools/guest-diag/wall2/find_ring_va.py b/tools/guest-diag/wall2/find_ring_va.py new file mode 100644 index 0000000..c02ea40 --- /dev/null +++ b/tools/guest-diag/wall2/find_ring_va.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Guest-side: read the nv_vm_map_pages vmap areas via /proc/kcore and find the +# GPFIFO entry pattern (entry0 raw 0x0005980004480000 for va=0x4480000 len=1432). +import struct, sys +pat = bytes.fromhex("0000480400980500") # 0x0005980004480000 LE +class Kcore: + def __init__(self): + self.fd = open("/proc/kcore", "rb") + hdr = self.fd.read(64) + e_phoff = struct.unpack_from("<Q", hdr, 32)[0] + e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54) + self.segs = [] + for i in range(e_phnum): + self.fd.seek(e_phoff + i * e_phentsize) + ph = self.fd.read(56) + p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from("<IIQQQQ", ph, 0) + if p_type == 1: + self.segs.append((p_va, p_va + p_sz, p_off)) + def read(self, va, n): + for lo, hi, off in self.segs: + if lo <= va and va + n <= hi: + self.fd.seek(off + (va - lo)) + return self.fd.read(n) + return None +kc = Kcore() +areas = [] +for line in open("/proc/vmallocinfo"): + p = line.split() + if len(p) < 2: continue + if "nv_vm_map_pages" not in line: continue + lo, hi = p[0].split("-") + areas.append((int(lo, 16), int(hi, 16))) +print("nv_vm_map_pages areas:", len(areas)) +for lo, hi in areas: + size = hi - lo + d = kc.read(lo, size) + if not d: + print(" 0x%x-0x%x unreadable" % (lo, hi)); continue + idx = d.find(pat) + print(" 0x%x-0x%x size=0x%x hit=%s" % (lo, hi, size, ("+0x%x -> VA 0x%x" % (idx, lo + idx)) if idx >= 0 else "no")) + if idx >= 0: + # dump the page around the hit + base = lo + (idx & ~0xfff) + page = kc.read(base, 0x100) + print(" page head: %s" % page.hex()) diff --git a/tools/guest-diag/wall2/gdb675.gdb b/tools/guest-diag/wall2/gdb675.gdb new file mode 100644 index 0000000..e4b73ce --- /dev/null +++ b/tools/guest-diag/wall2/gdb675.gdb @@ -0,0 +1,21 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc0d0b9f0 +condition 1 *(unsigned int*)($rdi+0x110) == 0xc46f +commands 1 +silent +printf "GRFILL ch=%p idx=%d len=%d\n", $rdi, $esi, $edx +printf " +0x048=%p +0x0b8=%p +0x0e8=%p\n", *(unsigned long*)($rdi+0x48), *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0xe8) +printf " +0x060=%p +0x0c0=%p +0x0c8=%p +0x0d0=%p +0x0d8=%p\n", *(unsigned long*)($rdi+0x60), *(unsigned long*)($rdi+0xc0), *(unsigned long*)($rdi+0xc8), *(unsigned long*)($rdi+0xd0), *(unsigned long*)($rdi+0xd8) +printf " +0x0e0=%p +0x0f0=%p +0x0f8=%p +0x108=%p +0x140=%p +0x148=%p\n", *(unsigned long*)($rdi+0xe0), *(unsigned long*)($rdi+0xf0), *(unsigned long*)($rdi+0xf8), *(unsigned long*)($rdi+0x108), *(unsigned long*)($rdi+0x140), *(unsigned long*)($rdi+0x148) +x/8gx $rdi+0x40 +x/8gx $rdi+0x80 +x/8gx $rdi+0xc0 +x/8gx $rdi+0xe0 +x/8gx $rdi+0x100 +x/8gx $rdi+0x140 +continue +end +continue diff --git a/tools/guest-diag/wall2/gdb675c.gdb b/tools/guest-diag/wall2/gdb675c.gdb new file mode 100644 index 0000000..25b5ca0 --- /dev/null +++ b/tools/guest-diag/wall2/gdb675c.gdb @@ -0,0 +1,12 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc0d0b9f0 +commands 1 +silent +printf "FILL ch=%p cls100=0x%x b8=0x%x stride48=0x%x idx=%d len=%d\n", $rdi, *(unsigned int*)($rdi+0x100), *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0x48), $esi, $edx +printf " d0=%p d8=%p e8=%p f8=%p 60=%p\n", *(unsigned long*)($rdi+0xd0), *(unsigned long*)($rdi+0xd8), *(unsigned long*)($rdi+0xe8), *(unsigned long*)($rdi+0xf8), *(unsigned long*)($rdi+0x60) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdb675d.gdb b/tools/guest-diag/wall2/gdb675d.gdb new file mode 100644 index 0000000..f9dc460 --- /dev/null +++ b/tools/guest-diag/wall2/gdb675d.gdb @@ -0,0 +1,11 @@ +set pagination off +set confirm off +set print elements 0 +target remote :1234 +hbreak *0xffffffffc1605fd0 +commands 1 +silent +printf "NVWRITE ch=%p idx=%u len=%u b8=0x%x f68=%p f50=0x%x fb0=0x%x fc0=0x%x f198=%p\n", $rdi, $esi, $edx, *(unsigned long*)($rdi+0xb8), *(unsigned long*)($rdi+0x68), *(unsigned int*)($rdi+0x50), *(unsigned int*)($rdi+0xb0), *(unsigned int*)($rdi+0xc0), *(unsigned long*)($rdi+0x198) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbfill.gdb b/tools/guest-diag/wall2/gdbfill.gdb new file mode 100644 index 0000000..41d1ea3 --- /dev/null +++ b/tools/guest-diag/wall2/gdbfill.gdb @@ -0,0 +1,13 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +set $e = *(unsigned long*)($rdi+0x1428) +if *(unsigned short*)($e+0x2e) == 0xffff + set {unsigned short}($e+0x2e) = 0 + printf "FILLED slot at %p+0x2e -> 0\n", $e +end +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbgap3.sh b/tools/guest-diag/wall2/gdbgap3.sh new file mode 100644 index 0000000..bf19370 --- /dev/null +++ b/tools/guest-diag/wall2/gdbgap3.sh @@ -0,0 +1,28 @@ +#!/bin/bash +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk65.out 2>&1 &) +sleep 5 +PID=$(pgrep -x vk_frame | head -1) +cp /proc/$PID/maps /tmp/maps_d.txt +gdb -p $PID -batch -ex "set pagination off" -ex "bt 12" 2>&1 | grep -E "^#" > /tmp/bt_gap3.txt +python3 - <<'PY' +import re +maps=open('/tmp/maps_d.txt').read().splitlines() +segs=[] +for m in maps: + p=m.split() + if len(p)<6: continue + lo,hi=[int(x,16) for x in p[0].split('-')] + off=int(p[2],16) + segs.append((lo,hi,off,p[5])) +for line in open('/tmp/bt_gap3.txt'): + m=re.match(r'#(\d+)\s+0x([0-9a-f]+)', line) + if not m: continue + a=int(m.group(2),16) + for lo,hi,off,name in segs: + if lo<=a<hi: + print('#%s %s + 0x%x' % (m.group(1), name.split('/')[-1], off+(a-lo))) + break + else: + print('#%s 0x%x unmapped' % (m.group(1), a)) +PY diff --git a/tools/guest-diag/wall2/gdbparam.sh b/tools/guest-diag/wall2/gdbparam.sh new file mode 100644 index 0000000..3f30d49 --- /dev/null +++ b/tools/guest-diag/wall2/gdbparam.sh @@ -0,0 +1,16 @@ +#!/bin/bash +cd /tmp +sudo dmesg -C +( sudo bash /tmp/slotfill2.sh > /dev/null 2>&1 & ) +sleep 0.5 +(timeout 60 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk79.out 2>&1 &) +sleep 5 +PID=$(pgrep -x vk_frame | head -1) +gdb -p $PID -batch \ + -ex "set pagination off" \ + -ex "catch syscall ioctl" \ + -ex "condition 1 \$rsi == 0xc020462a" \ + -ex "continue" \ + -ex "printf \"ioctl nr=0x2a arg=%p\n\", \$rdx" \ + -ex "x/40wx \$rdx" \ + -ex "detach" 2>/dev/null | tail -45 diff --git a/tools/guest-diag/wall2/gdbpoll6.gdb b/tools/guest-diag/wall2/gdbpoll6.gdb new file mode 100644 index 0000000..d7fccfd --- /dev/null +++ b/tools/guest-diag/wall2/gdbpoll6.gdb @@ -0,0 +1,9 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +printf "CHK rsi=%d cnt=%d e0=%p v=%x\n", $rsi, *(int*)($rdi+0x22efc), *(unsigned long*)($rdi+0x1428), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)+($rsi<<4)) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbspin.sh b/tools/guest-diag/wall2/gdbspin.sh new file mode 100644 index 0000000..989e584 --- /dev/null +++ b/tools/guest-diag/wall2/gdbspin.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk57.out 2>&1 &) +sleep 25 +PID=$(pgrep -x vk_frame | head -1) +echo "pid=$PID" +BASE=$(grep 'eglcore' /proc/$PID/maps | head -1 | cut -d- -f1) +echo "base=$BASE" +SPIN=$(printf '0x%x' $((0x$BASE + 0xa10e90))) +echo "spin=$SPIN" +gdb -p $PID -batch \ + -ex "set pagination off" \ + -ex "break *$SPIN" \ + -ex "continue" \ + -ex "printf \"HIT rip=%p rdi=%p rsi=%p rdx=%p rbx=%p rbp=%p\n\", \$rip, \$rdi, \$rsi, \$rdx, \$rbx, \$rbp" \ + -ex "bt" \ + -ex "info registers r12 r13 r14 r15" \ + -ex "x/12i \$rip-24" \ + -ex "detach" 2>&1 | tail -40 diff --git a/tools/guest-diag/wall2/gdbu16.gdb b/tools/guest-diag/wall2/gdbu16.gdb new file mode 100644 index 0000000..61f335b --- /dev/null +++ b/tools/guest-diag/wall2/gdbu16.gdb @@ -0,0 +1,9 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +printf "CHK rsi=%d cnt=%d e0=%p u16@2e=%x u32@20=%x u32@0=%x\n", $rsi, *(int*)($rdi+0x22efc), *(unsigned long*)($rdi+0x1428), *(unsigned short*)(*(unsigned long*)($rdi+0x1428)+0x2e), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)+0x20), *(unsigned int*)(*(unsigned long*)($rdi+0x1428)) +continue +end +continue diff --git a/tools/guest-diag/wall2/gdbwatch.gdb b/tools/guest-diag/wall2/gdbwatch.gdb new file mode 100644 index 0000000..36b3339 --- /dev/null +++ b/tools/guest-diag/wall2/gdbwatch.gdb @@ -0,0 +1,18 @@ +set pagination off +set confirm off +break *POLLADDR +commands 1 +silent +set $e = *(unsigned long*)($rdi+0x1428) +watch *(unsigned short*)($e+0x2e) +printf "WATCH set at %p+0x2e\n", $e +disable 1 +continue +end +commands 2 +silent +printf "WRITE to slot+0x2e: old=%x new=%x by pc=%p\n", *(unsigned short*)($e+0x2e), $eax, $pc +bt 3 +continue +end +continue diff --git a/tools/guest-diag/wall2/gr_walk.py b/tools/guest-diag/wall2/gr_walk.py new file mode 100644 index 0000000..cbdb02d --- /dev/null +++ b/tools/guest-diag/wall2/gr_walk.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# gr_walk.py - find the RM window memdesc records in the guest's own RAM +# (via /proc/kcore direct map) and walk their kernel page-list pointers in +# the SAME pass (addresses go stale otherwise). Prints VA->PA candidates. +import sys, struct +sys.path.insert(0, "/tmp") +import guest_pin_stream as gps + +PAT_SIZE = struct.pack("<Q", 0x20000) +PAT_FLAGS = struct.pack("<Q", 0x03008000) + +def main(): + kc = gps.Kcore() + if not kc.load(): + print("# kcore load failed"); return 1 + pov = kc.page_offset_base + print("# page_offset_base=0x%x" % pov) + found = [] + CH = 1 << 20 + pa = 0x100000000 + end = 0x180000000 + while pa < end and len(found) < 12: + va = pov + pa + d = kc.read(va, CH) + if d: + st = 0 + while True: + i = d.find(PAT_SIZE, st) + if i < 0: + break + st = i + 1 + if i + 16 <= len(d) and d[i+8:i+16] == PAT_FLAGS: + found.append((pa + i, d[i:i+0x60])) + pa += CH + print("# memdesc records: %d" % len(found)) + for (gpa, rec) in found: + size, flags = struct.unpack_from("<QQ", rec, 0) + wpa = struct.unpack_from("<Q", rec, 0x10)[0] + kptrs = [struct.unpack_from("<Q", rec, o)[0] for o in (0x18, 0x20, 0x28, 0x30, 0x38)] + print("rec@0x%x size=0x%x pa=0x%x kptrs=%s" % + (gpa, size, wpa, ["0x%x" % k for k in kptrs])) + # walk each kptr for PA-like lists + for kp in kptrs: + if not gps.is_kern_ptr(kp): + continue + dd = kc.read(kp, 0x80) + if not dd: + print(" kptr 0x%x: unreadable" % kp); continue + words = struct.unpack_from("<%dQ" % (len(dd)//8), dd, 0) + pas = [w for w in words if 0x100000000 <= w < 0x180000000] + npas = [w for w in words if 0x100000000 <= (w & ~0xfff) < 0x180000000] + print(" kptr 0x%x: %s" % (kp, " ".join("0x%x" % w for w in words[:10]))) + if len(npas) >= 2: + print(" PA-like: %s" % ["0x%x" % w for w in npas[:16]]) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/gr_walk2.py b/tools/guest-diag/wall2/gr_walk2.py new file mode 100644 index 0000000..7a50bc8 --- /dev/null +++ b/tools/guest-diag/wall2/gr_walk2.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# gr_walk2.py - deeper walk: from the window memdesc records, follow the +# kernel pointers 2-3 levels and search for page-array records +# {vmap_cpu, flags, PA, 0x1000} (2+ consecutive) to recover the page lists. +import sys, struct +sys.path.insert(0, "/tmp") +import guest_pin_stream as gps + +PAT_SIZE = struct.pack("<Q", 0x20000) +PAT_FLAGS = struct.pack("<Q", 0x03008000) + +def is_pa(v): + return 0x100000000 <= v < 0x180000000 + +def page_array_at(kc, va, n=0x400): + """Return [(off, [PA,...])] for runs of {ptr,flags,PA,0x1000} records.""" + d = kc.read(va, n) + if not d: + return [] + out = [] + i = 0 + while i + 32 <= len(d): + p, fl, pa, z = struct.unpack_from("<4Q", d, i) + if z == 0x1000 and is_pa(pa) and gps.is_kern_ptr(p): + run = [pa] + j = i + 32 + while j + 32 <= len(d): + p2, fl2, pa2, z2 = struct.unpack_from("<4Q", d, j) + if z2 == 0x1000 and is_pa(pa2): + run.append(pa2) + j += 32 + else: + break + if len(run) >= 2: + out.append((i, run)) + i = j + else: + i += 8 + return out + +def main(): + kc = gps.Kcore() + if not kc.load(): + print("# kcore load failed"); return 1 + pov = kc.page_offset_base + # find memdesc records (fast direct-map scan) + found = [] + CH = 1 << 20 + pa = 0x100000000 + while pa < 0x180000000 and len(found) < 12: + d = kc.read(pov + pa, CH) + if d: + st = 0 + while True: + i = d.find(PAT_SIZE, st) + if i < 0: + break + st = i + 1 + if i + 16 <= len(d) and d[i+8:i+16] == PAT_FLAGS: + found.append((pa + i, d[i:i+0x40])) + pa += CH + print("# memdesc records: %d" % len(found)) + for (gpa, rec) in found: + size, flags = struct.unpack_from("<QQ", rec, 0) + wpa = struct.unpack_from("<Q", rec, 0x10)[0] + print("rec@0x%x size=0x%x pa=0x%x" % (gpa, size, wpa)) + kptrs = [struct.unpack_from("<Q", rec, o)[0] for o in range(0x18, 0x40, 8)] + for kp in kptrs: + if not gps.is_kern_ptr(kp): + continue + # level 1: read the target, look for page arrays directly + for arr in page_array_at(kc, kp): + print(" L1 kptr 0x%x: page-array run at +0x%x: %s" % + (kp, arr[0], ["0x%x" % x for x in arr[1][:12]])) + # level 2: follow pointers in the target + d = kc.read(kp, 0x100) + if not d: + continue + for o in range(0, 0x100, 8): + p2 = struct.unpack_from("<Q", d, o)[0] + if not gps.is_kern_ptr(p2): + continue + for arr in page_array_at(kc, p2): + print(" L2 0x%x -> 0x%x: page-array at +0x%x: %s" % + (kp, p2, arr[0], ["0x%x" % x for x in arr[1][:12]])) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/guest_words.sh b/tools/guest-diag/wall2/guest_words.sh new file mode 100755 index 0000000..6ac200c --- /dev/null +++ b/tools/guest-diag/wall2/guest_words.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# guest_words.sh - print "VA PA" for the +0x80 word of every 16 KB rw-s +# /dev/nvidiactl mapping of the vk_frame client (pagemap PFN -> PA). +PID=$(pgrep -x vk_frame | head -1) +[ -z "$PID" ] && exit 0 +sudo grep -E 'rw-s' /proc/$PID/maps | grep nvidiactl | while read -r line; do + RNG=$(echo "$line" | awk '{print $1}') + LO=${RNG%-*}; HI=${RNG#*-} + SZ=$((0x$HI - 0x$LO)) + [ "$SZ" -ne 16384 ] && continue + VP=$(( (0x$LO + 0x80) / 4096 )) + ENT=$(sudo dd if=/proc/$PID/pagemap bs=8 skip=$VP count=1 2>/dev/null | xxd -e -g8 | awk '{print $2}') + [ -z "$ENT" ] && continue + PFN=$(( 0x$ENT & 0x7fffffffffffff )) + PA=$(( PFN * 4096 + 0x80 )) + echo "0x$LO 0x$(printf %x $PA)" +done diff --git a/tools/guest-diag/wall2/host_resolve.py b/tools/guest-diag/wall2/host_resolve.py new file mode 100644 index 0000000..258d994 --- /dev/null +++ b/tools/guest-diag/wall2/host_resolve.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# host_resolve.py - host-side push resolver: read the service's sema-pushall +# lines (pa=0x0, non-zero first8), find the content page in +# /dev/shm/vgpu-guest-ram by the push's first8 bytes, and inject the page +# mapping via sema_map_add. Replaces the guest-side kcore walker: the +# service already tells us push VA + content prefix, the host RAM file has +# the bytes. +import mmap, os, re, subprocess, sys, time + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +BASE = 0x80000000 # file offset = pa - BASE + +def inject(va_page, pa): + cmd = "echo 0x%x,0x%x,0x1000,1 > %s" % (va_page, pa, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + ok = r.returncode == 0 + print("inject 0x%x -> 0x%x : %s" % (va_page, pa, "ok" if ok else r.stderr.decode().strip())) + return ok + +def main(): + seen = set() + f = open(RAM, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + injected = 0 + deadline = time.time() + (float(sys.argv[1]) if len(sys.argv) > 1 else 600) + while time.time() < deadline and injected < 40: + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + lines = [l for l in out.splitlines() if "sema-pushall" in l] + for l in lines[-800:]: + m = re.search(r"va=(0x[0-9a-f]+) pa=0x0 len=\d+ allow=\d trusted=\d first8=([0-9a-f]{8}) ([0-9a-f]{8})", l) + if not m: + continue + va = int(m.group(1), 16) + w0, w1 = int(m.group(2), 16), int(m.group(3), 16) + if (w0, w1) == (0, 0) and (w0 == 0 or w1 == 0): + pass + if w0 == 0 and w1 == 0: + continue + key = (va & ~0xFFF, w0, w1) + if key in seen: + continue + pat = w0.to_bytes(4, "little") + w1.to_bytes(4, "little") + inpage = va & 0xFFF + # search with the in-page offset constraint + start = 0 + hit = None + while True: + i = mm.find(pat, start) + if i < 0: + break + if (i & 0xFFF) == inpage: + hit = i + break + start = i + 1 + if hit is None: + # fallback: any hit within the same page offset modulo + start = 0 + while True: + i = mm.find(pat, start) + if i < 0: + break + hit = i + break + # proceed only if the constraint is loose? no - skip + hit = None + if hit is None: + seen.add(key) + print("no match for va=0x%x first8=%08x %08x" % (va, w0, w1)) + continue + pa = (hit & ~0xFFF) + BASE + seen.add(key) + if inject(va & ~0xFFF, pa): + injected += 1 + time.sleep(0.5) + mm.close() + f.close() + print("done: injected=%d" % injected) + +if __name__ == "__main__": + main() diff --git a/tools/guest-diag/wall2/live_walk.py b/tools/guest-diag/wall2/live_walk.py new file mode 100644 index 0000000..48f131f --- /dev/null +++ b/tools/guest-diag/wall2/live_walk.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# live_walk.py - live PT-walk + inject loop for the running client. +# Polls dmesg for the published page directory (pdb) and for the VAs the +# service is failing to read (sema-entry: unread / entry[] lines / registered +# gpfifo), walks each VA through the pdb with the service's own GMMU math, +# and injects resolved pairs via sema_map_add (ap=1 sysmem). +import mmap, re, subprocess, sys, time + +F = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +SEMA_PA_MAX = 0x180000000 + +WINDOW_VAS = [0x44a0000, 0x4500000, 0x4560000, # rings + 0x4480000, 0x4485000, 0x4ca0000, 0x4ca5000, + 0x4d00000, 0x4b17000, 0x4020000, 0x4100000, 0x4160000, + 0x4213000, 0x45a0bb0, # ctx-init releases (deterministic) + 0x124935060, 0x102e34060, 0x108d4f060, 0x12a3a4060, 0x116c0b060, # notif families + 0x20400f000, 0x20400ffc0, 0x20480fef0, 0x204827f7c] # counter pages / tracker VAs + +mm = None +def off(pa): return pa if pa < 0x80000000 else pa - 0x80000000 +def rd64(pa): + if pa + 8 > SEMA_PA_MAX: return None + d = mm[off(pa):off(pa)+8] + return int.from_bytes(d, "little") if len(d) == 8 else None +def pde_ap(ap): return "FB" if ap == 1 else ("SYS" if ap in (2, 3) else None) +def pte_ap(ap): return "FB" if ap == 0 else (None if ap == 1 else "SYS") +def pte_addr(e, fb): return ((e >> 8) & 0x1FFFFFF) << 12 if fb else ((e >> 8) & 0x3FFFFFFFFFFF) << 12 +def pte_leaf(table, idx): + e = rd64(table + 8 * idx) + if e is None or not (e & 1): return None + ap = pte_ap((e >> 1) & 3) + if ap is None: return None + a = pte_addr(e, ap == "FB") + return (a, ap) if a + 8 <= SEMA_PA_MAX else None + +def walk(pdb, va, entries=4): + import math + root_bits = max(1, min(12, int(math.log2(entries or 1)))) + table = pdb + for level in range(3): + if level == 0: + bits, shift = root_bits, 49 - root_bits + else: + bits, shift = 9, 38 if level == 1 else 29 + idx = (va >> shift) & ((1 << bits) - 1) + e = rd64(table + 8 * idx) + if e is None or (e & 1): return None + ap = pde_ap((e >> 1) & 3) + if ap is None: return None + table = pte_addr(e, ap == "FB") + idx = (va >> 21) & 0xff + big = rd64(table + 16 * idx); small = rd64(table + 16 * idx + 8) + if big is None or small is None: return None + if not (small & 1): + sap = pde_ap((small >> 1) & 3) + if sap is not None: + r = pte_leaf(pte_addr(small, sap == "FB"), (va >> 12) & 0x1ff) + if r: return r[0] + (va & 0xfff), r[1] + if big & 1: + bap = pte_ap((big >> 1) & 3) + if bap is not None: + a = pte_addr(big, bap == "FB") + (va & ((1 << 21) - 1)) + if a + 8 <= SEMA_PA_MAX: return a, bap + else: + bap = pde_ap((big >> 1) & 3) + if bap is not None: + r = pte_leaf(pte_addr(big, bap == "FB"), (va >> 16) & 0x1f) + if r: return r[0] + (va & 0xffff), r[1] + return None + +def inject(va, pa, ln=0x1000): + cmd = "echo 0x%x,0x%x,0x%x,1 > %s" % (va & ~0xfff, pa & ~0xfff, ln, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print(" INJECT 0x%x -> 0x%x len=0x%x : %s" % + (va & ~0xfff, pa & ~0xfff, ln, "ok" if r.returncode == 0 else "FAIL"), flush=True) + return r.returncode == 0 + +def main(): + global mm + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 300 + end = time.time() + dur + pdbs = [] # (pdb_pa, entries) + done = set() # (pdb, va_page) + seen_va = set() + last_lines = 0 + print("# live_walk start (%.0fs)" % dur, flush=True) + while time.time() < end: + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + lines = out.splitlines()[-500:] + out = "\n".join(lines) + # pdb offers/registrations + for m in re.finditer(r"page directory (?:offered as candidate )?pdb=0x([0-9a-f]+) entries=(\d+)", out): + pdb, ent = int(m.group(1), 16), int(m.group(2)) + if (pdb, ent) not in pdbs: + pdbs.append((pdb, ent)) + print("# pdb 0x%x entries=%u" % (pdb, ent), flush=True) + # VAs the service is failing on + vas = [] + for m in re.finditer(r"sema-entry: unread h=0x[0-9a-f]+ idx=\d+ va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"sema-chan: entry\[\d+\] h=0x[0-9a-f]+ eng=\d+ raw=[0-9a-f]+ va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"registered gpfifo=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"GR ring h=0x[0-9a-f]+ rank=\d+ not at fb 0x[0-9a-f]+ yet \(e0=0x[0-9a-f]+ e1=0x[0-9a-f]+ va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"rel=0x([0-9a-f]+) ", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"notif=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"counter page 0x([0-9a-f]+) unreadable", out): + vas.append(int(m.group(1), 16)) + for m in re.finditer(r"release (?:learned|derived) h=0x[0-9a-f]+ pa=0x[0-9a-f]+ \(from release va=0x([0-9a-f]+)", out): + vas.append(int(m.group(1), 16)) + vas = [v for v in vas if 0x4000000 <= v < 0x2000000000] + for va in WINDOW_VAS + vas: + seen_va.add(va) + # resolve + inject through every pdb + for (pdb, ent) in pdbs: + for va in list(seen_va): + key = (pdb, va & ~0xfff) + if key in done: + continue + r = walk(pdb, va, ent) + if not r: + continue + done.add(key) + pa, ap = r + if ap != "SYS": + print("# va=0x%x -> pa=0x%x ap=%s (skip non-sys)" % (va, pa, ap), flush=True) + continue + print("# walk ok pdb=0x%x va=0x%x -> pa=0x%x" % (pdb, va, pa), flush=True) + inject(va, pa) + time.sleep(0.4) + mm.close(); f.close() + print("# live_walk done", flush=True) + +if __name__ == "__main__": + main() diff --git a/tools/guest-diag/wall2/nudge.sh b/tools/guest-diag/wall2/nudge.sh new file mode 100755 index 0000000..143b1cd --- /dev/null +++ b/tools/guest-diag/wall2/nudge.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# nudge.sh - host-side readiness-word nudger for the vk_frame wait. +# Polls the guest for the 16 KB nvidiactl mapping words (guest_words.sh), +# and writes 1 to any that are still 0 in the guest RAM file. +# Usage: nudge.sh <duration_seconds> +DUR=${1:-300} +F=/dev/shm/vgpu-guest-ram +END=$(( $(date +%s) + DUR )) +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/guest_words.sh ubuntu@127.0.0.1:/tmp/guest_words.sh >/dev/null 2>&1 +while [ "$(date +%s)" -lt "$END" ]; do + OUT=$(timeout 20 sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 "sudo bash /tmp/guest_words.sh" 2>/dev/null) + echo "[$(date +%H:%M:%S)] words:"; echo "$OUT" + echo "$OUT" | while read -r VA PA; do + [ -z "$PA" ] && continue + PAD=$((PA)); FOD=$((PAD - 0x80000000)) + [ $FOD -lt 0 ] && continue + CUR=$(sudo dd if=$F bs=4 skip=$((FOD/4)) count=1 2>/dev/null | xxd -p) + if [ "$CUR" = "00000000" ]; then + printf '\x01\x00\x00\x00' | sudo dd of=$F bs=1 seek=$FOD count=4 conv=notrunc 2>/dev/null + echo " NUDGE $PA (va=$VA) -> 1" + else + echo " skip $PA (va=$VA) = $CUR" + fi + done + sleep 2 +done +echo NUDGE-DONE diff --git a/tools/guest-diag/wall2/nudge2.sh b/tools/guest-diag/wall2/nudge2.sh new file mode 100755 index 0000000..16ebcd8 --- /dev/null +++ b/tools/guest-diag/wall2/nudge2.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# nudge2.sh - widened readiness-word nudger: ALL rw-s nvidia mappings <=256K, +# writes 1 to any +0x80 u32 that is 0. +DUR=${1:-300} +F=/dev/shm/vgpu-guest-ram +END=$(( $(date +%s) + DUR )) +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/guest_words2.sh ubuntu@127.0.0.1:/tmp/guest_words2.sh >/dev/null 2>&1 +while [ "$(date +%s)" -lt "$END" ]; do + OUT=$(timeout 12 sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 "sudo bash /tmp/guest_words2.sh" 2>/dev/null) + [ -n "$OUT" ] && echo "[$(date +%H:%M:%S)] words:" && echo "$OUT" + echo "$OUT" | while read -r VA PA SZ; do + [ -z "$PA" ] && continue + PAD=$((PA)); FOD=$((PAD - 0x80000000)); [ $FOD -lt 0 ] && FOD=$((PAD)) + CUR=$(sudo dd if=$F bs=4 skip=$((FOD/4)) count=1 2>/dev/null | xxd -p) + if [ "$CUR" = "00000000" ]; then + printf '\x01\x00\x00\x00' | sudo dd of=$F bs=1 seek=$FOD count=4 conv=notrunc 2>/dev/null + echo " NUDGE va=$VA pa=$PA -> 1" + fi + done + sleep 1.5 +done +echo NUDGE2-DONE diff --git a/tools/guest-diag/wall2/perfstall5.sh b/tools/guest-diag/wall2/perfstall5.sh new file mode 100644 index 0000000..ae1c161 --- /dev/null +++ b/tools/guest-diag/wall2/perfstall5.sh @@ -0,0 +1,9 @@ +#!/bin/sh +cd /tmp +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk55.out 2>&1 &) +sleep 25 +PID=$(pgrep -x vk_frame | head -1) +cp /proc/$PID/maps /tmp/vk_maps2.txt +sudo perf record -p $PID -g -o /tmp/perf6.data -- sleep 3 2>/dev/null +sudo perf script -i /tmp/perf6.data -F ip,dso 2>/dev/null | grep eglcore | awk '{print $1}' | sort | uniq -c | sort -rn | head -8 > /tmp/spin_addrs.txt +cat /tmp/spin_addrs.txt diff --git a/tools/guest-diag/wall2/pt_bind.py b/tools/guest-diag/wall2/pt_bind.py new file mode 100644 index 0000000..126ed01 --- /dev/null +++ b/tools/guest-diag/wall2/pt_bind.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# pt_bind.py - bind GMMU PT pages to VA bases and read out VA->PA for the +# client's ring/window family. Strategy: +# 1. ring-page candidates: pages whose 8B-aligned u64 pairs look like +# GPFIFO entries (e0 = push VA & ~3 in the 0x44x-0x4ex families with +# va<4G; e1 top byte 0; length bits sane). +# 2. anchor: for each candidate PA and each family VA, check every 4KB +# page for entry idx=(VA>>12)&0x1ff whose PA part == candidate PA. +# A hit binds (PT page, VA, PA). +# 3. read out: from bound PT pages, decode entries for the whole VA list. +import mmap, struct, sys + +F = "/dev/shm/vgpu-guest-ram" +FAMILY = [ + 0x4020000, 0x4100000, 0x4160000, # r689 eng rings + 0x44a0000, 0x4500000, 0x4560000, # r690 ring VAs + 0x4480000, 0x4485000, 0x4480980, 0x4480f1c, + 0x4ca0000, 0x4ca5000, 0x4d00000, 0x4b17000, +] + +def gpa_to_off(pa): + return pa if pa < 0x80000000 else pa - 0x80000000 + +def off_to_gpa(off): + return off if off < 0x80000000 else off + 0x80000000 + +def pte_pa(v): + return (v >> 8) << 12 + +def main(): + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + size = len(mm) + print("# file size 0x%x" % size) + + # ---- pass 1: ring-page candidates by content ---- + cands = [] + pos = 0 + step = 4096 + while pos + step <= size: + d = mm[pos:pos+step] + hits = [] + for i in range(0, step, 8): + e0 = int.from_bytes(d[i:i+4], "little") + e1 = int.from_bytes(d[i+4:i+8], "little") + if 0x4400000 <= e0 <= 0x4e00000 and (e0 & 3) == 0 and (e1 & 0xff) == 0: + lw = (e1 >> 10) & 0x1fffff + if 0x40 <= lw <= 0x100000: + hits.append((i, e0, lw << 2)) + if len(hits) >= 3: + cands.append((off_to_gpa(pos), hits)) + if len(cands) >= 12: + break + pos += step + print("# ring-page candidates: %d" % len(cands)) + for pa, hits in cands: + print("cand pa=0x%x hits=%d first=%s" % + (pa, len(hits), ["0x%x/0x%x" % (h[1], h[2]) for h in hits[:4]])) + if not cands: + return 1 + + # ---- pass 2: anchor PT pages for candidate PAs x family VAs ---- + idxs = {} + for va in FAMILY: + idxs.setdefault((va >> 12) & 0x1ff, []).append(va) + cand_pas = set(pa for pa, _ in cands) + bound = [] + pos = 0 + while pos + step <= size: + pg = off_to_gpa(pos) + # skip scanning the candidate pages themselves + if pg not in cand_pas: + d = mm[pos:pos+step] + for idx, vas in idxs.items(): + o = idx * 8 + v = int.from_bytes(d[o:o+8], "little") + if v & 1: + pa = pte_pa(v) + if pa in cand_pas: + bound.append((pg, idx, vas, pa, v & 0xff)) + pos += step + print("# PT anchors found: %d" % len(bound)) + for pg, idx, vas, pa, fl in bound[:20]: + print("PT pa=0x%x entry[0x%x] flags=0x%x -> PA=0x%x (VA cands %s)" % + (pg, idx, fl, pa, ["0x%x" % x for x in vas])) + + # ---- pass 3: read out PT entries for the whole family from each bound PT page ---- + seen_pt = set() + for pg, idx, vas, pa, fl in bound: + if pg in seen_pt: + continue + seen_pt.add(pg) + off = gpa_to_off(pg) + d = mm[off:off+4096] + print("== PT page pa=0x%x family readout:" % pg) + for va in FAMILY: + i = (va >> 12) & 0x1ff + v = int.from_bytes(d[i*8:i*8+8], "little") + if v & 1: + print(" va=0x%x idx=0x%x -> pa=0x%x (raw=0x%x fl=0x%x)" % + (va, i, pte_pa(v), v, v & 0xff)) + else: + print(" va=0x%x idx=0x%x -> absent (raw=0x%x)" % (va, i, v)) + mm.close(); f.close() + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/run75g.sh b/tools/guest-diag/wall2/run75g.sh new file mode 100755 index 0000000..001a80f --- /dev/null +++ b/tools/guest-diag/wall2/run75g.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# run75g.sh - trace + record-word sweep on r680. +cd /tmp/opencode +rm -f /tmp/opencode/t75g-sweep.log +sshpass -p ubuntu scp -o StrictHostKeyChecking=no -P 2222 /tmp/opencode/t75g.py ubuntu@127.0.0.1:/tmp/t75g.py >/dev/null 2>&1 +( + sshpass -p ubuntu ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@127.0.0.1 \ + 'sudo bash /tmp/slotfill2.sh > /tmp/slotfill75g.log 2>&1 & sleep 0.5; timeout 400 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk_t75g.out 2>&1 & sleep 8; PID=$(pgrep -x vk_frame | head -1); echo "PID=$PID"; sudo timeout 380 gdb -q -batch -p $PID -x /tmp/t75g.py > /tmp/t75g.out 2>&1; echo GDB_DONE rc=$?' +) > /tmp/opencode/t75g-ssh.log 2>&1 & +GPID=$! +sleep 35 +setvar() { echo "$2" | sudo tee /sys/module/vgsp/parameters/$1 >/dev/null; } +mark() { echo "[$(date +%s.%N)] $*" >> /tmp/opencode/t75g-sweep.log; } +# phase 1: baseline echo (client already polling) +mark "baseline echo" +sleep 20 +# phase 2: all-zero records +setvar wall2_a097_rec0 0; setvar wall2_a097_rec1 0; setvar wall2_a097_rec2 0; setvar wall2_a097_rec3 0 +setvar wall2_a097_rec4 0; setvar wall2_a097_rec5 0; setvar wall2_a097_rec6 0; setvar wall2_a097_rec7 0 +mark "records all zero"; sleep 20 +# phase 3: progress=100 +setvar wall2_a097_rec1 100 +mark "rec1=100"; sleep 20 +# phase 4: progress=1000 +setvar wall2_a097_rec1 1000 +mark "rec1=1000"; sleep 20 +# phase 5: progress=100000 +setvar wall2_a097_rec1 100000 +mark "rec1=100000"; sleep 20 +# phase 6: progress=1000000 + rec0=0 +setvar wall2_a097_rec1 1000000 +setvar wall2_a097_rec0 0 +mark "rec1=1000000 rec0=0"; sleep 20 +# phase 7: all ones +setvar wall2_a097_rec0 3735928559; setvar wall2_a097_rec1 3735928559; setvar wall2_a097_rec2 3735928559; setvar wall2_a097_rec3 3735928559 +mark "rec0-3=0xdeadbeef"; sleep 20 +# phase 8: back to echo +setvar wall2_a097_rec0 4294967295; setvar wall2_a097_rec1 4294967295; setvar wall2_a097_rec2 4294967295; setvar wall2_a097_rec3 4294967295 +mark "baseline echo again" +wait $GPID +echo SWEEP-DONE diff --git a/tools/guest-diag/wall2/scan_r673.py b/tools/guest-diag/wall2/scan_r673.py new file mode 100644 index 0000000..71a0128 --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# Analyze the live r673 guest RAM: locate every reference to the validated ring +# PA (0x12a783000) including flag-bit variants, and characterize the record +# arrays around each hit. Also dump the two pages named in the handoff. +import numpy as np, struct, sys, time + +RAM="/dev/shm/vgpu-guest-ram" +RING=0x12a783000 +MASK=~0xfff & ((1<<64)-1) + +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) + +img=open(RAM,"rb") +import os +fsz=os.path.getsize(RAM) +print("ram size 0x%x" % fsz) + +t0=time.time() +hits=[] +CH=64<<20 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero((a & np.uint64(MASK))==np.uint64(RING))[0] + for i in idx: + off=pos+int(i)*8 + val=int(a[i]) + hits.append((off,val)) +print("hits=%d scan=%.1fs" % (len(hits), time.time()-t0)) +for off,val in hits: + print(" off=0x%x gpa=0x%x val=0x%x (low=0x%x)" % (off,off2gpa(off),val,val&0xfff)) + +# characterize each hit page +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +def plausible_pa(v): return 0x100000000 <= (v & MASK) < 0x180000000 +def ptrish(v): + return (0xffff800000000000 <= v < 0xffffffffffffffff) + +for off,val in hits: + gpa=off2gpa(off) + page=gpa & MASK + rec_off=gpa-page + d=rd(page,0x1000) + print("=== hit gpa=0x%x page=0x%x rec_off=0x%x ===" % (gpa,page,rec_off)) + for stride in (0x20,0x40,0x60,0x30,0x50,0x28,0x48): + # record start such that the hit qword falls inside the record + for rel in range(0,stride,8): + start=rec_off-rel + if start<0: continue + # walk records; count plausible PA fields anywhere in each record + n_pa=0; n_ptr=0; n_1000=0 + for o in range(start,0x1000-stride,stride): + rec=d[o:o+stride] + qs=struct.unpack_from("<%dQ"%(stride//8),rec,0) + if any(plausible_pa(q) for q in qs): n_pa+=1 + if any(ptrish(q) for q in qs): n_ptr+=1 + if 0x1000 in qs: n_1000+=1 + if n_pa>=8: + print(" stride=0x%x start=0x%x: PA-records=%d ptr-records=%d 0x1000=%d" % (stride,start,n_pa,n_ptr,n_1000)) + # dump the first 0x100 bytes of the page and around the hit + print(" page head: %s" % d[:0x60].hex()) + s=max(0,rec_off-0x40); e=min(0x1000,rec_off+0x80) + print(" around hit [0x%x..0x%x]: %s" % (s,e,d[s:e].hex())) + +# dump the two named pages +for pa in (0x101499000,0x127cf2000): + d=rd(pa,0x1000) + print("=== page 0x%x ===" % pa) + if d is None: print(" unreadable"); continue + for o in range(0,0x120,0x20): + qs=struct.unpack_from("<4Q",d,o) + print(" +0x%03x: %016x %016x %016x %016x" % ((o,)+qs)) diff --git a/tools/guest-diag/wall2/scan_r673b.py b/tools/guest-diag/wall2/scan_r673b.py new file mode 100644 index 0000000..11613b1 --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673b.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# r673: dump the full record page 0x101499000 and search RAM for the ring/push +# VA structures to identify which record is the 16-page push (VA 0x4500000). +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +MASK=~0xfff & ((1<<64)-1) +img=open(RAM,"rb") +fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +print("== full record page 0x101499000 ==") +d=rd(0x101499000,0x1000) +nrec=0 +for o in range(0,0x1000,32): + p,f,pa,sz=struct.unpack_from("<QQQQ",d,o) + if p==0 and f==0 and pa==0 and sz==0: continue + nrec+=1 + print(" rec[%3d] vmap=%016x flags=%016x pa=%012x size=0x%x" % (o//32,p,f,pa,sz)) +print("nonempty records:",nrec) + +print() +print("== search RAM for qword 0x4520000 (ring VA) ==") +t0=time.time(); CH=64<<20 +for pat_name,pat in (("0x4520000",0x4520000),("0x4500000",0x4500000)): + print("-- pattern %s --" % pat_name) + hits=[] + for pos in range(0,fsz,CH): + img.seek(pos); dd=img.read(min(CH,fsz-pos)) + n=len(dd)//8 + a=np.frombuffer(dd[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(pat))[0] + for i in idx: + hits.append(pos+int(i)*8) + print(" hits=%d (%.1fs)" % (len(hits),time.time()-t0)) + for off in hits[:40]: + gpa=off2gpa(off); page=gpa&MASK + # show 32 bytes before/after + dd=rd(page,0x1000) + if not dd: continue + ro=gpa-page + s=max(0,ro-32); e=min(0x1000,ro+64) + print(" gpa=0x%x page=0x%x off=0x%x ctx=%s" % (gpa,page,ro,dd[s:e].hex())) + if len(hits)>40: print(" ... %d more" % (len(hits)-40)) diff --git a/tools/guest-diag/wall2/scan_r673c.py b/tools/guest-diag/wall2/scan_r673c.py new file mode 100644 index 0000000..db69b5d --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673c.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# r673: find the push page. Patterns: ring entry raw qwords, the sem VA, and +# method-stream signatures. Dump contexts. +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +pats=[ + ("entry0raw", 0x0005980004500000), + ("entry1raw", 0x0000040004500598), + ("entry2raw", 0x0003d0000450059c), + ("sem_va", 0x4500598), + ("sem_va2", 0x450059c), + ("m20038090", 0x20038090), + ("push_va", 0x4500000), +] +CH=64<<20 +for name,pat in pats: + hits=[] + t0=time.time() + for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(pat))[0] + for i in idx: hits.append(pos+int(i)*8) + print("== %s pat=0x%x hits=%d (%.1fs)" % (name,pat,len(hits),time.time()-t0)) + for off in hits[:12]: + gpa=off2gpa(off); page=gpa&~0xfff + d=rd(page,0x1000) + ro=gpa-page + s=max(0,ro-24); e=min(0x1000,ro+40) + print(" gpa=0x%x page=0x%x off=0x%x ctx=%s" % (gpa,page,ro,d[s:e].hex() if d else "?")) + if len(hits)>12: print(" ... %d more" % (len(hits)-12)) diff --git a/tools/guest-diag/wall2/scan_r673d.py b/tools/guest-diag/wall2/scan_r673d.py new file mode 100644 index 0000000..1fa89ca --- /dev/null +++ b/tools/guest-diag/wall2/scan_r673d.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# r673 deep-dive: all ring-PA refs with low bits, ring page dump, RM record +# pages, and a RAM-wide scan for 0xa3bf20 VA-table records mentioning 0x4500000. +import numpy as np, struct, os, time + +RAM="/dev/shm/vgpu-guest-ram" +RING=0x12a783000; TAG=0xa3bf20 +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) + +MASK=~0xfff & ((1<<64)-1) +print("== all qwords (v & ~0xfff)==ring_pa ==") +CH=64<<20 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero((a & np.uint64(MASK))==np.uint64(RING))[0] + for i in idx: + off=pos+int(i)*8 + gpa=off2gpa(off); page=gpa&~0xfff + dd=rd(page,0x1000); ro=gpa-page + s=max(0,ro-32); e=min(0x1000,ro+48) + print(" gpa=0x%x low=0x%x ctx=%s" % (gpa,int(a[i])&0xfff,dd[s:e].hex() if dd else "?")) + +print() +print("== ring page 0x%x first 0x120 ==" % RING) +d=rd(RING,0x120) +for o in range(0,0x120,0x20): + print(" +0x%03x: %s" % (o,d[o:o+0x20].hex())) + +for pa in (0x1013a6000,0x104acb000,0x114760000,0x1202cd000,0x13e2c1000,0x13e62e000): + print() + print("== page 0x%x first 0x140 ==" % pa) + d=rd(pa,0x140) + for o in range(0,0x140,0x20): + q=struct.unpack_from("<4Q",d,o) + print(" +0x%03x: %016x %016x %016x %016x" % ((o,)+q)) + +print() +print("== RAM-wide: 32-byte records {va, cpu, 0xa3bf20, pages} with va==0x4500000 ==") +t0=time.time() +found=0 +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(TAG))[0] + for i in idx: + off=pos+int(i)*8 + gpa=off2gpa(off); page=gpa&~0xfff + dd=rd(page,0x1000) + ro=gpa-page + if ro>=16 and ro+8<=0x1000: + va,cpu=struct.unpack_from("<QQ",dd,ro-16) + pgs=struct.unpack_from("<Q",dd,ro+8)[0] + if va==0x4500000 or (0x4400000<=va<=0x4600000): + print(" rec gpa=0x%x va=0x%x cpu=0x%x pages=0x%x" % (gpa,va,cpu,pgs)) + found+=1 +print("total", found, "(%.1fs)" % (time.time()-t0)) diff --git a/tools/guest-diag/wall2/session73-notes.md b/tools/guest-diag/wall2/session73-notes.md new file mode 100644 index 0000000..d2079d3 --- /dev/null +++ b/tools/guest-diag/wall2/session73-notes.md @@ -0,0 +1,51 @@ +# SESSION 73 notes (wall #2 live validation, r674 + r675) + +## r674 cycle results (8 client runs, all VKFAIL -4) +- Client vk_frame reproducibly fails at vkCreateDevice (-4) as before. +- wall2_resolver2 (content-first CE group 0x20038090 + tuple arrays): no candidates found + during the stall; tuple arrays contain the ring PA but not the push. +- livecap/livecap2/livecap3 (0x20038090 shape / 0x20050017 host-release shape): NO data + hits in guest RAM during the stall; only driver CODE pages match. +- Conclusion: the GR channel-init push content is NOT findable in /dev/shm/vgpu-guest-ram + by these patterns, even live. (The ring page content IS there and matches the service's + decode.) + +## Structural findings +- The push is RM-built (nvidia.ko), NOT UVM: kprobe on + uvm_hal_turing_host_semaphore_release never fired during the client run. +- nvidia.ko functions are `notrace` -> kprobe_events refuses them + ("Could not probe notrace function ... [nvidia]"). UVM module probes DO work. + QEMU gdbstub hardware breakpoints work fine (hbreak *addr). +- Service rescue matcher is CE-shaped only (hdr 0x20038090, hi==1, lo&0xfff==slot, + w5/v5 shape) -> can never match a GR push. Entries 0/1/2 are consumed as "stale" + after ~150 failed rescues each; gr_wedge=1. +- channelFillGpFifo(channel, idx, len) disasm: rdi=channel; +0x178=gpu; + +0xe8=cached ring CPU VA; +0xb8=push GPU VA base; +0x48=stride; + +0x110=class (0xc5b5 seen for a CE channel; GR = 0xc46f). + Entry encoding: [0]=va&~3 (u32), [1]=(va>>32 &0xff)|((len<<8)&0x7ffffc00). +- r674 ring PAs per run: 0x12a783000(?) no: run1 0x1050b6000, run2 0x10297d000, + later 0x102317000 (ring line format: "h=0x... gpfifo=0x44a0000 ent=32768 userd=0xffe00 as=2 put=3 get=0 ring rc=0 pa=0x... ap=1"). +- Push VA = 0x4480000, ring VA = 0x44a0000 (delta 0x20000); entry0 len=1432, + entry1 {va=0x4480598,len=4}, entry2 {va=0x448059c,len=976}. +- The pushbuffer (UVM) = 0x20000 (32 pages) aligned; UVM_PUSHBUFFER_SIZE=0x20000 + (UVM_MAX_PUSH_SIZE=0x400). UVM_RM_MEM_TYPE_SYS with CPU map. +- RM memdesc records {size=0x20000, flags=0x03008000, first_PA, ptr} exist in RAM + (11 found post-run) but their PAs did not contain the release group. +- RM window record {0x4480000, 0x20000} + head/tail list nodes exist in RAM. +- The r673 VA-table record {0x10, 0x4500000, 0x786fd9598ab0, 0xa3bf20} has a + "cpu" field that may be a user VA of the client (0x64..-0x78.. range). + +## Ops +- r674 wedged after the gdb-interrupted vk31 run (SSH + guest agent dead, vCPUs idle). + Fresh cycle r675 started. gdb674d still attached to the dead QEMU (never kill gdbs). +- /dev/shm 53%. + +## r675 plan +1. gdb675.gdb: hbreak channelFillGpFifo if class(+0x110)==0xc46f; on hit dump the + channel struct pointer fields (see gdb675.gdb) -> identify the push's CPU mapping + (direct-map or vmalloc) by content. +2. Compute PA (direct-map: VA - 0xffff8916c0000000 style base) and read the push + content from the RAM file; find the release group. +3. Inject 0x4480000 -> PA via sema_map_add BEFORE/DURING the stall; watch the service + apply the release and vkCreateDevice pass. +4. Durability: integrate into pin_gr_stream.py demand path or a service hook. diff --git a/tools/guest-diag/wall2/slotfill.sh b/tools/guest-diag/wall2/slotfill.sh new file mode 100644 index 0000000..a3416a9 --- /dev/null +++ b/tools/guest-diag/wall2/slotfill.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Watch the vk_frame client and fill the completion-slot u16 (base+0x2e) in +# every small /dev/nvidiactl rw-s mapping while it waits. +cd /tmp +rm -f /tmp/slotfill.log +(timeout 120 /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk76.out 2>&1 &) +PID="" +for i in $(seq 1 100); do PID=$(pgrep -x vk_frame | head -1); [ -n "$PID" ] && break; sleep 0.1; done +echo "pid=$PID" >> /tmp/slotfill.log +END=$((SECONDS+60)) +while [ $SECONDS -lt $END ]; do + [ -d /proc/$PID ] || break + for m in $(grep 'nvidiactl' /proc/$PID/maps | awk '$2 ~ /rw-s/ {print $1}'); do + lo=${m%-*}; hi=${m#*-} + size=$((0x$hi - 0x$lo)) + [ $size -gt 65536 ] && continue + off=$((0x$lo + 0x2e)) + v=$(dd if=/proc/$PID/mem bs=1 skip=$off count=2 2>/dev/null | xxd -p) + if [ "$v" = "ffff" ]; then + printf '\x00\x00' | dd of=/proc/$PID/mem bs=1 seek=$off count=2 conv=notrunc 2>/dev/null + echo "$(date +%s.%N) FILL $m+0x2e was ffff" >> /tmp/slotfill.log + fi + done + sleep 0.2 +done +echo "done" >> /tmp/slotfill.log +echo "=== client ===" +cat /tmp/vk76.out +echo "=== fills ===" +grep -c FILL /tmp/slotfill.log diff --git a/tools/guest-diag/wall2/slotfill2.sh b/tools/guest-diag/wall2/slotfill2.sh new file mode 100644 index 0000000..33ad702 --- /dev/null +++ b/tools/guest-diag/wall2/slotfill2.sh @@ -0,0 +1,19 @@ +#!/bin/bash +cd /tmp +PID="" +for i in $(seq 1 200); do PID=$(pgrep -x vk_frame | head -1); [ -n "$PID" ] && break; sleep 0.05; done +END=$((SECONDS+45)) +while [ $SECONDS -lt $END ]; do + [ -d /proc/$PID ] || break + for m in $(grep 'nvidiactl' /proc/$PID/maps 2>/dev/null | awk '$2 ~ /rw-s/ {print $1}'); do + lo=${m%-*}; hi=${m#*-} + size=$((0x$hi - 0x$lo)) + [ $size -gt 65536 ] && continue + off=$((0x$lo + 0x2e)) + v=$(dd if=/proc/$PID/mem bs=1 skip=$off count=2 2>/dev/null | xxd -p) + if [ "$v" = "ffff" ]; then + printf '\x00\x00' | dd of=/proc/$PID/mem bs=1 seek=$off count=2 conv=notrunc 2>/dev/null + fi + done + sleep 0.1 +done diff --git a/tools/guest-diag/wall2/slots_probe.sh b/tools/guest-diag/wall2/slots_probe.sh new file mode 100644 index 0000000..71e6fb7 --- /dev/null +++ b/tools/guest-diag/wall2/slots_probe.sh @@ -0,0 +1,11 @@ +#!/bin/bash +PID=$1 +sudo grep -E 'rw-s' /proc/$PID/maps | grep nvidiactl | while read -r line; do + RNG=$(echo "$line" | awk '{print $1}') + LO=${RNG%-*}; HI=${RNG#*-} + SZ=$((0x$HI-0x$LO)) + [ $SZ -gt 262144 ] && continue + V2E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0x2e)) count=2 2>/dev/null | xxd -p) + V0E=$(sudo dd if=/proc/$PID/mem bs=1 skip=$((0x$LO+0xe)) count=2 2>/dev/null | xxd -p) + echo "$RNG sz=$SZ w2e=$V2E w0e=$V0E" +done diff --git a/tools/guest-diag/wall2/stracevk.sh b/tools/guest-diag/wall2/stracevk.sh new file mode 100644 index 0000000..b2752c4 --- /dev/null +++ b/tools/guest-diag/wall2/stracevk.sh @@ -0,0 +1,7 @@ +#!/bin/sh +cd /tmp +sudo timeout 90 strace -f -T -tt -e trace=ioctl -o /tmp/vk_strace.out /tmp/vk_frame 1 64 64 5 /tmp 0 > /tmp/vk58.out 2>&1 +echo "=== client out ===" +cat /tmp/vk58.out +echo "=== ioctl tail ===" +tail -30 /tmp/vk_strace.out diff --git a/tools/guest-diag/wall2/t101.py b/tools/guest-diag/wall2/t101.py new file mode 100644 index 0000000..436c609 --- /dev/null +++ b/tools/guest-diag/wall2/t101.py @@ -0,0 +1,55 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid)) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception as e: + if state["chits"] <= 4: + print("chitfail %s" % e, flush=True) + if state["cleared"] >= 24: + self.enabled = False + print("BC-DISABLED", flush=True) + return False +try: + BC(); print("BPS-SET", flush=True) + gr("continue") +except Exception as e: + print("bpfail %s" % e, flush=True) +print("T101-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True) diff --git a/tools/guest-diag/wall2/t102.py b/tools/guest-diag/wall2/t102.py new file mode 100644 index 0000000..759d974 --- /dev/null +++ b/tools/guest-diag/wall2/t102.py @@ -0,0 +1,31 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid)) +try: + pc = int(gdb.parse_and_eval("$pc")) + print("PC 0x%x off=0x%x" % (pc, pc - base if base else 0)) +except Exception as e: + print("pcfail %s" % e) +try: + C = base + 0x2070B7C + print("A8COUNT %u" % int(gdb.parse_and_eval("*(unsigned int*)0x%x" % C))) +except Exception as e: + print("countfail %s" % e) +try: + print(gr("bt 6", to_string=True)) +except Exception as e: + print("btfail %s" % e) +gr("detach") +gr("quit") +print("T102-DONE") diff --git a/tools/guest-diag/wall2/t104.py b/tools/guest-diag/wall2/t104.py new file mode 100644 index 0000000..5a4c059 --- /dev/null +++ b/tools/guest-diag/wall2/t104.py @@ -0,0 +1,56 @@ +import gdb, time +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid), flush=True) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0, "t0":time.time(), "last":time.time()} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + state["last"] = time.time() + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + if state["cleared"] >= 8 or state["chits"] >= 40 or (time.time() - state["last"]) > 25.0: + self.enabled = False + print("BC-EXIT cleaned=%d chits=%d" % (state["cleared"], state["chits"]), flush=True) + return True + return False +try: + BC(); print("BPS-SET", flush=True) + gr("continue") +except Exception as e: + print("bpfail %s" % e, flush=True) +print("T104-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True) diff --git a/tools/guest-diag/wall2/t106.py b/tools/guest-diag/wall2/t106.py new file mode 100644 index 0000000..ceaac44 --- /dev/null +++ b/tools/guest-diag/wall2/t106.py @@ -0,0 +1,61 @@ +import gdb, time +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid), flush=True) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0, "done":False} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + if state["done"]: + return False + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + if state["cleared"] >= 5 or state["chits"] >= 14: + state["done"] = True + try: + gr("detach") + print("BC-DETACHED cleaned=%d chits=%d" % (state["cleared"], state["chits"]), flush=True) + except Exception as e: + print("detachfail %s" % e, flush=True) + return True + return False +try: + BC(); print("BPS-SET", flush=True) + gr("continue") +except Exception as e: + print("contend %s" % e, flush=True) +print("T106-DONE chits=%d cleared=%d" % (state["chits"], state["cleared"]), flush=True) diff --git a/tools/guest-diag/wall2/t75g.py b/tools/guest-diag/wall2/t75g.py new file mode 100644 index 0000000..2a0881a --- /dev/null +++ b/tools/guest-diag/wall2/t75g.py @@ -0,0 +1,170 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid + +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found") + gdb.execute("detach") + sys.exit(1) +print("EGLBASE 0x%x PID %d" % (base, pid)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"out": 0, "E": 0, "S": 0, "armed150": False, "armedS": False, + "bt_done": False, "post_done": False} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def a97(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + r14 = int(gdb.parse_and_eval("$r14")) + out = u64(rbp+0x10) + state["out"] = out + q = rd(out, 0x28) + rec_ptr = int.from_bytes(q[8:16], 'little') + print("A97 out=0x%x count=%d f10=%08x f20=%08x f24=%08x recp=0x%x" % + (out, int.from_bytes(q[0:4], 'little'), int.from_bytes(q[0x10:0x14], 'little'), + int.from_bytes(q[0x20:0x24], 'little'), int.from_bytes(q[0x24:0x28], 'little'), rec_ptr)) + if rec_ptr and rec_ptr > 0x1000: + try: + rec = rd(rec_ptr, 16) + print("A97 rec=%08x %08x %08x %08x" % + tuple(int.from_bytes(rec[i*4:i*4+4], 'little') for i in range(4))) + except Exception as e: + print("A97 rec-unreadable", e) + return False + +def post(bp): + if state["post_done"]: + return False + state["post_done"] = True + rsp = int(gdb.parse_and_eval("$rsp")) + q = rd(rsp, 0x40) + print("POST rsp=0x%x out=0x%x" % (rsp, state["out"])) + for i in range(0, 0x40, 8): + v = int.from_bytes(q[i:i+8], 'little') + print("POST rsp+%02x = 0x%016x" % (i, v)) + if state["out"]: + oq = rd(state["out"], 0x40) + for i in range(0, 0x40, 8): + v = int.from_bytes(oq[i:i+8], 'little') + print("POST out+%02x = 0x%016x" % (i, v)) + return False + +def t150(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + if not state.get("vt_done"): + state["vt_done"] = True + rdi = u64(rbp+0x148) + vt = u64(rdi) if rdi else 0 + m118 = u64(vt+0x118) if vt else 0 + m170 = u64(vt+0x170) if vt else 0 + print("VT E148=0x%x vt=0x%x m118=0x%x m170=0x%x" % (rdi, vt, m118, m170)) + if not state.get("eng_done"): + state["eng_done"] = True + n = u32(rbp+8) + ids = [u32(rbp+0x54+i*4) for i in range(min(n,8))] + print("ENG n=%d ids=%s" % (n, ",".join("0x%x" % x for x in ids))) + + # cfg area pointer is rip-relative 0x2019320 at runtime = base + (0x2019320 - 0) + # eglcore base == min load addr; file offset == vaddr for the .text; data seg present? + try: + vals = rd(base + 0x2019320 + 0xf80, 0xe0) + for i in range(0, 0xe0, 0x10): + v4 = [int.from_bytes(vals[i+j:i+j+4],'little') for j in (0,4,8,12)] + print("CFG +0x%03x = %s" % (0xf80+i, ["0x%x"%x for x in v4])) + except Exception as e: + print("CFG dump err", e) + v = u32(rbp+0x150) + b38 = rd(rbp+0x1b038, 1)[0]; b39 = rd(rbp+0x1b039, 1)[0] + p = u64(rbp+0x1b030); s8 = u64(p+8) if p else 0 + print("T150 E=0x%x v=0x%08x b038=%d b039=%d p=0x%x p8=0x%x" % + (rbp, v, b38, b39, p, s8)) + if not state["bt_done"]: + state["bt_done"] = True + print(gdb.execute("bt 8", to_string=True)) + if not state["armed150"]: + state["armed150"] = True + state["E"] = rbp + try: + gdb.execute("frame 1") + S = int(gdb.parse_and_eval("$rbp")) + gdb.execute("frame 0") + print("S=0x%x S120=%08x S128=%08x" % (S, u32(S+0x120), u32(S+0x128))) + state["S"] = S + except Exception as e: + print("S capture err", e) + S = 0 + W("*(unsigned int *)0x%x" % (rbp+0x150), rbp, 0x150, 4, "W150") + W("*(unsigned char *)0x%x" % (rbp+0x1b038), rbp, 0x1b038, 1, "W038") + W("*(unsigned char *)0x%x" % (rbp+0x1b039), rbp, 0x1b039, 1, "W039") + if S: + W("*(unsigned int *)0x%x" % (S+0x120), S, 0x120, 4, "W120") + print("WATCHES armed E=0x%x S=0x%x" % (rbp, S)) + return False + +def sl(bp): + rbp = int(gdb.parse_and_eval("$rbp")) + print("SLOOP S=0x%x S120=%08x" % (rbp, u32(rbp+0x120))) + if not state["armedS"]: + state["armedS"] = True + W("*(unsigned int *)0x%x" % (rbp+0x120), rbp, 0x120, 4, "W120") + print("W120 armed on S+0x120") + return False + +class W(gdb.Breakpoint): + def __init__(self, spec, E, off, size, tag): + super().__init__(spec, type=gdb.BP_WATCHPOINT) + self.silent = True; self.E = E; self.off = off; self.size = size + self.tag = tag; self.hits = 0 + def stop(self): + self.hits += 1 + cur = int.from_bytes(rd(self.E+self.off, self.size), 'little') + pc = int(gdb.parse_and_eval("$pc")) + print("%s v=0x%x pc=0x%x (rel 0x%x)" % (self.tag, cur, pc, pc-base)) + if self.hits <= 8: + print(gdb.execute("bt 10", to_string=True)) + return self.hits >= 400 + +B("*0x%x" % (base+0xecd43d), a97, 220) +B("*0x%x" % (base+0xd5e4d3), post, 100000) +B("*0x%x" % (base+0xd5e449), t150, 220) +B("*0x%x" % (base+0xd6ef76), sl, 240) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +print("TRACE-END") +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t79.py b/tools/guest-diag/wall2/t79.py new file mode 100644 index 0000000..c2fe381 --- /dev/null +++ b/tools/guest-diag/wall2/t79.py @@ -0,0 +1,99 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found"); gdb.execute("detach"); sys.exit(1) +print("EGLBASE 0x%x PID %d" % (base, pid)) + +maps = [] +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x, 16) for x in parts[0].split('-')] + maps.append((lo, hi, parts[1], parts[-1] if len(parts) > 5 else "anon")) + except Exception: + pass + +def resolve(addr): + for lo, hi, perm, name in maps: + if lo <= addr < hi: + return "0x%x-0x%x %s %s" % (lo, hi, perm, name) + return "?" + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"done": False} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def loophead(bp): + if state["done"]: + return False + state["done"] = True + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx+0x70) + print("WAIT rbx=0x%x sub=0x%x" % (rbx, sub)) + try: + obj2 = u64(sub+0x58) + off = u32(obj2+0x60) + holder = u64(obj2+0x58) + base2 = u64(holder+0x58) + word_addr = base2 + off + val = u32(word_addr) + print("WAIT obj2=0x%x off=0x%x holder=0x%x base=0x%x word=0x%x val=0x%x" % + (obj2, off, holder, base2, word_addr, val)) + print("WAIT word map:", resolve(word_addr)) + print("WAIT holder obj2+0x50=0x%x obj2+0x64=0x%x" % (u64(obj2+0x50), u32(obj2+0x64))) + try: + hd = rd(holder, 0x30) + print("WAIT holder dump:", " ".join("%02x" % b for b in hd)) + except Exception as e: + print("WAIT holder dump err", e) + except Exception as e: + print("WAIT chain err", e) + # dump the client's nvidiactl rw-s mappings + print("WAIT nvidiactl maps:") + for lo, hi, perm, name in maps: + if "nvidiactl" in name and "rw-s" in perm: + print(" 0x%x-0x%x size=0x%x" % (lo, hi, hi-lo)) + return True + +B("*0x%x" % (base+0xdcebd9), loophead, 1) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +print("TRACE-END") +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t80r.py b/tools/guest-diag/wall2/t80r.py new file mode 100644 index 0000000..ac2263b --- /dev/null +++ b/tools/guest-diag/wall2/t80r.py @@ -0,0 +1,32 @@ +import gdb, sys +inf = gdb.selected_inferior() +pid = inf.pid +cands = [] +for line in open("/proc/%d/maps" % pid): + if "nvidiactl" in line and "rw-s" in line: + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if hi - lo == 0x4000: + cands.append(lo + 0x80) +print("CANDIDATES %s" % ["0x%x" % c for c in cands]) +class W(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*(unsigned int*)0x%x" % addr, gdb.BP_WATCHPOINT, gdb.WP_READ, internal=False) + self.addr = addr + self.silent = True + def stop(self): + print("READWORD 0x%x" % self.addr) + return True +armed = 0 +for c in cands: + try: + W(c); armed += 1 + except Exception as e: + print("watchfail 0x%x: %s" % (c, e)) +print("ARMED %d" % armed) +if armed: + gdb.execute("continue") +print("T80R-DONE") diff --git a/tools/guest-diag/wall2/t83.py b/tools/guest-diag/wall2/t83.py new file mode 100644 index 0000000..5e3cce2 --- /dev/null +++ b/tools/guest-diag/wall2/t83.py @@ -0,0 +1,25 @@ +import gdb +from collections import Counter +gr = gdb.execute +pcs = Counter() +inf = gdb.selected_inferior() +for i in range(300): + try: + gr("stepi", to_string=True) + except Exception as e: + print("stepfail %s" % e); break + try: + pcs[int(gdb.parse_and_eval("$pc"))] += 1 + except Exception: + break +print("HOTPCS") +top = pcs.most_common(4) +for pc, n in top: + print("PC 0x%x hits=%d" % (pc, n)) +for pc, n in top[:1]: + try: + print(gr("x/10i 0x%x" % pc, to_string=True)) + print(gr("info registers rax rbx rcx rdx rsi rdi rbp r8 r9 r10 r11 r12 r13 r14 r15", to_string=True)) + except Exception as e: + print("fail %s" % e) +print("T83-DONE") diff --git a/tools/guest-diag/wall2/t89.py b/tools/guest-diag/wall2/t89.py new file mode 100644 index 0000000..92c1de3 --- /dev/null +++ b/tools/guest-diag/wall2/t89.py @@ -0,0 +1,17 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +print(gr("x/70i 0x%x" % (base + 0xB28519), to_string=True)) +print(gr("x/40i 0x%x" % (base + 0xB28560), to_string=True)) +print("T89-DONE") diff --git a/tools/guest-diag/wall2/t90.py b/tools/guest-diag/wall2/t90.py new file mode 100644 index 0000000..8518235 --- /dev/null +++ b/tools/guest-diag/wall2/t90.py @@ -0,0 +1,57 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found"); gdb.execute("detach"); sys.exit(1) +COUNT = base + 0x2070b7c +print("EGLBASE 0x%x PID %d COUNT_ADDR 0x%x" % (base, pid, COUNT)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') + +state = {"n": 0} + +class B(gdb.Breakpoint): + def __init__(self, spec, limit=100000): + super().__init__(spec, internal=False) + self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + v = u32(COUNT) + if v: + state["n"] += 1 + if state["n"] <= 8: + print("CLEAR count %u -> 0 (hit %d)" % (v, self.hits)) + gdb.execute("set {unsigned int}0x%x = 0" % COUNT) + except Exception as e: + print("ERR", e) + return self.hits >= 400 + +B("*0x%x" % (base+0xa85d24), 400) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +except KeyboardInterrupt: + pass +print("TRACE-END clears=%d" % state["n"]) +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t91.py b/tools/guest-diag/wall2/t91.py new file mode 100644 index 0000000..5c84ba9 --- /dev/null +++ b/tools/guest-diag/wall2/t91.py @@ -0,0 +1,95 @@ +import gdb, sys + +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + if "libnvidia-eglcore.so.570.144" in line: + parts = line.split() + lo, hi = [int(x, 16) for x in parts[0].split('-')] + off = int(parts[2], 16) + if off == 0: + base = lo + break +if base is None: + print("EGLBASE not found"); gdb.execute("detach"); sys.exit(1) +COUNT = base + 0x2070b7c +print("EGLBASE 0x%x PID %d COUNT 0x%x" % (base, pid, COUNT)) + +def rd(a, n): + return inf.read_memory(a, n).tobytes() +def u32(a): + return int.from_bytes(rd(a, 4), 'little') +def u64(a): + return int.from_bytes(rd(a, 8), 'little') + +state = {"clears": 0, "mp": 0, "am": 0} + +class B(gdb.Breakpoint): + def __init__(self, spec, fn, limit=100000): + super().__init__(spec, internal=False) + self.fn = fn; self.limit = limit; self.hits = 0 + self.silent = True + def stop(self): + self.hits += 1 + try: + stop = bool(self.fn(self)) + except Exception as e: + print("ERR@0x%x %s" % (int(gdb.parse_and_eval("$pc")), e)) + stop = False + return stop or self.hits >= self.limit + +def clear(bp): + try: + v = u32(COUNT) + if v: + state["clears"] += 1 + if state["clears"] <= 5: + print("CLEAR count %u -> 0" % v) + gdb.execute("set {unsigned int}0x%x = 0" % COUNT) + except Exception as e: + print("CLEAR err", e) + return False + +def mp(bp): + rsi = int(gdb.parse_and_eval("$rsi")) + state["mp"] += 1 + if state["mp"] <= 3: + print("MEMPROPS ptr=0x%x" % rsi) + return False + +def am(bp): + rsi = int(gdb.parse_and_eval("$rsi")) + state["am"] += 1 + if state["am"] <= 10: + try: + st = u32(rsi) + size = u64(rsi+0x10) + tidx = u32(rsi+0x18) + print("ALLOC sType=%d size=0x%x typeIndex=%d" % (st, size, tidx)) + except Exception as e: + print("ALLOC err", e) + return False + +B("*0x%x" % (base+0xa85d24), clear, 100000) +try: + B("vkGetPhysicalDeviceMemoryProperties", mp, 5) +except Exception as e: + print("MP BP err", e) +try: + B("vkAllocateMemory", am, 20) +except Exception as e: + print("AM BP err", e) + +try: + gdb.execute("continue") +except gdb.error as e: + print("CONT-ERR %s" % e) +except KeyboardInterrupt: + pass +print("TRACE-END clears=%d" % state["clears"]) +try: + gdb.execute("detach") +except gdb.error as e: + print("DETACH-ERR %s" % e) +gdb.execute("quit") diff --git a/tools/guest-diag/wall2/t93.py b/tools/guest-diag/wall2/t93.py new file mode 100644 index 0000000..6a0c23d --- /dev/null +++ b/tools/guest-diag/wall2/t93.py @@ -0,0 +1,57 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +class B(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True; self.done = False + def stop(self): + if self.done: + return True + self.done = True + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + print("RETRYFN obj=0x%x n=0x%x" % (obj, n)) + cnt = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % (obj + 0x22efc))) + print("CNT %d" % cnt) + wrote = 0 + for i in range(0, min(cnt, 32)): + p = int(gdb.parse_and_eval("*(unsigned long*)0x%x" % (obj + 0x1428 + i*8))) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + v0 = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % tgt)) + print("E%d ptr=0x%x tgt=0x%x w0=0x%x w16e=0x%x" % (i, p, tgt, v0, v16)) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + wrote += 1 + except Exception as e: + print("E%d fail %s" % (i, e)) + print("CLEARED %d" % wrote) + gr("detach") + except Exception as e: + print("fail %s" % e) + return True +try: + B() + print("BP-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T92-DONE") diff --git a/tools/guest-diag/wall2/t94.py b/tools/guest-diag/wall2/t94.py new file mode 100644 index 0000000..0d2aa27 --- /dev/null +++ b/tools/guest-diag/wall2/t94.py @@ -0,0 +1,54 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +state = {"hits":0, "cleared":0} +class B(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["hits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % (obj + 0x22efc))) + if state["hits"] <= 6: + print("HIT %d obj=0x%x n=0x%x cnt=%d" % (state["hits"], obj, n, cnt)) + for i in range(0, min(cnt, 32)): + p = int(gdb.parse_and_eval("*(unsigned long*)0x%x" % (obj + 0x1428 + i*8))) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + if state["cleared"] <= 12: + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt)) + except Exception as e: + if state["hits"] <= 6: + print("E%d fail %s" % (i, e)) + except Exception as e: + if state["hits"] <= 6: + print("hitfail %s" % e) + return state["hits"] >= 4000 +try: + B(); print("BP-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T94-DONE hits=%d cleared=%d" % (state["hits"], state["cleared"])) diff --git a/tools/guest-diag/wall2/t95.py b/tools/guest-diag/wall2/t95.py new file mode 100644 index 0000000..8daf6ca --- /dev/null +++ b/tools/guest-diag/wall2/t95.py @@ -0,0 +1,19 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +print("--- wait site +0xA12D90..:") +print(gr("x/60i 0x%x" % (base + 0xA12D90), to_string=True)) +print("--- caller +0xD690D0..:") +print(gr("x/40i 0x%x" % (base + 0xD690D0), to_string=True)) +print("T95-DONE") diff --git a/tools/guest-diag/wall2/t96b.py b/tools/guest-diag/wall2/t96b.py new file mode 100644 index 0000000..cd3b7ab --- /dev/null +++ b/tools/guest-diag/wall2/t96b.py @@ -0,0 +1,40 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +class B(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xA12DDF), internal=False) + self.silent = True; self.done = False + def stop(self): + if self.done: + return True + self.done = True + try: + rdi = int(gdb.parse_and_eval("$rdi")) + rsi = int(gdb.parse_and_eval("$rsi")) + rdx = int(gdb.parse_and_eval("$rdx")) + print("POLL nfds=%d timeout=%d arr=0x%x" % (rsi, rdx, rdi)) + fd = int(gdb.parse_and_eval("*(int*)0x%x" % rdi)) + ev = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (rdi + 4))) + rv = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (rdi + 6))) + print("FDPOLL fd=%d events=0x%x revents=0x%x" % (fd, ev, rv)) + except Exception as e: + print("fail %s" % e) + return True +try: + B(); print("BP-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T96-DONE") diff --git a/tools/guest-diag/wall2/t97.py b/tools/guest-diag/wall2/t97.py new file mode 100644 index 0000000..5a12348 --- /dev/null +++ b/tools/guest-diag/wall2/t97.py @@ -0,0 +1,22 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x" % (base or 0)) +C = base + 0x2070b7c +v = int(gdb.parse_and_eval("*(unsigned int*)0x%x" % C)) +print("COUNT %u" % v) +print(gr("x/8wx 0x%x" % (C - 0x10), to_string=True)) +if v: + gr("set {unsigned int}0x%x = 0" % C) + print("WROTE 0") +print("T97-DONE") diff --git a/tools/guest-diag/wall2/t99.py b/tools/guest-diag/wall2/t99.py new file mode 100644 index 0000000..ed665fb --- /dev/null +++ b/tools/guest-diag/wall2/t99.py @@ -0,0 +1,79 @@ +import gdb +gr = gdb.execute +inf = gdb.selected_inferior() +pid = inf.pid +base = None +for line in open("/proc/%d/maps" % pid): + parts = line.split() + try: + lo, hi = [int(x,16) for x in parts[0].split('-')] + except Exception: + continue + if base is None and "libnvidia-eglcore" in line and int(parts[2],16) == 0: + base = lo +print("BASE 0x%x PID %d" % (base or 0, pid)) +def u64(a): return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) +def u32(a): return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +state = {"chits":0, "cleared":0, "whits":0, "wrote":0} +class BC(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xB33D30), internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i*8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + if state["cleared"] <= 10: + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt)) + except Exception: + pass + except Exception as e: + if state["chits"] <= 4: + print("chitfail %s" % e) + return state["chits"] >= 6000 +class BW(gdb.Breakpoint): + def __init__(self): + super().__init__("*0x%x" % (base + 0xDCEBD9), internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val)) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + if state["wrote"] <= 10: + print("WORDWROTE 0x%x" % word) + except Exception as e: + if state["whits"] <= 4: + print("whitfail %s" % e) + return False +try: + BC(); BW(); print("BPS-SET") + gr("continue") +except Exception as e: + print("bpfail %s" % e) +print("T99-DONE chits=%d cleared=%d whits=%d wrote=%d" % (state["chits"], state["cleared"], state["whits"], state["wrote"])) diff --git a/tools/guest-diag/wall2/trace_push674.py b/tools/guest-diag/wall2/trace_push674.py new file mode 100644 index 0000000..ced4168 --- /dev/null +++ b/tools/guest-diag/wall2/trace_push674.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# r674: find every reference to the push VA 0x4480000 and follow RM structures +# (direct-map pointers: PA = ptr - page_offset_base) toward a page list. +import numpy as np, struct, os, time, sys + +RAM="/dev/shm/vgpu-guest-ram" +VA=0x4480000 +BASE=0xffff8916c0000000 # r674 page_offset_base (from dmesg rebuild #15) +img=open(RAM,"rb"); fsz=os.path.getsize(RAM) +def gpa2off(pa): + if pa<0x80000000: return pa + if pa<0x180000000: return 0x80000000+(pa-0x100000000) + return None +def off2gpa(off): + return off if off<0x80000000 else 0x100000000+(off-0x80000000) +def rd(pa,n): + off=gpa2off(pa) + if off is None or off+n>fsz: return None + img.seek(off); return img.read(n) +def ptr2pa(v): + if BASE<=v<BASE+0x180000000: + return v-BASE + return None + +CH=64<<20 +hits=[] +t0=time.time() +for pos in range(0,fsz,CH): + img.seek(pos); d=img.read(min(CH,fsz-pos)) + n=len(d)//8 + a=np.frombuffer(d[:n*8],dtype=np.uint64) + idx=np.nonzero(a==np.uint64(VA))[0] + for i in idx: hits.append(off2gpa(pos+int(i)*8)) +print("VA hits:", len(hits), "(%.1fs)" % (time.time()-t0)) + +def tag(v): + bs=struct.pack("<Q",v) + return bs.decode() if all(32<=c<127 for c in bs) else "" + +for gpa in hits: + page=gpa&~0xfff + d=rd(page,0x1000) + if not d: continue + ro=gpa-page + print("== hit gpa=0x%x page=0x%x off=0x%x ==" % (gpa,page,ro)) + # print 0x60 before, 0x80 after + s=max(0,ro-0x60); e=min(0x1000,ro+0x80) + for o in range(s,e,0x10): + a,b=struct.unpack_from("<QQ",d,o) + print(" +0x%03x: %016x %016x %s %s" % (o,a,b,tag(a),tag(b))) + # follow direct-map pointers in this page + for o in range(0,0x1000-8,8): + v=struct.unpack_from("<Q",d,o)[0] + pa=ptr2pa(v) + if pa is None: continue + dd=rd(pa&~0xfff,0x1000) + if not dd: continue + # search that page for PA runs (>=8 page-aligned RAM PAs ascending/descending) + runs=[]; cur=[] + for oo in range(0,0xff8,8): + vv=struct.unpack_from("<Q",dd,oo)[0] + if 0x100000000<=vv<0x180000000 and (vv&0xfff)==0: + cur.append((oo,vv)) + else: + if len(cur)>=8: runs.append(cur) + cur=[] + if len(cur)>=8: runs.append(cur) + if runs: + print(" ptr +0x%03x -> pa=0x%x has %d PA-run(s):" % (o,pa&~0xfff,len(runs))) + for r in runs[:4]: + print(" run at +0x%x: %d pages first=0x%x last=0x%x" % (r[0][0],len(r),r[0][1],r[-1][1])) diff --git a/tools/guest-diag/wall2/unwait.py b/tools/guest-diag/wall2/unwait.py new file mode 100644 index 0000000..94e677c --- /dev/null +++ b/tools/guest-diag/wall2/unwait.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# unwait.py - guest-side, zero-ptrace wait-word poller: writes 1 to the +0x80 +# u32 of every zero 16 KB rw-s nvidiactl mapping of the newest vk_frame. +import os, re, subprocess, sys, time +DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600 +def newest_pid(): + try: + out = subprocess.run(["pgrep", "-n", "-x", "vk_frame"], capture_output=True).stdout + return int(out.decode().strip()) + except Exception: + return None +def maps(pid): + try: + return open("/proc/%d/maps" % pid).read() + except Exception: + return "" +logged = set() +end = time.time() + DUR +while time.time() < end: + pid = newest_pid() + if pid: + for line in maps(pid).splitlines(): + p = line.split() + if len(p) < 6 or "nvidia" not in p[-1] or "rw-s" not in p[1]: + continue + try: + lo, hi = [int(x, 16) for x in p[0].split("-")] + except Exception: + continue + if hi - lo != 0x4000: + continue + wa = lo + 0x80 + try: + fd = os.open("/proc/%d/mem" % pid, os.O_RDWR) + os.lseek(fd, wa, 0) + v = os.read(fd, 4) + if len(v) == 4 and v == b"\x00\x00\x00\x00": + os.lseek(fd, wa, 0) + os.write(fd, b"\x01\x00\x00\x00") + key = (pid, wa) + if key not in logged: + logged.add(key) + print("WORDWROTE pid=%d va=0x%x" % (pid, wa), flush=True) + os.close(fd) + except Exception: + pass + time.sleep(0.4) +print("UNWAIT-DONE", flush=True) diff --git a/tools/guest-diag/wall2/unwait2.py b/tools/guest-diag/wall2/unwait2.py new file mode 100644 index 0000000..cd4169b --- /dev/null +++ b/tools/guest-diag/wall2/unwait2.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# unwait2.py - no-ptrace guest-side harness: +# (a) writes 1 to any zero u32 at +0x80 of every 0x4000-byte rw-s nvidia +# mapping of EVERY live vk_frame; +# (b) scans each live vk_frame's anonymous/heap rw-p mappings (16-byte +# aligned) for marker slots {u32@0==0, u16@+0xe==0xffff} and clears +# u16@+0xe to 0 (the instance-phase pending markers the t105 gdb used +# to clear). No ptrace anywhere. +import os, subprocess, sys, time +DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600 +def pids(): + try: + out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout + return [int(x) for x in out.split()] + except Exception: + return [] +def rd(fd, off, n): + try: + os.lseek(fd, off, 0) + return os.read(fd, n) + except Exception: + return b"" +def wr(fd, off, data): + try: + os.lseek(fd, off, 0) + os.write(fd, data) + return True + except Exception: + return False +wlog = set(); mlog = set() +end = time.time() + DUR +last_scan = 0.0 +while time.time() < end: + for pid in pids(): + try: + fd = os.open("/proc/%d/mem" % pid, os.O_RDWR) + except Exception: + continue + try: + for line in open("/proc/%d/maps" % pid): + p = line.split() + if len(p) < 6 or "nvidia" not in p[-1] or "rw-s" not in p[1]: + continue + lo, hi = [int(x, 16) for x in p[0].split("-")] + if hi - lo != 0x4000: + continue + wa = lo + 0x80 + v = rd(fd, wa, 4) + if len(v) == 4 and v == b"\x00\x00\x00\x00": + if wr(fd, wa, b"\x01\x00\x00\x00"): + k = (pid, wa) + if k not in wlog: + wlog.add(k) + print("WORDWROTE pid=%d va=0x%x" % (pid, wa), flush=True) + except Exception: + pass + if time.time() - last_scan > 1.5: + budget = 64 + try: + for line in open("/proc/%d/maps" % pid): + p = line.split() + if len(p) < 6 or p[1] != "rw-p": + continue + path = p[-1] + if path.startswith("/") and "nvidia" not in path and path != "[heap]": + continue + lo, hi = [int(x, 16) for x in p[0].split("-")] + if hi - lo > 0x10000000: + hi = lo + 0x10000000 + off = lo + while off < hi: + chunk = rd(fd, off, 0x100000) + if len(chunk) < 32: + break + j = 0 + while j + 16 <= len(chunk): + if chunk[j:j+4] == b"\x00\x00\x00\x00" and chunk[j+0xe:j+0x10] == b"\xff\xff": + wa = off + j + 0xe + if (pid, wa) not in mlog: + if wr(fd, wa, b"\x00\x00"): + mlog.add((pid, wa)) + print("MARKCLR pid=%d va=0x%x" % (pid, wa), flush=True) + j += 16 + off += 0x100000 + budget -= 1 + if budget <= 0: + break + if budget <= 0: + break + except Exception: + pass + last_scan = time.time() + os.close(fd) + time.sleep(0.4) +print("UNWAIT2-DONE", flush=True) diff --git a/tools/guest-diag/wall2/vk_alloc_test.py b/tools/guest-diag/wall2/vk_alloc_test.py new file mode 100644 index 0000000..0bcf35a --- /dev/null +++ b/tools/guest-diag/wall2/vk_alloc_test.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# vk_alloc_test.py - reproduce vk_frame's DEVICE_LOCAL image allocation: +# instance -> device -> OPTIMAL R8G8B8A8 storage image -> mem reqs -> +# pick first DEVICE_LOCAL type -> vkAllocateMemory -> report rc/type/size. +import ctypes, sys + +vk = ctypes.CDLL("libvulkan.so.1") + +class VkApplicationInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("pApplicationName", ctypes.c_char_p), ("applicationVersion", ctypes.c_uint32), + ("pEngineName", ctypes.c_char_p), ("engineVersion", ctypes.c_uint32), + ("apiVersion", ctypes.c_uint32)] + +class VkInstanceCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("pApplicationInfo", ctypes.c_void_p), + ("enabledLayerCount", ctypes.c_uint32), ("ppEnabledLayerNames", ctypes.c_void_p), + ("enabledExtensionCount", ctypes.c_uint32), ("ppEnabledExtensionNames", ctypes.c_void_p)] + +class VkMemoryType(ctypes.Structure): + _fields_ = [("propertyFlags", ctypes.c_uint32), ("heapIndex", ctypes.c_uint32)] +class VkMemoryHeap(ctypes.Structure): + _fields_ = [("size", ctypes.c_uint64), ("flags", ctypes.c_uint32)] +class VkPhysicalDeviceMemoryProperties(ctypes.Structure): + _fields_ = [("memoryTypeCount", ctypes.c_uint32), + ("memoryTypes", VkMemoryType * 32), + ("memoryHeapCount", ctypes.c_uint32), + ("memoryHeaps", VkMemoryHeap * 16)] + +class VkDeviceQueueCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("queueFamilyIndex", ctypes.c_uint32), + ("queueCount", ctypes.c_uint32), ("pQueuePriorities", ctypes.c_void_p)] +class VkDeviceCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("queueCreateInfoCount", ctypes.c_uint32), + ("pQueueCreateInfos", ctypes.c_void_p), + ("enabledLayerCount", ctypes.c_uint32), ("ppEnabledLayerNames", ctypes.c_void_p), + ("enabledExtensionCount", ctypes.c_uint32), ("ppEnabledExtensionNames", ctypes.c_void_p), + ("pEnabledFeatures", ctypes.c_void_p)] + +class VkExtent3D(ctypes.Structure): + _fields_ = [("width", ctypes.c_uint32), ("height", ctypes.c_uint32), ("depth", ctypes.c_uint32)] +class VkImageCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), ("flags", ctypes.c_uint32), + ("imageType", ctypes.c_uint32), ("format", ctypes.c_uint32), + ("extent", VkExtent3D), ("mipLevels", ctypes.c_uint32), ("arrayLayers", ctypes.c_uint32), + ("samples", ctypes.c_uint32), ("tiling", ctypes.c_uint32), ("usage", ctypes.c_uint32), + ("sharingMode", ctypes.c_uint32), ("queueFamilyIndexCount", ctypes.c_uint32), + ("pQueueFamilyIndices", ctypes.c_void_p), ("initialLayout", ctypes.c_uint32)] +class VkMemoryRequirements(ctypes.Structure): + _fields_ = [("size", ctypes.c_uint64), ("alignment", ctypes.c_uint64), ("memoryTypeBits", ctypes.c_uint32)] +class VkMemoryAllocateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("allocationSize", ctypes.c_uint64), ("memoryTypeIndex", ctypes.c_uint32)] + +app = VkApplicationInfo(0, None, b"alloctest", 1, b"alloctest", 1, 0x401000) +ci = VkInstanceCreateInfo(1, None, 0, ctypes.cast(ctypes.pointer(app), ctypes.c_void_p), 0, None, 0, None) +inst = ctypes.c_void_p() +print("vkCreateInstance rc=%d" % vk.vkCreateInstance(ctypes.byref(ci), None, ctypes.byref(inst))) + +n = ctypes.c_uint32(0) +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), None) +devs = (ctypes.c_void_p * n.value)() +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), devs) +phys = devs[0] +print("phys=0x%x" % (phys or 0)) + +prio = ctypes.c_float(1.0) +qci = VkDeviceQueueCreateInfo(2, None, 0, 0, 1, ctypes.cast(ctypes.pointer(prio), ctypes.c_void_p)) +dci = VkDeviceCreateInfo(3, None, 0, 1, ctypes.cast(ctypes.pointer(qci), ctypes.c_void_p), 0, None, 0, None, None) +dev = ctypes.c_void_p() +rc = vk.vkCreateDevice(phys, ctypes.byref(dci), None, ctypes.byref(dev)) +print("vkCreateDevice rc=%d dev=0x%x" % (rc, dev or 0)) +if rc != 0: + sys.exit(1) + +mp = VkPhysicalDeviceMemoryProperties() +vk.vkGetPhysicalDeviceMemoryProperties(phys, ctypes.byref(mp)) + +img_ci = VkImageCreateInfo(14, None, 0, 1, 37, # VK_IMAGE_TYPE_2D, R8G8B8A8_UNORM + VkExtent3D(64, 64, 1), 1, 1, 1, 0, # OPTIMAL + 0x2 | 0x1, 0, 0, None, 0) # STORAGE | TRANSFER_SRC, UNDEFINED +img = ctypes.c_void_p() +rc = vk.vkCreateImage(dev, ctypes.byref(img_ci), None, ctypes.byref(img)) +print("vkCreateImage rc=%d img=0x%x" % (rc, img or 0)) +if rc != 0: + sys.exit(1) + +mr = VkMemoryRequirements() +vk.vkGetImageMemoryRequirements(dev, img, ctypes.byref(mr)) +print("memreqs size=0x%x align=0x%x typeBits=0x%x" % (mr.size, mr.alignment, mr.memoryTypeBits)) + +tidx = 0 +found = False +for k in range(mp.memoryTypeCount): + if (mr.memoryTypeBits & (1 << k)) and (mp.memoryTypes[k].propertyFlags & 0x1): + tidx = k + found = True + break +print("selected type=%d found_device_local=%s (types: %s)" % + (tidx, found, ", ".join("%d:0x%x" % (k, mp.memoryTypes[k].propertyFlags) for k in range(mp.memoryTypeCount)))) + +ma = VkMemoryAllocateInfo(5, None, mr.size, tidx) +mem = ctypes.c_void_p() +rc = vk.vkAllocateMemory(dev, ctypes.byref(ma), None, ctypes.byref(mem)) +print("vkAllocateMemory size=0x%x type=%d rc=%d" % (mr.size, tidx, rc)) diff --git a/tools/guest-diag/wall2/vk_memprops.py b/tools/guest-diag/wall2/vk_memprops.py new file mode 100644 index 0000000..3cf6ba2 --- /dev/null +++ b/tools/guest-diag/wall2/vk_memprops.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# vk_memprops.py - enumerate Vulkan devices and dump their memory properties +# via ctypes (no vkCreateDevice needed; safe alongside other clients). +import ctypes, ctypes.util, sys + +vk = ctypes.CDLL("libvulkan.so.1") + +class VkApplicationInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("pApplicationName", ctypes.c_char_p), ("applicationVersion", ctypes.c_uint32), + ("pEngineName", ctypes.c_char_p), ("engineVersion", ctypes.c_uint32), + ("apiVersion", ctypes.c_uint32)] + +class VkInstanceCreateInfo(ctypes.Structure): + _fields_ = [("sType", ctypes.c_uint32), ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), ("pApplicationInfo", ctypes.c_void_p), + ("enabledLayerCount", ctypes.c_uint32), ("ppEnabledLayerNames", ctypes.c_void_p), + ("enabledExtensionCount", ctypes.c_uint32), ("ppEnabledExtensionNames", ctypes.c_void_p)] + +class VkMemoryType(ctypes.Structure): + _fields_ = [("propertyFlags", ctypes.c_uint32), ("heapIndex", ctypes.c_uint32)] + +class VkMemoryHeap(ctypes.Structure): + _fields_ = [("size", ctypes.c_uint64), ("flags", ctypes.c_uint32)] + +class VkPhysicalDeviceMemoryProperties(ctypes.Structure): + _fields_ = [("memoryTypeCount", ctypes.c_uint32), + ("memoryTypes", VkMemoryType * 32), + ("memoryHeapCount", ctypes.c_uint32), + ("memoryHeaps", VkMemoryHeap * 16)] + +app = VkApplicationInfo(0, None, b"memprops", 1, b"memprops", 1, 0x401000) +ci = VkInstanceCreateInfo(1, None, 0, ctypes.cast(ctypes.pointer(app), ctypes.c_void_p), 0, None, 0, None) +inst = ctypes.c_void_p() +rc = vk.vkCreateInstance(ctypes.byref(ci), None, ctypes.byref(inst)) +print("vkCreateInstance rc=%d" % rc) +if rc != 0: + sys.exit(1) + +n = ctypes.c_uint32(0) +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), None) +print("devices=%d" % n.value) +devs = (ctypes.c_void_p * n.value)() +vk.vkEnumeratePhysicalDevices(inst, ctypes.byref(n), devs) + +for i in range(n.value): + props = VkPhysicalDeviceMemoryProperties() + vk.vkGetPhysicalDeviceMemoryProperties(devs[i], ctypes.byref(props)) + print("=== DEV %d phys=0x%x ===" % (i, devs[i] or 0)) + print(" memoryTypeCount=%d memoryHeapCount=%d" % (props.memoryTypeCount, props.memoryHeapCount)) + for t in range(props.memoryTypeCount): + mt = props.memoryTypes[t] + print(" type[%d] flags=0x%08x heap=%d" % (t, mt.propertyFlags, mt.heapIndex)) + for h in range(props.memoryHeapCount): + mh = props.memoryHeaps[h] + print(" heap[%d] size=0x%x (%u MiB) flags=0x%x" % (h, mh.size, mh.size >> 20, mh.flags)) diff --git a/tools/guest-diag/wall2/vmap_watch2.py b/tools/guest-diag/wall2/vmap_watch2.py new file mode 100644 index 0000000..194b0d6 --- /dev/null +++ b/tools/guest-diag/wall2/vmap_watch2.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# Guest-side live watcher v2: scan ALL vmallocinfo areas via /proc/kcore for +# (a) GR ring entry patterns, (b) pages starting with a small INC_METHOD header. +import struct, time +ENTRIES = [bytes.fromhex("0000480400980500"), bytes.fromhex("8000040004480598"), bytes.fromhex("0003d0000448059c")] +class Kcore: + def __init__(self): + self.fd = open("/proc/kcore", "rb") + hdr = self.fd.read(64) + e_phoff = struct.unpack_from("<Q", hdr, 32)[0] + e_phentsize, e_phnum = struct.unpack_from("<HH", hdr, 54) + self.segs = [] + for i in range(e_phnum): + self.fd.seek(e_phoff + i * e_phentsize) + ph = self.fd.read(56) + p_type, _fl, p_off, p_va, _pa, p_sz = struct.unpack_from("<IIQQQQ", ph, 0) + if p_type == 1: + self.segs.append((p_va, p_va + p_sz, p_off)) + def read(self, va, n): + for lo, hi, off in self.segs: + if lo <= va and va + n <= hi: + self.fd.seek(off + (va - lo)) + return self.fd.read(n) + return None +kc = Kcore() +def areas(): + out = [] + for line in open("/proc/vmallocinfo"): + p = line.split() + if len(p) < 2 or "-" not in p[0]: continue + try: + lo, hi = p[0].split("-") + lo, hi = int(lo,16), int(hi,16) + except Exception: + continue + out.append((lo, hi, line.strip()[-60:])) + return out +def hdr_ok(page): + v = struct.unpack_from("<I", page, 0)[0] + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return op == 1 and 1 <= cnt <= 8 +print("vmap_watch2 start", flush=True) +found = set() +t0 = time.time() +it = 0 +while time.time() - t0 < 300: + it += 1 + nareas = 0 + for lo, hi, tag in areas(): + size = hi - lo + if size > 8 << 20: continue + nareas += 1 + d = kc.read(lo, size) + if not d: continue + for pat in ENTRIES: + i = d.find(pat) + if i >= 0: + key = (lo, i & ~0xfff, pat.hex()) + if key not in found: + found.add(key) + print("%.2f RING %s VA 0x%x (+0x%x) %s" % (time.time()-t0, pat.hex(), lo+(i&~0xfff), i&~0xfff, tag), flush=True) + for off in range(0, size - 0x1000, 0x1000): + page = d[off:off+0x1000] + if hdr_ok(page): + key = (lo, off, "h") + if key not in found: + found.add(key) + print("%.2f HDR VA 0x%x (+0x%x) %s head=%s" % (time.time()-t0, lo+off, off, tag, page[:0x30].hex()), flush=True) + if it % 10 == 0: + print("%.1f iter %d areas %d" % (time.time()-t0, it, nareas), flush=True) + time.sleep(0.2) +print("vmap_watch2 done", flush=True) diff --git a/tools/guest-diag/wall2/walk.py b/tools/guest-diag/wall2/walk.py new file mode 100644 index 0000000..3624007 --- /dev/null +++ b/tools/guest-diag/wall2/walk.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# walk.py - host-side replica of the service's svc_gmmu_walk + pd_discover, +# run against /dev/shm/vgpu-guest-ram. Discovers the page directory that +# reproduces the service's map[] sample pairs, then walks the ring/window +# VAs through it. +import mmap, struct, sys + +F = "/dev/shm/vgpu-guest-ram" +SEMA_PA_MAX = 0x180000000 + +# samples from the r690 dmesg map[] lines +SAMPLES = [ + (0x100000000, 0x100326000), + (0x100e2f000, 0x10096e000), + (0x100e33000, 0x10096b000), + (0x100e34000, 0x10096a000), + (0x100e35000, 0x100969000), + (0x100e37000, 0x102dcb000), + (0x100e38000, 0x102dca000), + (0x100e39000, 0x102dc9000), +] + +TARGET_VAS = [0x44a0000, 0x4500000, 0x4560000, 0x4480000, 0x4485000, + 0x4ca0000, 0x4ca5000, 0x4d00000, 0x4b17000, 0x4020000, + 0x4100000, 0x4160000] + +mm = None + +def off(pa): + return pa if pa < 0x80000000 else pa - 0x80000000 + +def rd(pa, n=8): + if pa + n > SEMA_PA_MAX: + return None + try: + d = mm[off(pa):off(pa)+n] + return d if len(d) == n else None + except Exception: + return None + +def rd64(pa): + d = rd(pa, 8) + return None if d is None else int.from_bytes(d, "little") + +def pde_aperture(ap): + if ap == 1: return "FB" + if ap in (2, 3): return "SYS" + return None + +def pte_aperture(ap): + if ap == 0: return "FB" + if ap == 1: return None + return "SYS" + +def pte_addr(e, fb): + if fb: + return ((e >> 8) & 0x1FFFFFF) << 12 + return ((e >> 8) & 0x3FFFFFFFFFFF) << 12 + +def pte_leaf(table, idx): + e = rd64(table + 8 * idx) + if e is None or not (e & 1): + return None + ap = pte_aperture((e >> 1) & 0x3) + if ap is None: + return None + a = pte_addr(e, ap == "FB") + if not fb_ok(a): + return None + return a, ap + +def fb_ok(a): + return a + 8 <= SEMA_PA_MAX + +def walk(va): + table = None + # root_bits = ilog2(entries=4) = 2; shift = 49-2 = 47 + e = rd64(table) if False else None + # level 0 + idx = (va >> 47) & 0x3 + # root table is the candidate itself (passed separately) + return None + +def walk_from(pdb, va, trace=False): + table = pdb + for level in range(3): + if level == 0: + bits, shift = 2, 47 + else: + bits, shift = 9, 38 if level == 1 else 29 + idx = (va >> shift) & ((1 << bits) - 1) + e = rd64(table + 8 * idx) + if e is None: + return None + if trace: + print(" L%d table=0x%x idx=%u e=0x%016x" % (level, table, idx, e)) + if e & 1: + return None + ap = pde_aperture((e >> 1) & 0x3) + if ap is None: + return None + table = pte_addr(e, ap == "FB") + if not fb_ok(table): + return None + idx = (va >> 21) & 0xff + big = rd64(table + 16 * idx) + small = rd64(table + 16 * idx + 8) + if big is None or small is None: + return None + if trace: + print(" PD0 table=0x%x idx=%u big=0x%016x small=0x%016x" % + (table, idx, big, small)) + if not (small & 1): + sap = pde_aperture((small >> 1) & 0x3) + if sap is not None: + t = pte_addr(small, sap == "FB") + r = pte_leaf(t, (va >> 12) & 0x1ff) + if r: + return r[0] + (va & 0xfff), r[1] + if big & 1: + bap = pte_aperture((big >> 1) & 0x3) + if bap is not None: + a = pte_addr(big, bap == "FB") + (va & ((1 << 21) - 1)) + if fb_ok(a): + return a, bap + else: + bap = pde_aperture((big >> 1) & 0x3) + if bap is not None: + t = pte_addr(big, bap == "FB") + r = pte_leaf(t, (va >> 16) & 0x1f) + if r: + return r[0] + (va & 0xffff), r[1] + return None + +def main(): + global mm + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + size = len(mm) + print("# scanning for directory candidates...") + cands = [] + for fo in range(0, size, 4096): + e = int.from_bytes(mm[fo:fo+8], "little") + if e and not (e & 1): + ap = (e >> 1) & 0x3 + if ap in (1, 2, 3): + pa = fo if fo < 0x80000000 else fo + 0x80000000 + if pa < SEMA_PA_MAX: + cands.append(pa) + print("# candidates: %d" % len(cands)) + best = [] + for c in cands: + hits = 0 + for (sva, spa) in SAMPLES: + r = walk_from(c, sva) + if r and (r[0] & ~0xfff) == (spa & ~0xfff): + hits += 1 + if hits >= 2: + best.append((hits, c)) + best.sort(reverse=True) + print("# candidates with >=2 hits: %d" % len(best)) + for hits, c in best[:10]: + print(" pdb=0x%x hits=%d/8" % (c, hits)) + if not best: + return 1 + pdb = best[0][1] + print("== walking targets through pdb=0x%x:" % pdb) + for va in TARGET_VAS: + r = walk_from(pdb, va) + if r: + print(" va=0x%x -> pa=0x%x ap=%s" % (va, r[0], r[1])) + else: + print(" va=0x%x -> no translation" % va) + mm.close(); f.close() + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap.py b/tools/guest-diag/wall2/wall2_livecap.py new file mode 100644 index 0000000..af68c03 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# wall2_livecap.py - live capture of the GR channel-init push page. +# +# The release group in the push is {0x20038090, sem_va_hi, sem_va_lo, payload, +# continuation}: sem_va = (hi<<32)|lo, and for the channel-init entry the sem +# sits at push_va + entry_len (entry0: len=1432=0x598, sem at +0x598), with the +# group itself at push_va + entry_len - 24 = +0x580. +# +# This watcher scans guest RAM for 0x20038090 groups during the client's stall, +# classifies them, and injects push_va -> page_pa via sema_map_add when the +# group's sem VA and offset identify the push page. Prints full evidence. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap.log" +MASK = ~0xfff & ((1 << 64) - 1) + +ENTRY_LEN = int(os.environ.get("W2_LEN", "1432")) +WATCH_S = int(os.environ.get("W2_WATCH", "120")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def cont_ok(v): + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return (op in (1, 3, 5) and cnt != 0) or op in (4, 7) + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + g = d[i:i+24] + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, hi, lo, pay, w5, v5, g)) + pos += CH + return out + +def main(): + log("=== livecap start len=%d watch=%ds ===" % (ENTRY_LEN, WATCH_S)) + injected = {} + deadline = time.time() + WATCH_S + while time.time() < deadline: + hits = scan() + cands = [] + for (page, off, hi, lo, pay, w5, v5, g) in hits: + sem = (hi << 32) | lo + # classify + pool = (hi == 1 and lo < 0x10000 and 0 < pay < (1 << 24) + and (w5 >> 29) == 1 and ((w5 >> 13) & 7) == 4 + and ((w5 & 0x1fff) << 2) == 0x300 and (v5 & 3) == 0) + if pool: + continue # CE pool release family, not the push + if not (0x4000000 <= sem < 0x100000000): + continue + if not (0 < pay < (1 << 24)): + continue + if not cont_ok(w5): + continue + push_va = sem - ENTRY_LEN + cands.append((page, off, sem, pay, push_va, w5, v5, g)) + if cands: + # prefer the canonical group offset 0x580 / sem-consistent push VA + cands.sort(key=lambda c: (c[1] != (ENTRY_LEN - 24), -c[3])) + page, off, sem, pay, push_va, w5, v5, g = cands[0] + d = rd(page, 0x100) + log("livecap: %d cand(s); page=0x%x off=0x%x sem=0x%x pay=%u push_va=0x%x w5=0x%08x v5=0x%08x" + % (len(cands), page, off, sem, pay, push_va, w5, v5)) + log(" page head: %s" % d[:0x40].hex()) + if (push_va & 0xfff) == 0 and push_va not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (push_va, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x (off=0x%x)" % (push_va, page, off)) + injected[push_va] = page + except OSError as e: + log(" inject FAIL", e) + time.sleep(1) + log("=== livecap done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap2.py b/tools/guest-diag/wall2/wall2_livecap2.py new file mode 100644 index 0000000..a33b25b --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap2.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# wall2_livecap2.py - capture the GR channel-init push page by its release +# group {0x20038090, hi, lo, payload, continuation} at page offset 0x598. +# The push VA comes from the service's own entry[0] decode in dmesg. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap2.log" +MASK = ~0xfff & ((1 << 64) - 1) +WATCH_S = int(os.environ.get("W2_WATCH", "180")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def prefix_ok(d): + off = 0 + while off + 4 <= len(d): + hdr = struct.unpack_from("<I", d, off)[0] + if hdr: break + off += 4 + if off + 4 > len(d): return False + op = (hdr >> 29) & 7 + if op in (4, 7): return True + if op not in (1, 3, 5): return False + return ((hdr >> 16) & 0xfff) != 0 + +def cont_ok(v): + op = (v >> 29) & 7 + cnt = (v >> 16) & 0xfff + return (op in (1, 3, 5) and cnt != 0) or op in (4, 7) + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + g = d[i:i+24] + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, hi, lo, pay, w5, v5)) + pos += CH + return out + +def main(): + log("=== livecap2 start watch=%ds ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + seen_pages = {} + while time.time() < deadline: + hits = scan() + for (page, off, hi, lo, pay, w5, v5) in hits: + if not (0 < pay < (1 << 24)): + continue + if not cont_ok(w5): + continue + d = rd(page, 0x1000) + if not d: continue + pok = prefix_ok(d[:0x100]) + key = (page, off) + if key in seen_pages: + continue + seen_pages[key] = 1 + log("hit page=0x%x off=0x%x sem=0x%x pay=%u w5=0x%08x v5=0x%08x prefix_ok=%s" + % (page, off, (hi << 32) | lo, pay, w5, v5, pok)) + log(" head: %s" % d[:0x30].hex()) + log(" @0x590: %s" % d[0x590:0x5c0].hex()) + if pok and off in (0x598, 0x590, 0x580, 0x588): + va = 0x4480000 + if va not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (va, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x" % (va, page)) + injected.add(va) + except OSError as e: + log(" inject FAIL", e) + time.sleep(0.5) + log("=== livecap2 done: %d injections ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_livecap3.py b/tools/guest-diag/wall2/wall2_livecap3.py new file mode 100644 index 0000000..db28bc2 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_livecap3.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# wall2_livecap3.py - capture the UVM GR channel-init push page via the host +# (C46F) semaphore release group emitted by uvm_hal_turing_host_semaphore_release: +# {0x20050017 (INC_METHOD count=5 addr=0x5c), sem_lo, sem_hi, payload, 0, exec} +# The group sits at push_va + entry_len - 24 = push_va + 0x580 (entry0 len 1432), +# with the header at +0x598 (entry1's 4-byte push). The push VA comes from the +# service's own entry[0] decode in dmesg. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_livecap3.log" +MASK = ~0xfff & ((1 << 64) - 1) +WATCH_S = int(os.environ.get("W2_WATCH", "240")) +PUSH_VA_DEFAULT = 0x4480000 + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def push_va_now(): + # newest entry[0] line for the Vulkan client's channel + va = None + for ln in dmesg().splitlines(): + m = re.search(r"entry\[0\] h=0x([0-9a-f]+) eng=0 raw=\S+ va=0x([0-9a-f]+) len=(\d+)", ln) + if m: + va = int(m.group(2), 16) + return va + +def scan(): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<I", 0x20050017) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + lo, hi, pay, w4, w5 = struct.unpack_from("<5I", d, i + 4) + # data-word sanity: payload_hi==0, RELEASE op, sane payload + if w4 != 0: continue + if (w5 & 0x7) != 1: continue + if not (0 < pay < (1 << 24)): continue + sem = ((hi & 0xffffffff) << 32) | ((lo & 0x3fffffff) << 2) + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, sem, pay, w5)) + pos += CH + return out + +def main(): + log("=== livecap3 start watch=%ds ===" % WATCH_S) + injected = {} + seen = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + hits = scan() + if hits: + for (page, off, sem, pay, w5) in hits: + key = (page, off) + if key in seen: continue + seen.add(key) + d = rd(page, 0x1000) + head = d[:0x20].hex() if d else "?" + log("hit page=0x%x off=0x%x sem=0x%x pay=%u exec=0x%08x head=%s" + % (page, off, sem, pay, w5, head)) + log(" @0x580: %s" % (d[0x580:0x5b0].hex() if d else "?")) + if off in (0x598, 0x580, 0x590): + pv = push_va_now() or PUSH_VA_DEFAULT + if pv not in injected: + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,1\n" % (pv, page, 0x1000)) + log(" INJECT 0x%x -> 0x%x" % (pv, page)) + injected[pv] = page + except OSError as e: + log(" inject FAIL", e) + time.sleep(0.5) + log("=== livecap3 done: %d injections ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_recscan.py b/tools/guest-diag/wall2/wall2_recscan.py new file mode 100644 index 0000000..7bc4e2c --- /dev/null +++ b/tools/guest-diag/wall2/wall2_recscan.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# wall2_recscan.py - scan the guest RAM for the RM's low-aperture window +# records and their memdesc PAs. +# +# Record layout (verified r685/r686): 0x40-byte stride +# {u64 tag; u64 VA; u64 SIZE; u64 0; u32 0x7461696c ("tail" as a u32 - +# on disk the BYTES are "liat"); u32 0} +# The tag is NOT the byte string "tail" - searching for b'tail' finds +# nothing; search for the 4-byte pattern 6c 69 61 74 (struct '<I' +# 0x7461696c) at record+0x20 and decode {tag,VA,SIZE} at -0x20/-0x18/-0x10. +# +# Usage: wall2_recscan.py [VA_prefix_or_full_VA] +# Prints every record and the size-matching memdesc {SIZE,0x03008000,PA} +# found in the same page, and dumps the first words of each memdesc page so +# semaphore windows can be identified by the release payload values. +import struct, sys + +RAM = "/dev/shm/vgpu-guest-ram" +TAG = struct.pack("<I", 0x7461696c) + +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) + +def gpa2off(pa): + if pa < 0x80000000: + return pa + if pa < 0x180000000: + return 0x80000000 + (pa - 0x100000000) + return None + +def main(): + want = None + if len(sys.argv) > 1: + want = int(sys.argv[1], 0) + img = open(RAM, "rb", buffering=0) + img.seek(0, 2); fsz = img.tell() + recs = [] + pos = 0; CH = 64 << 20 + while pos < fsz: + img.seek(pos); buf = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = buf.find(TAG, st) + if i < 0: + break + st = i + 1 + if i >= 0x20: + tag, va, sz, z = struct.unpack_from("<QQQQ", buf, i - 0x20) + if 0x4000000 <= va < 0x100000000 and \ + 0x1000 <= sz <= 0x1000000 and (sz & 0xfff) == 0 and z == 0: + if want is None or va == want or (va & ~0xfffff) == (want & ~0xfffff): + recs.append((off2gpa(pos + i - 0x20), tag, va, sz)) + pos += CH + for gpa, tag, va, sz in sorted(recs, key=lambda r: r[2]): + page = gpa & ~0xfff + img.seek(gpa2off(page)); d = img.read(0x1000) + descs = [] + for o in range(0, 0x1000 - 24, 8): + s2, f2 = struct.unpack_from("<QQ", d, o) + pa = struct.unpack_from("<Q", d, o + 16)[0] + if f2 == 0x03008000 and s2 == sz and 0x100000000 <= pa < 0x180000000: + descs.append(pa) + print("rec gpa=0x%x tag=0x%x va=0x%x size=0x%x" % (gpa, tag, va, sz)) + for pa in descs: + img.seek(gpa2off(pa)); pd = img.read(0x20) + print(" desc pa=0x%x head=%s" % (pa, pd.hex(" "))) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver2.py b/tools/guest-diag/wall2/wall2_resolver2.py new file mode 100644 index 0000000..785e5e2 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver2.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# wall2_resolver2.py - HOST-SIDE wall#2 resolver v2 (r674+). +# +# Why: the vk_frame client's channel-init push (va=0x4500000 len=1432) is +# kernel-built and kernel-only; the sema service cannot translate it +# ("false translation for push ... rescuing"), the client times out (~11 s) +# per attempt, retries, then VKFAIL -4. Guest RAM is host-readable, so the +# push page can be found and injected via sema_map_add. +# +# v2 recipe (per the r672/r673 structure analysis): +# 1. wait for the channel ring line in dmesg (validated RING_PA + handle) +# 2. scan guest RAM for the push page: +# a) CONTENT: the 24-byte release group the service's own rescue matches +# (hdr=0x20038090, hi=1, payload sane, w5/v5 shape). The channel-init +# entry (va=0x4500000 len=1432) puts that group at page offset 0x580. +# b) TUPLES: pages of 32-byte records {vmap_ptr; flags; PA; size} (both +# observed layouts) matched on (PA & ~0xfff) == RING_PA with strides +# 0x20/0x40/0x60; entries below the ring entry in VA order map +# 0x4500000.. (push VA = ring VA 0x4520000 - 0x20000 = 32 pages). +# 3. inject (va -> pa, len) through /sys/module/vgsp/parameters/sema_map_add +# 4. keep scanning/re-injecting for the watch window (retries allocate new +# pages) and log everything. +# +# Run this FIRST (foreground), then launch the client in the guest. +import subprocess, struct, time, sys, re, os, signal + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver2.log" +RING_VA = 0x4520000 +PUSH_VA = 0x4500000 +DELTA = (RING_VA - PUSH_VA) // 0x1000 # 32 pages +REL_OFF = 0x580 # group offset in the push page +MASK = ~0xfff & ((1 << 64) - 1) + +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "150")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,%d\n" % (va, pa, ln, ap)) + log("INJECT va=0x%x -> pa=0x%x len=0x%x" % (va, pa, ln)) + return True + except OSError as e: + log("inject FAIL", e) + return False + +def prefix_ok(d): + off = 0 + while off + 4 <= len(d): + hdr = struct.unpack_from("<I", d, off)[0] + if hdr: + break + off += 4 + if off + 4 > len(d): + return False + op = (hdr >> 29) & 7 + if op in (4, 7): + return True + if op not in (1, 3, 5): + return False + return ((hdr >> 16) & 0xfff) != 0 + +def rel_shape(g): + hdr, hi, lo, pay, w5, v5 = struct.unpack("<6I", g) + return (hdr == 0x20038090 and hi == 1 and 0 < pay < (1 << 24) + and (w5 >> 29) == 1 and ((w5 >> 13) & 7) == 4 + and ((w5 & 0x1fff) << 2) == 0x300 and (v5 & 3) == 0), lo & 0xfff, pay + +img = None +def open_ram(): + global img + if img is None: + img = open(RAM, "rb", buffering=0) + return img + +def gpa2off(pa): + if pa < 0x80000000: + return pa + if pa < 0x180000000: + return 0x80000000 + (pa - 0x100000000) + return None + +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) + +def rd(pa, n): + off = gpa2off(pa) + if off is None: + return None + f = open_ram() + try: + f.seek(off) + return f.read(n) + except OSError: + return None + +def find_content_candidates(): + """Pages containing the release group; returns [(page_pa, off, slot, pay)].""" + f = open_ram() + f.seek(0, 2) + fsz = f.tell() + pat = b"\x90\x80\x03\x20" + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + f.seek(pos) + d = f.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: + break + st = i + 1 + if i + 24 > len(d): + continue + ok, slot, pay = rel_shape(d[i:i + 24]) + if not ok: + continue + gpa = off2gpa(pos + i) + page = gpa & MASK + off = gpa - page + out.append((page, off, slot, pay)) + pos += CH + return out + +def find_tuple_arrays(ring_pa): + """Find record pages containing the ring PA; returns list of + (page_gpa, stride, ring_index, entries[list of pa]).""" + f = open_ram() + f.seek(0, 2) + fsz = f.tell() + pat = struct.pack("<Q", ring_pa) + arrays = [] + CH = 64 << 20 + pos = 0 + hits = [] + while pos < fsz: + f.seek(pos) + d = f.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: + break + hits.append(off2gpa(pos + i)) + st = i + 1 + pos += CH + seen = set() + for gpa in hits: + page = gpa & MASK + if page in seen: + continue + seen.add(page) + d = rd(page, 0x1000) + if not d: + continue + rec_off = gpa - page + for stride in (0x20, 0x40, 0x60): + for rel in range(0, min(stride, rec_off + 1), 8): + start = rec_off - rel + ents = [] + for o in range(start, 0x1000 - stride + 1, stride): + # PA field anywhere in the record: try +0x10 first, then any qword + vals = struct.unpack_from("<%dQ" % (stride // 8), d, o) + pa = None + for cand_off in (0x10, 0x8, 0x18, 0x0, 0x20, 0x28): + if cand_off + 8 <= stride: + v = vals[cand_off // 8] + if (v & MASK) == ring_pa: + pa = v & MASK + break + if pa is None: + continue + ents.append((o, pa)) + if len(ents) >= 4: + idx = [e[1] for e in ents].index(ring_pa) + arrays.append((page, stride, start, idx, [e[1] for e in ents])) + break + if arrays and arrays[-1][0] == page: + break + return arrays + +def main(): + log("=== wall2_resolver2 start (wait_ring=%ss watch=%ss) ===" % (WAIT_RING_S, WATCH_S)) + ring_pa = None + ring_h = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + for ln in dmesg().splitlines(): + m = re.search(r"h=(0x[0-9a-f]+) .*ring rc=0 pa=0x([0-9a-f]+)", ln) + if m: + ring_h = m.group(1) + ring_pa = int(m.group(2), 16) + break + if ring_pa: + break + time.sleep(1) + if not ring_pa: + log("no ring line seen in %ss; exiting" % WAIT_RING_S) + return 1 + log("RING %s PA=0x%x" % (ring_h, ring_pa)) + + injected = set() + watch_deadline = time.time() + WATCH_S + last_pass = 0 + while time.time() < watch_deadline: + # ---- content pass + try: + cands = find_content_candidates() + except Exception as e: + log("content scan error:", e) + cands = [] + # prefer the canonical group offset 0x580, then highest payload + cands.sort(key=lambda c: (c[1] != REL_OFF, -c[3])) + if cands: + page, off, slot, pay = cands[0] + d = rd(page, 0x1000) + pok = bool(d) and prefix_ok(d[:0x100]) + log("content: %d candidate(s); best page=0x%x off=0x%x slot=0x%x pay=%u prefix_ok=%s" + % (len(cands), page, off, slot, pay, pok)) + key = ("C", PUSH_VA, page) + if key not in injected and pok: + if inject(PUSH_VA, page, 0x1000): + injected.add(key) + # also cover the following page (entry2 tail) if non-contiguous unknown; skip + else: + # ---- tuple fallback (only every ~10 s to save time) + if time.time() - last_pass > 10: + last_pass = time.time() + try: + arrs = find_tuple_arrays(ring_pa) + except Exception as e: + log("tuple scan error:", e) + arrs = [] + if arrs: + page, stride, start, idx, ents = arrs[0] + log("tuple: array page=0x%x stride=0x%x ring_index=%d entries=%d" + % (page, stride, idx, len(ents))) + for k in range(idx - DELTA, idx): + if k < 0 or k >= len(ents): + continue + va = PUSH_VA + (k - (idx - DELTA)) * 0x1000 + pa = ents[k] + key = ("T", va, pa) + if key in injected: + continue + d = rd(pa, 0x40) + if d and any(d): + inject(va, pa, 0x1000) + injected.add(key) + else: + log("tuple: no array with ring PA found") + # stop early when the service confirms our mapping + if len(injected) and "injected mapping va=0x%x" % PUSH_VA in dmesg(): + log("service confirmed the injected mapping; done") + break + time.sleep(1) + log("=== done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver3.py b/tools/guest-diag/wall2/wall2_resolver3.py new file mode 100644 index 0000000..4b16686 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver3.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# wall2_resolver3.py - HOST-SIDE wall#2 resolver v3: the RM window-memdesc recipe. +# +# Proven structure (r674/r675): the RM keeps per-window records +# {..., VA, SIZE, ...} (head/tail list nodes) +# and per-window memdescs in the same page: +# {SIZE, 0x03008000, FIRST_PA, ptr} +# where FIRST_PA is the PA of the window's FIRST VA page. Verified: the ring +# window {VA=0x44a0000, SIZE=0x40000} -> memdesc PA == the service's validated +# ring pa (0x1158ce000 r675). Therefore the push window +# {VA=0x4480000, SIZE=0x20000} -> memdesc PA == PA of VA 0x4480000 (the push +# page the sema service cannot translate). +# +# Recipe: wait for the ring line -> find the ring window memdesc +# {0x40000, 0x03008000, RING_PA} -> in the same page find the push window +# memdesc {0x20000, 0x03008000, PUSH_PA} -> inject 0x4480000 -> PUSH_PA. +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver3.log" +PUSH_VA = 0x4480000 +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "150")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + try: + with open(PARAM, "w") as f: + f.write("0x%x,0x%x,0x%x,%d\n" % (va, pa, ln, ap)) + log("INJECT va=0x%x -> pa=0x%x len=0x%x" % (va, pa, ln)) + return True + except OSError as e: + log("inject FAIL", e) + return False + +def find_memdescs(size): + """Find {size, 0x03008000, PA} patterns; returns [(page, off, pa)].""" + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", size, 0x03008000) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + if i + 24 > len(d): continue + pa = struct.unpack_from("<Q", d, i + 16)[0] + gpa = off2gpa(pos + i) + if 0x100000000 <= pa < 0x180000000: + out.append((gpa & ~0xfff, gpa - (gpa & ~0xfff), pa)) + pos += CH + return out + +def main(): + log("=== resolver3 start ===") + # count pre-existing ring lines; wait for a NEW one (fresh client kick) + def ring_lines(): + out = [] + for ln in dmesg().splitlines(): + m = re.search(r"ring rc=0 pa=0x([0-9a-f]+) ap=", ln) + if m: + out.append(int(m.group(1), 16)) + return out + base = ring_lines() + log("pre-existing ring lines: %d" % len(base)) + ring_pa = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + cur = ring_lines() + if len(cur) > len(base): + ring_pa = cur[-1] + break + time.sleep(1) + if not ring_pa: + log("no NEW ring line; exit"); return 1 + log("NEW RING PA = 0x%x" % ring_pa) + + injected = set() + watch = time.time() + WATCH_S + while time.time() < watch: + # ring window memdesc {0x40000, 0x03008000, ring_pa} + rd_hits = [x for x in find_memdescs(0x40000) if x[2] == ring_pa] + if not rd_hits: + time.sleep(0.5); continue + for (page, off, _pa) in rd_hits: + d = rd(page, 0x1000) + if not d: continue + # find sibling push memdesc in the same page + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if not (0x100000000 <= pa < 0x180000000): + continue + key = (page, pa) + if key in injected: continue + log("memdesc page=0x%x ring_desc@0x%x push_desc@0x%x push_pa=0x%x" + % (page, off, o, pa)) + # content sanity: read the push page + pd = rd(pa, 0x40) + log(" push page head: %s" % (pd.hex() if pd else "?")) + if inject(PUSH_VA, pa, 0x1000): + injected.add(key) + if injected and ("injected mapping va=0x%x" % PUSH_VA) in dmesg(): + log("service confirmed; keep watching") + time.sleep(0.5) + log("=== resolver3 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver4.py b/tools/guest-diag/wall2/wall2_resolver4.py new file mode 100644 index 0000000..140aaa7 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver4.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# wall2_resolver4.py - HOST-SIDE wall#2 resolver v4 (final recipe). +# +# Association proven on r675: the RM keeps, in ONE page, the client's channel +# window records and their memdescs: +# push window record {VA=0x4480000, SIZE=0x20000} +# ring window record {VA=0x44a0000, SIZE=0x40000} +# push memdesc {0x20000, 0x03008000, FIRST_PA, ptr} +# ring memdesc {0x40000, 0x03008000, FIRST_PA, ptr} +# where FIRST_PA is the PA of the window's FIRST VA page. Verified: the ring +# memdesc's PA == the service's validated ring pa. So the push memdesc's PA == +# the PA of VA 0x4480000 (the push page the service cannot translate). +# +# Recipe: wait for a NEW ring line -> find the page containing the ring window +# record {0x44a0000, 0x40000} -> require the sibling push window record +# {0x4480000, 0x20000} and the memdescs in the same page -> cross-check the +# ring memdesc PA == ring_pa -> inject 0x4480000 -> push memdesc PA (sudo). +import subprocess, struct, time, sys, re, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver4.log" +PUSH_VA = 0x4480000 +RING_VA = 0x44a0000 +WAIT_RING_S = int(os.environ.get("W2_WAIT_RING", "420")) +WATCH_S = int(os.environ.get("W2_WATCH", "180")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def dmesg(): + return subprocess.run(["sudo", "dmesg"], capture_output=True, text=True).stdout + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x len=0x%x OK" % (va, pa, ln)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + """Find qword pairs {a,b}; returns [(page, off)].""" + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + gpa = off2gpa(pos + i) + out.append((gpa & ~0xfff, gpa - (gpa & ~0xfff))) + pos += CH + return out + +def scan_page_descs(page): + d = rd(page, 0x1000) + if not d: return [] + out = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if fl != 0x03008000: continue + if sz not in (0x20000, 0x40000): continue + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + out.append((sz, o, pa)) + return out + +def main(): + log("=== resolver4 start ===") + def ring_lines(): + return re.findall(r"ring rc=0 pa=0x([0-9a-f]+) ap=", dmesg()) + base = ring_lines() + log("pre-existing ring lines: %d" % len(base)) + ring_pa = None + deadline = time.time() + WAIT_RING_S + while time.time() < deadline: + cur = ring_lines() + if len(cur) > len(base): + ring_pa = int(cur[-1], 16) + break + time.sleep(1) + if not ring_pa: + log("no NEW ring line; exit"); return 1 + log("NEW RING PA = 0x%x" % ring_pa) + + injected = set() + LEARNED = [0x101e39000, 0x102a62000] # RM window-record pages (stable across runs) + def try_page(page): + descs = scan_page_descs(page) + rdesc = [x for x in descs if x[0] == 0x40000] + pdesc = [x for x in descs if x[0] == 0x20000] + if not rdesc or not pdesc: + return False + if rdesc[0][2] != ring_pa: + return False + ok = False + for (_sz, o, pa) in pdesc: + key = (page, o) + if key in injected: continue + pd = rd(pa, 0x40) + log("page=0x%x ring_desc@0x%x(pa=0x%x OK) push_desc@0x%x pa=0x%x head=%s" + % (page, rdesc[0][1], rdesc[0][2], o, pa, + pd.hex() if pd else "?")) + if inject(PUSH_VA, pa, 0x1000): + injected.add(key); ok = True + return ok + log("learned pages: %s" % [hex(p) for p in LEARNED]) + watch = time.time() + WATCH_S + last_full = 0 + while time.time() < watch: + hit = False + for page in LEARNED: + if try_page(page): + hit = True + if not hit and time.time() - last_full > 5: + last_full = time.time() + for (page, _off) in find_qpair(RING_VA, 0x40000): + if try_page(page): + if page not in LEARNED: + LEARNED.append(page) + hit = True + time.sleep(0.25) + log("=== resolver4 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver5.py b/tools/guest-diag/wall2/wall2_resolver5.py new file mode 100644 index 0000000..faec902 --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver5.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# wall2_resolver5.py - final: continuous scan, all-candidate injection. +# +# Structure (r675, verified): an RM page holds the client channel's window +# records and memdescs: +# {0x4480000, 0x20000} push window record +# {0x44a0000, 0x40000} ring window record +# {0x20000, 0x03008000, PUSH_PA} push window memdesc +# {0x40000, 0x03008000, RING_PA} ring window memdesc (PA == validated ring pa) +# The resolver scans CONTINUOUSLY (no ring-line gate), finds pages holding the +# pair of window records, and injects 0x4480000 -> every 0x20000 memdesc PA in +# the same page (last write wins in the service; content check arbitrates). +import subprocess, struct, time, sys, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver5.log" +PUSH_VA = 0x4480000 +RING_VA = 0x44a0000 +WATCH_S = int(os.environ.get("W2_WATCH", "600")) + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x" % (va, pa)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + gpa = off2gpa(pos + i) + out.append(gpa & ~0xfff) + pos += CH + return out + +def main(): + log("=== resolver5 start (watch=%ds) ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + try: + push_pages = set(find_qpair(PUSH_VA, 0x20000)) + ring_pages = set(find_qpair(RING_VA, 0x40000)) + except Exception as e: + log("scan error:", e); time.sleep(2); continue + for page in sorted(push_pages & ring_pages): + d = rd(page, 0x1000) + if not d: continue + # collect 0x20000 descs in this page + cands = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + cands.append((o, pa)) + for (o, pa) in cands: + key = (page, o) + if key in injected: continue + pd = rd(pa, 0x40) + nz = sum(1 for b in pd if b) if pd else -1 + log("page=0x%x desc@0x%x pa=0x%x live_nz=%d head=%s" + % (page, o, pa, nz, pd.hex()[:64] if pd else "?")) + if inject(PUSH_VA, pa): + injected.add(key) + time.sleep(1) + log("=== resolver5 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver6.py b/tools/guest-diag/wall2/wall2_resolver6.py new file mode 100644 index 0000000..4653efa --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver6.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# wall2_resolver6.py - resolver5 logic for BOTH current push windows: +# pair A: push 0x4480000 (0x20000) + ring 0x44a0000 (0x40000) +# pair B: push 0x4ca0000 (0x20000) + ring 0x4cc0000 (0x40000) +# Scans guest RAM continuously; injects PUSH_VA -> every 0x20000 memdesc PA +# found in a page holding both window records. +import subprocess, struct, time, sys, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver6.log" +WATCH_S = int(os.environ.get("W2_WATCH", "420")) +PAIRS = [(0x4480000, 0x44a0000), (0x4ca0000, 0x4cc0000)] + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def off2gpa(off): + return off if off < 0x80000000 else 0x100000000 + (off - 0x80000000) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x" % (va, pa)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + out.append(off2gpa(pos + i) & ~0xfff) + pos += CH + return out + +def main(): + log("=== resolver6 start (watch=%ds) ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + for (pva, rva) in PAIRS: + try: + pages = set(find_qpair(pva, 0x20000)) & set(find_qpair(rva, 0x40000)) + except Exception as e: + log("scan error:", e); continue + for page in sorted(pages): + d = rd(page, 0x1000) + if not d: continue + cands = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + if sz == 0x20000 and fl == 0x03008000: + pa = struct.unpack_from("<Q", d, o + 16)[0] + if 0x100000000 <= pa < 0x180000000: + cands.append((o, pa)) + for (o, pa) in cands: + key = (pva, page, o) + if key in injected: continue + pd = rd(pa, 0x40) + nz = sum(1 for b in pd if b) if pd else -1 + log("pair=0x%x page=0x%x desc@0x%x pa=0x%x live_nz=%d head=%s" + % (pva, page, o, pa, nz, pd.hex()[:64] if pd else "?")) + if inject(pva, pa): + injected.add(key) + time.sleep(1) + log("=== resolver6 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/guest-diag/wall2/wall2_resolver7.py b/tools/guest-diag/wall2/wall2_resolver7.py new file mode 100644 index 0000000..5d9e58e --- /dev/null +++ b/tools/guest-diag/wall2/wall2_resolver7.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# wall2_resolver7.py - resolver6 + release-window injection + live-first ordering. +# Fixes the resolver6 zero-page overwrite: candidates are injected good-first +# (live_nz>0) and zero pages last, so the readable mapping wins by order if the +# service keeps the newest write. +# New: the GR init release windows (0x4b16000 size 0x1000 from the record page, +# and any {VA,0x1000} record alongside) are injected from the size-0x1000 +# memdescs of the same page. +import subprocess, struct, time, sys, os + +RAM = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" +LOG = "/tmp/opencode/wall2_resolver7.log" +WATCH_S = int(os.environ.get("W2_WATCH", "420")) +PAIRS = [(0x4480000, 0x44a0000), (0x4ca0000, 0x4cc0000)] +REL_WINS = [0x4b16000] + +logf = open(LOG, "a", buffering=1) +def log(*a): + s = " ".join(str(x) for x in a) + print(s, flush=True) + logf.write("[%s] %s\n" % (time.strftime("%H:%M:%S"), s)) + +img = open(RAM, "rb", buffering=0) +def off2gpa(off): return off if off < 0x80000000 else 0x100000000+(off-0x80000000) +def gpa2off(pa): + if pa < 0x80000000: return pa + if pa < 0x180000000: return 0x80000000 + (pa - 0x100000000) + return None +def rd(pa, n): + off = gpa2off(pa) + if off is None: return None + img.seek(off); return img.read(n) + +def inject(va, pa, ln=0x1000, ap=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va, pa, ln, ap, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0: + log("INJECT va=0x%x -> pa=0x%x" % (va, pa)) + return True + log("inject FAIL rc=%d %s" % (r.returncode, r.stderr.strip())) + return False + +def find_qpair(a, b): + img.seek(0, 2); fsz = img.tell() + pat = struct.pack("<QQ", a, b) + out = [] + CH = 64 << 20 + pos = 0 + while pos < fsz: + img.seek(pos); d = img.read(min(CH, fsz - pos)) + st = 0 + while True: + i = d.find(pat, st) + if i < 0: break + st = i + 1 + out.append(off2gpa(pos + i) & ~0xfff) + pos += CH + return out + +def collect(page): + d = rd(page, 0x1000) + if not d: return None + descs = [] + wins = [] + for o in range(0, 0x1000 - 24, 8): + sz, fl = struct.unpack_from("<QQ", d, o) + pa = struct.unpack_from("<Q", d, o + 16)[0] + if fl == 0x03008000 and 0x1000 <= sz <= 0x400000 and \ + 0x100000000 <= pa < 0x180000000: + descs.append((o, sz, pa)) + va2 = sz + if 0x4000000 <= va2 < 0x100000000 and (va2 & 0xfff) == 0: + sz2 = fl + if 0x1000 <= sz2 <= 0x1000000 and (sz2 & 0xfff) == 0: + wins.append((va2, sz2)) + return descs, wins + +def main(): + log("=== resolver7 start (watch=%ds) ===" % WATCH_S) + injected = set() + deadline = time.time() + WATCH_S + while time.time() < deadline: + pages = set() + for (pva, rva) in PAIRS: + try: + pages |= set(find_qpair(pva, 0x20000)) & set(find_qpair(rva, 0x40000)) + except Exception as e: + log("scan error:", e) + for page in sorted(pages): + got = collect(page) + if not got: continue + descs, wins = got + # push windows: inject live-first (nz>0 first, zeros last) + for (pva, _rva) in PAIRS: + cands = [(o, sz, pa) for (o, sz, pa) in descs if sz == 0x20000] + order = [] + for (o, sz, pa) in cands: + pd = rd(pa, 0x40) + nz = sum(1 for b in pd if b) if pd else -1 + log("pair=0x%x page=0x%x desc@0x%x pa=0x%x live_nz=%d" % (pva, page, o, pa, nz)) + order.append((nz, o, pa)) + order.sort(key=lambda t: t[0]) # zeros first, live last + for (nz, o, pa) in order: + key = (pva, page, o) + if key in injected: continue + if inject(pva, pa): + injected.add(key) + # release windows: any {VA, 0x1000} record on the page -> size-0x1000 descs + for (va, sz) in wins: + if va not in REL_WINS and not (0x4b00000 <= va <= 0x4b20000): + continue + for (o, dsz, pa) in descs: + if dsz != sz: + continue + key = (va, page, o) + if key in injected: continue + if inject(va, pa): + injected.add(key) + time.sleep(1) + log("=== resolver7 done: %d injection(s) ===" % len(injected)) + return 0 + +if __name__ == "__main__": + sys.exit(main()) |