diff options
| -rw-r--r-- | docs/04-implementation/04-real-render-walls.md | 34 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 26 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 18 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 1650 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 2 | ||||
| -rw-r--r-- | kernel/main.c | 55 | ||||
| -rw-r--r-- | tools/guest-diag/fenceword_assist.py | 91 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/README-session77.md | 132 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/a8clear.py | 52 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/fasttick.py | 44 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/fence_assist.py | 112 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/find_push.py | 59 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/kc_dump_fn.py | 21 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/pdbtick.py | 69 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/runbook.py | 58 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/t112c-gdbstub.py | 42 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/t120.py | 148 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/t121.py | 151 | ||||
| -rw-r--r-- | tools/guest-diag/wall2/t122.py | 151 | ||||
| -rw-r--r-- | tools/vk/vk_frame.c | 29 |
20 files changed, 2879 insertions, 65 deletions
diff --git a/docs/04-implementation/04-real-render-walls.md b/docs/04-implementation/04-real-render-walls.md index f6ea39d..4298772 100644 --- a/docs/04-implementation/04-real-render-walls.md +++ b/docs/04-implementation/04-real-render-walls.md @@ -185,3 +185,37 @@ gdb leaves the VM stopped (recover with a detach-only connect); `stepi` loops over the stub freeze the guest's userspace for minutes - prefer breakpoints; every probe run with the caps-fill breakpoint changes the driver's path, so keep the fill in each experiment. + +## 8. UPDATE 2026-10-25 #3 (SESSION 126): the fence fold landed - the service authors the Vulkan fence releases from the driver's own push bytes; the compute-dispatch shape decoded (direct C5C0 method stream, no inline QMD) + +**The fence wall is closed.** The client's per-frame fence wait polls a +16 KB BO word that a submit-push INC-5 semaphore group targets +(`[2005c017][0x45ba090][0][payload 1][0][0x02100001]`). The group sits in a +GPFIFO entry beyond the driver's kick boundary (GP_PUT=11 while the ring +already carries the entries it pre-built for the next submits, idx12..46), +so the GP_PUT-bounded pump walk never decoded it and a host-side assist had +to write the word. The service now sweeps that pre-built continuation +itself: `sema_gr_ahead` (default 64) walks entries beyond GP_PUT through the +ring's validated translation, accepts only pushes that pass the strict +method-stream gate, extracts the addr-0x5c RELEASE groups and applies them +forward-only through the standard pending ledger. Live evidence: the sweep +finds the groups and the ledger applies the fence word once per frame; +`VKFRAME seq=1..3/DONE` completes with no assist. The frames are still +black - the shader work itself is unexecuted. + +**The compute-dispatch shape (why nothing executes).** The workload is a +single compute dispatch (640x480 storage image). Its push is a DIRECT +method-stream programming of the Turing compute class: `SET_OBJECT 0xC5C0` +on subchannel 1, then the context windows (`0x7b4` local-memory window B, +`0x2a4` shared-memory window B, `0x270`), an immediate-config block, and +the code/const data runs (`0x3800`/`0x39C8` pairs plus a subchannel-0 data +port stream of `0x11c`/`0x114` runs in a separate 17 KB push). There is NO +inline-QMD write (no `0x318/0x31c/0x320..0x41c` run), so the launch capture +that feeds the physical-engine forward path never fires - and the CE-shaped +ops path is not even consulted for GR-class channels by construction. Real +pixels therefore require a translator that recognizes the C5C0 section, +builds the vendor launch the physical channel expects (the same prelude/QMD +shape the CUDA path already executes) from the section's fields and data +runs, and forwards it with the existing operand/relocation machinery. The +method vocabulary overlaps the physical prelude's existing knowledge +(`0x7b0/0x7b4`, `0x2a4`), so the landmarks for that translator are known. diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 6fae2f2..65fa429 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17678,8 +17678,29 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) * launches resolve 968-969 records (the client's whole arena); a launch * with only a handful after the fill's drops has an unresolved argument * set and must not be handed to the engine (see the submit guard). + * + * SESSION 127: the Vulkan inline-QMD launch (the C5C0 translator) is a + * compute dispatch whose operand set is one descriptor page and one + * parameter bank - a handful of records is healthy for it. Tunable module + * parameter (default 16 preserves the CUDA-era threshold); the translator + * cycles pass gsp_phys_gr_min_data=2. */ #define GSP_PHYS_GR_MIN_DATA_RECS 16u +static unsigned int gsp_phys_gr_min_data_param = GSP_PHYS_GR_MIN_DATA_RECS; +module_param_named(gsp_phys_gr_min_data, gsp_phys_gr_min_data_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_min_data, + "GR submit minimum data records (degenerate-set refusal threshold, clamped 1..4096; default 16)"); + +static unsigned int gsp_phys_gr_min_data_value(void) +{ + unsigned int v = READ_ONCE(gsp_phys_gr_min_data_param); + + if (!v) + v = 1; + if (v > 4096) + v = 4096; + return v; +} #define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */ /* * relfix PART 3 (r217 span containment): the late path authors ONLY the @@ -19289,10 +19310,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * into the same software-completion fallback without touching the * engine, and cannot wedge it for the next client. */ - if (req->n_data < GSP_PHYS_GR_MIN_DATA_RECS) { + if (req->n_data < gsp_phys_gr_min_data_value()) { gsp_phys_gr_refused++; pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", - req->n_data, GSP_PHYS_GR_MIN_DATA_RECS, + req->n_data, + gsp_phys_gr_min_data_value(), req->push_va); return -ENODATA; } diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 1855908..03f861e 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2665,6 +2665,20 @@ static void gsp_rm_handle_kgr_get_context_buffers_info( * L2/LTC topology from here. Echo the requested indices and fill in the * emulated TU104 values. */ +/* + * m113: FB_GET_INFO_V2 COMPRESSION_SIZE (index 0x01, bytes). The header: + * "A return value of 0 indicates the GPU does not support compression" - + * but the guest driver still attempts a COMPR-enabled, 64 KB-aligned + * comptag allocation (vendor VidHeap alloc, attr COMPR=2) and fails it + * NV_ERR_NO_MEMORY (0x51) when the comptag pool has no space, which + * libcuda maps to VK_ERROR_OUT_OF_DEVICE_MEMORY (the vkAllocateMemory -2 + * gate). Report a comptag pool sized like the real Turing parts (~fb/16). + * 0 = the historic "no compression" reply. + */ +static uint gsp_rm_fb_compr_mb = 128; +module_param_named(wall2_fb_compr_mb, gsp_rm_fb_compr_mb, uint, 0644); +MODULE_PARM_DESC(wall2_fb_compr_mb, "FB_GET_INFO_V2 COMPRESSION_SIZE in MiB (default 128, 0 = none)"); + static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index) { u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu); @@ -2673,7 +2687,8 @@ static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index) case TU104_FB_INFO_IDX_TILE_REGION_COUNT: return 0; case TU104_FB_INFO_IDX_COMPRESSION_SIZE: - return 0; + /* m113: non-zero comptag pool (bytes) - see gsp_rm_fb_compr_mb. */ + return gsp_rm_fb_compr_mb << 20; case TU104_FB_INFO_IDX_PARTITION_COUNT: return 1; case TU104_FB_INFO_IDX_BAR1_SIZE: @@ -3536,6 +3551,7 @@ static uint gsp_rm_9096_ent_type = 3; /* entry +0x28 (u32) */ module_param_named(wall2_9096_ent_type, gsp_rm_9096_ent_type, uint, 0644); MODULE_PARM_DESC(wall2_9096_ent_type, "0x90960107 entry word at +0x28 (default 3)"); static uint gsp_rm_9096_ent_flag = 1; /* entry +0x30 (u8) */ + module_param_named(wall2_9096_ent_flag, gsp_rm_9096_ent_flag, uint, 0644); MODULE_PARM_DESC(wall2_9096_ent_flag, "0x90960107 entry flag byte at +0x30 (default 1)"); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b311c0d..f77b75d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1136,6 +1136,17 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u32 fwd_gr_nocand_log; /* SESSION 127: first-N + * request-without-candidate + * witnesses */ + u64 fwd_gr_nordy; /* SESSION 127: backend-not- + * ready request refusals */ + u64 fwd_gr_holdbusy;/* SESSION 127: hold-active + * request busy verdicts */ + u64 fwd_gr_inflight;/* SESSION 127: inflight-bit + * request busy verdicts */ + u32 fwd_gr_enoe_log;/* SESSION 127: first-N fill + * ENOENT page witnesses */ u64 fwd_gr_defer; /* m150: held for data-arg retry */ u64 fwd_gr_dstpend; /* r541: held for an unexecuted * device write on a data page */ @@ -1686,6 +1697,8 @@ static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ +static u32 svc_gr_push_dump; /* SESSION 127: bounded capture tail dump + * (sema_dbg only, 4 per boot) */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ @@ -1838,6 +1851,52 @@ void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, * the lesser evil; it is logged loudly for the record. */ #define SVC_VAS_CAND_AGE_MS 15000u +/* + * r705h: the candidate age-out deadline is runtime-tunable. The parked + * COPY_SERVER_PDES candidate for the GL/Vulkan client ages out (UNVERIFIED) + * at exactly the moment the client's channels first register, so the + * content-verified walk can never anchor it; raising the deadline keeps the + * candidate parked until a host tool supplies a verified ring pair via + * sema_pdb_discover. 0 disables the age-out entirely. + */ +static u32 svc_vas_cand_age_ms = SVC_VAS_CAND_AGE_MS; +module_param_named(sema_vas_cand_ms, svc_vas_cand_age_ms, uint, 0644); + +/* + * SESSION 127 C5C0 translator gate. When 1, the sweep additionally + * translates continuation entries into launch work (tail-aware method + * parse, inline-QMD capture, LOAD_INLINE_DATA materialization, C5B5 + * copy-out capture, candidate-owned release withholding). Default 0 keeps + * the historical sweep behaviour for the CUDA batteries; the Vulkan + * translator cycles pass sema_gr_xlate=1. Flip the default once the CUDA + * battery is proven clean with it on. + */ +static bool gsp_sema_gr_xlate; +module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644); +MODULE_PARM_DESC(sema_gr_xlate, + "C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)"); +/* + * SESSION 128 utility-channel inline-download materialization + * (sema_gr_util_scan, default 0 = off): the driver's non-GR upload + * channels (the MEM2MEM/A140 + 2D utilities, registered but with an + * unresolved USERD so the pump's put-bounded entry loop never decodes + * them) carry the engine-side code/data downloads: the + * GRCE/INLINE_TO_MEMORY operand group on the mem2mem subchannel (0x180.. + * 0x18c), LAUNCH_DMA (0x1b0) and a LOAD_INLINE_DATA (0x1b4) run whose + * payload the engine fetches from the next GPFIFO entry (the + * source-view call). Without this pass those destinations stay zero in + * the vidmem shadow - observed: the Vulkan client's code-heap/SASS + * upload (12 KB) and command-stream staging (256 B) never land, so the + * forwarded launch has no program content. Bounded: scan up to N + * entries per channel per pass, materialize once-only keyed by + * (push_va, content hash, dst, bytes) via the same + * svc_grce_inline_write the GR sweep uses. Requires sema_gr_xlate. + */ +static u32 gsp_sema_gr_util_scan; +module_param_named(sema_gr_util_scan, gsp_sema_gr_util_scan, uint, 0644); +MODULE_PARM_DESC(sema_gr_util_scan, + "Utility-channel i2m sweep: entries scanned per pass (default 0 = off; requires sema_gr_xlate)"); +MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)"); static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) { @@ -1848,8 +1907,9 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) if (!svc_vas_cand[i].valid) continue; - if (time_before(jiffies, svc_vas_cand[i].stamp + - msecs_to_jiffies(SVC_VAS_CAND_AGE_MS))) + if (!svc_vas_cand_age_ms || + time_before(jiffies, svc_vas_cand[i].stamp + + msecs_to_jiffies(svc_vas_cand_age_ms))) continue; memset(&d, 0, sizeof(d)); d.h_vaspace = svc_vas_cand[i].h_vaspace; @@ -1857,7 +1917,7 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) d.num_entries = svc_vas_cand[i].entries; d.b_vidmem = svc_vas_cand[i].vidmem; pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n", - d.h_vaspace, SVC_VAS_CAND_AGE_MS, d.pdb, + d.h_vaspace, svc_vas_cand_age_ms, d.pdb, d.num_entries); gsp_sema_service_vaspace_add(&d); svc_vas_cand[i].valid = false; @@ -3008,6 +3068,15 @@ struct svc_ipin_alt { static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS]; static u64 svc_pushalt_hits; /* push reads served by a displaced pair */ static u32 svc_pushalt_log; +/* r707x: per-client push-read attempt witness (pump read site). */ +static u32 svc_rdw_log; +static u8 svc_read_last_src; +/* r715: PA the winning source of the last push read resolved to. */ +static u64 svc_read_last_pa; +/* r715: per-client push-decode branch witness (pump read site). */ +static u32 svc_pushdec_log; +/* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */ +static u32 svc_pushfail_log; static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) { @@ -6354,10 +6423,11 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.param_size, svc_gr_cand.rel_va, svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, svc_gr_cand.n_pointer, svc_gr_cand.n_inline); - if (push && push_len >= 16) { + if (gsp_sema_dbg && svc_gr_push_dump < 4 && push && push_len >= 16) { u32 n = push_len / 4; u32 i, j; + svc_gr_push_dump++; if (n > 256) n = 256; for (i = 0; i < n; i += 16) { @@ -8872,18 +8942,20 @@ static bool svc_stream_parse(const u8 *buf, u32 len) memcpy(&hdr, buf + off, 4); if (hdr == 0) { /* - * UVM's large page-table pushes begin with a - * zero-filled header region (observed 0x1004 - * bytes) before the first method, so zeros before - * the first method are leading padding; zeros - * after one are trailing padding and end the - * stream. + * NVC06F_DMA_NOP is 0x00000000: a zero word is + * NOP padding. UVM's large page-table pushes begin + * with a zero-filled header region (observed 0x1004 + * bytes) and the GR submit pushes carry zero gaps + * between method groups (observed r715: the 1432- + * and 17316-byte pushes pad between the context + * setup and the dispatch groups), so zeros are + * skipped anywhere in the stream, not only before + * the first method or at the tail. The exact-end + * requirement and the valid-header walk below are + * the false-positive arbiters. */ - if (!methods) { - off += 4; - continue; - } - break; /* trailing zero padding */ + off += 4; + continue; } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; @@ -8903,16 +8975,27 @@ static bool svc_stream_parse(const u8 *buf, u32 len) return false; off += 4 + count * 4; break; + case 0: /* GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK */ + case 2: /* GRP2_USE_TERT */ case 4: /* IMMD_DATA_METHOD: header only */ - off += 4; - break; case 7: /* END_PB_SEGMENT: header only */ + /* + * r715: the class header defines op 0 as + * GRP0_USE_TERT (0) and op 2 as GRP2_USE_TERT - + * 4-byte header-only formats. The GR submit + * pushes emit 0x0001fff0 (SET_SUBDEVICE_MASK, + * value 0xfff) around their semaphore groups; the + * old parser rejected the first one on every big + * push, so the whole push read implausible and + * the entries were skipped without decode + * (session 124's consume-without-decode). + */ off += 4; break; default: return false; } - if (++methods > 1024) + if (++methods > 8192) return false; } while (off < len && buf[off] == 0) @@ -10038,8 +10121,19 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (st->qmd_n > svc_gr_qmd_seen) svc_gr_qmd_seen = st->qmd_n; - if (st->qmd_have == 3) - st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; + /* + * SESSION 127 C5C0 translator: the Vulkan driver writes + * SET_INLINE_QMD_ADDRESS_B only (the upper half stays absent for a + * <4G QMD; observed beec0100 idx23: `0x31c = 0x4b85f` then a 64-word + * 0x320 run). Accept the B-only form with an implicit zero upper + * half - the old A+B requirement left qmd_dev_va at 0 and the + * candidate logged with qmd=0x0 (the field is identity/logging + * only, but the 0x0 read confused every prior session's forensics). + */ + if (st->qmd_have & 2) + st->qmd_dev_va = + (((u64)((st->qmd_have & 1) ? st->qmd_hi : 0) << 32) | + st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { /* * Static: a 16-entry record array is ~1.2 KB and this is @@ -10061,15 +10155,85 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; if (have > GSP_BRIDGE_INLINE_MAX) have = GSP_BRIDGE_INLINE_MAX; - for (i = 0; i < have; i++) { - u32 idx = (svc_grce_dst_ring_next - 1 - i) % - SVC_GRCE_DST_MAX; - const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx]; - - recs[n].dst_va = rec->dst_va; - recs[n].len = rec->len; - memcpy(recs[n].data, rec->data, rec->len); - n++; + /* + * SESSION 127 C5C0 translator: the rec ring is session-wide + * and mixes every client's LOAD_INLINE_DATA destinations. A + * stale rec whose page this QMD does not name made the fill + * refuse the whole launch (GR fill ENOENT + * va=0x80412000 qmd_word=NONE covered=1 - the Vulkan + * client's cancel page inherited from the driver-init + * phase). Keep only recs whose page is one of the pages the + * captured QMD names (the QMD buffer, the parameter bank, + * the program, the pointer pairs) - exactly the pages the + * backend's inline-served fill can use; keep everything when + * the QMD names none (fail-open). + */ + { + u64 keep[12]; + u32 nkeep = 0, k; + + if (st->qmd_dev_va) + keep[nkeep++] = st->qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[32] | + ((u64)(st->qmd[33] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[48] | + ((u64)(st->qmd[49] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + for (k = 0; k < SVC_GR_POINTER_MAX - 1 && nkeep < 12; k++) { + u32 w = 34 + 2 * k; + u64 pv = (u64)st->qmd[w] | + ((u64)(st->qmd[w + 1] & 0x1ffff) + << 32); + + if (pv) + keep[nkeep++] = pv & + ~(u64)(SEMA_PAGE_SIZE - 1); + } + for (i = 0; i < have; i++) { + u32 idx = (svc_grce_dst_ring_next - 1 - i) % + SVC_GRCE_DST_MAX; + const struct svc_grce_rec *rec = + &svc_grce_dst_ring[idx]; + u64 rpage = rec->dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + bool keepit = !nkeep; + u32 j; + + for (j = 0; j < nkeep; j++) + if (keep[j] == rpage) { + keepit = true; + break; + } + /* + * SESSION 127 (gated): a stale session-wide + * rec whose page resolves nowhere made the + * fill refuse the whole launch; keep it when + * it resolves (the CUDA inline-served + * pages), drop only unresolvable unnamed + * ones. With the translator off, the + * historical keep-all behaviour stands. + */ + if (!keepit && gsp_sema_gr_xlate) { + u64 rpa = 0; + u32 rap = SVC_APERTURE_SYS; + + if (svc_resolve_va(vgpu, rpage, &rpa, + &rap) && + svc_walk_vaspaces(vgpu, rpage, + &rpa, &rap)) + continue; + } else if (!keepit) { + continue; + } + recs[n].dst_va = rec->dst_va; + recs[n].len = rec->len; + memcpy(recs[n].data, rec->data, rec->len); + n++; + } } /* * C10 (GAP-2 A5): duplicate destinations in the newest-first @@ -12679,6 +12843,10 @@ static void svc_gr_defer_reset(void); static void svc_session_flush(void) { + /* r707x: per-client witness budgets. */ + svc_pushfail_log = 0; + svc_rdw_log = 0; + svc_pushdec_log = 0; unsigned long flags; u32 i; @@ -13859,7 +14027,14 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) if (off + 4 > len) return true; /* all zeros: not refutable here */ op = (hdr >> 29) & 0x7; - if (op == 4 || op == 7) /* header-only methods */ + /* + * r715: op 0 (GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK) and op 2 + * (GRP2_USE_TERT) are 4-byte header-only formats per the class + * header; the GR submit pushes start with 0x0001fff0 + * (SET_SUBDEVICE_MASK), so a correct injected pin for such a page + * was rejected here and the read fell through to the walk. + */ + if (op == 0 || op == 2 || op == 4 || op == 7) /* header-only */ return true; if (op != 1 && op != 3 && op != 5) return false; @@ -13908,6 +14083,21 @@ static u32 svc_pushpg_split_log; * validated candidate). Reads stay defensively copied through * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds. */ +/* r707d: bounded push-read failure witness (see below). */ +static u32 svc_pushfail_log; +/* + * r707e: last content-backed ring pair resolved by any channel. The VAS + * candidate verify is retried with it on EVERY pump pass: the parked + * COPY_SERVER_PDES candidate for the client's user VAS parks at + * device-create and must register before the submit's first push read + * (a failed GR entry read is never retried with a fresh VAS), but the ring + * resolution and the park race within one pass - the every-pass retry + * closes that race in a single boot. + */ +static u64 svc_lastgood_ring_va; +static u64 svc_lastgood_ring_pa; + + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -13923,6 +14113,7 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, bool have_win, done = false; bool first = (p == buf); int last_rc = -ENOENT; + int trusted_rc = 0; u32 src; have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); @@ -13976,7 +14167,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, default: /* learned cache / discovered map / page * directories / targeted rm_mem resolve */ - if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + trusted_rc = svc_lookup_trusted(vgpu, va, + &pa, &ap); + if (trusted_rc) continue; break; } @@ -14037,10 +14230,34 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, */ if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; + svc_read_last_src = (u8)src; + svc_read_last_pa = pa; done = true; } - if (!done) + if (!done) { + svc_read_last_src = 0xff; + if (svc_pushfail_log < 96) { + u64 ipa = 0, apa = 0; + int iap = 0, aap = 0; + bool has_ipin = + (svc_ipin_lookup(va, &ipa, &iap) == 0); + bool has_alt = + (svc_ipin_alt_lookup(va, &apa, &aap) == 0); + + svc_pushfail_log++; + pr_info("sema-pushfail: va=0x%llx chunk=0x%llx page=0x%llx last_rc=%d trusted_rc=%d ipin=%d pa=0x%llx alt=%d pa=0x%llx win=%d pa=0x%llx\n", + (unsigned long long)va, + (unsigned long long)chunk, + (unsigned long long)page_va, + last_rc, trusted_rc, has_ipin, + (unsigned long long)ipa, + has_alt, + (unsigned long long)apa, + have_win, + (unsigned long long)win_pa); + } return last_rc; + } p += chunk; va += chunk; len -= chunk; @@ -15621,6 +15838,58 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, } /* + * r705h: host-triggered pdb discovery. The automatic latch + * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM + * ring scan, which sweeps [0,6GB) before the client's pushes exist and then + * gives up for the boot; and per-client VAS state is flushed at session end. + * This entry point lets a host tool that CONTENT-LOCATED a verified + * {ring VA -> ring PA} pair (the ring page's GPFIFO entries are its + * fingerprint) hand the pair to the service while the client is alive, so: + * 1. every parked COPY_SERVER_PDES candidate is verified against the pair + * (fast: a walk each), and + * 2. if none verifies, the vidmem walk-root sweep anchors on the pair. + * Registered VASes make the client's pushbuffer VAs walkable and the pump + * can then decode the submit's pushes (the fence release among them). + */ +static u64 svc_pdb_disc_va; +static u64 svc_pdb_disc_pa; +static bool svc_pdb_disc_pending; + +/* + * r707r: bounded channel rewind (debug). After the VAS pdb anchor lands, + * entries skipped before it never re-decode on their own (their unread + * window expires into a skip), so the release they carry would be lost for + * that put epoch. A host tool asks for a rewind (sema_chan_rewind: a hex + * h_object, or "all"); the pump re-processes the channel from slot 0 with + * the consumed-slot bitmap and the per-entry rescue state cleared. + */ +static u32 svc_rewind_h; +static bool svc_rewind_all; +static bool svc_rewind_pending; + +int gsp_sema_service_chan_rewind(u32 h_object) +{ + if (!svc.task) + return -ENODEV; + svc_rewind_h = h_object; + svc_rewind_all = (h_object == 0xffffffffu); + svc_rewind_pending = true; + wake_up_process(svc.task); + return 0; +} + +int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa) +{ + if (!svc.task) + return -ENODEV; + svc_pdb_disc_va = ring_va & ~0xfffULL; + svc_pdb_disc_pa = ring_pa & ~0xfffULL; + svc_pdb_disc_pending = true; + wake_up_process(svc.task); + return 0; +} + +/* * r496f: discover a kernel-RM client VAS's page directory in the emulated * VRAM by verification. The CPU RM builds internal-VAS page tables in * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally @@ -18558,6 +18827,156 @@ static void svc_fwd_hold_direct_note(void) } /* + * SESSION 127 C5C0 translator: pending GRCE copy-out list. + * + * The Vulkan compute client submits its frame work in the pre-built + * continuation beyond GP_PUT: beec0100 idx23 carries the inline-QMD launch + * (0x31c = 0x4b85f, 64-word 0x320 run) and idx24 carries the TURING_DMA_COPY_A + * (0xC5B5, subch 4) image->buffer copies that un-tile the storage image into + * the client's sysmem readback buffer. Both pushes are only visible to the + * sema_gr_ahead sweep (see svc_gr_ahead_push), not to the put-bounded main + * walk. + * + * Copy ordering: the copy must read the image AFTER the forwarded launch + * completed (the engine's output is copied back into the VA shadow on the + * completion edge). The sweep only records the copy; it is executed inside + * svc_fwd_hold_gr_release, i.e. after the backend published done and after + * the copy-out, immediately before the launch's guest release word is + * authored - so the client's fence never observes a pre-render image. + * A bounded fallback run (sweep, when no forward is owed) keeps the list + * from stalling when the launch was refused. + */ +#define SVC_GR_COPY_PEND 8 +struct svc_gr_copy_pend { + bool valid; + u32 h_object; + u64 push_va; + u64 hash; + u64 src; + u64 dst; + u32 bytes; /* per-row byte count */ + u32 rows; + u32 dpitch; + u32 spitch; + u32 dma; /* LAUNCH_DMA word (remap/transfer bits, logging) */ + u32 remap; /* SET_REMAP_COMPONENTS (logging) */ +}; +static struct svc_gr_copy_pend svc_gr_copy_pends[SVC_GR_COPY_PEND]; +static u32 svc_gr_copy_pend_next; +static u64 svc_gr_copy_run_total; +static u64 svc_gr_copy_run_log; + +static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, + u64 src, u64 dst, u32 bytes, u32 rows, + u32 dpitch, u32 spitch, u32 dma, u32 remap) +{ + u32 i; + + if (!bytes || !rows || !src || !dst) + return; + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + + if (p->valid && p->push_va == push_va && p->src == src && + p->dst == dst && p->bytes == bytes && p->rows == rows) + return; /* already recorded (idempotent re-sweep) */ + } + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + if (p->valid) + continue; /* keep live ones; take a free slot */ + p->valid = true; + p->h_object = h_object; + p->push_va = push_va; + p->hash = hash; + p->src = src; + p->dst = dst; + p->bytes = bytes; + p->rows = rows; + p->dpitch = dpitch ? dpitch : bytes; + p->spitch = spitch ? spitch : bytes; + p->dma = dma; + p->remap = remap; + pr_info("sema-gr: copy pend h=0x%x push=0x%llx src=0x%llx dst=0x%llx row=0x%x rows=%u dpitch=0x%x spitch=0x%x dma=0x%x remap=0x%x\n", + h_object, push_va, src, dst, bytes, rows, p->dpitch, + p->spitch, dma, remap); + return; + } + /* All slots live: overwrite the oldest position. */ + { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + pr_warn_ratelimited("sema-gr: copy pend overflow (dropping push=0x%llx)\n", + p->push_va); + memset(p, 0, sizeof(*p)); + } +} + +/* + * Execute and retire every pending GRCE copy. Called from the completion + * edge (svc_fwd_hold_gr_release) and, as a bounded fallback, from the sweep + * when no forward is owed. The executor is the same svc_copy() the CE/GCE + * op path uses (handles vidmem shadow sources and sysmem destinations). + */ +static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) +{ + u32 i; + + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + u64 done = 0; + int rc; + + if (!p->valid) + continue; + rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0, + SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done, + p->rows, p->dpitch, p->spitch, p->h_object, 0, + p->hash); + svc_gr_copy_run_total++; + if (rc == 0 || rc == SVC_COPY_FORWARDED) { + p->valid = false; + if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, p->rows, rc); + } + } else if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy retry h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rc=%d done=0x%llx\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, rc, done); + } + } +} + +/* + * Is this release VA owned by a live, not-yet-authored launch candidate + * captured from THIS push? (Only the launch's own push's release is + * withheld; a continuation fence entry elsewhere in the ring keeps the + * historical immediate-forward behaviour.) + */ +static bool svc_gr_ahead_owned_rel_va(u64 va, u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].rel_done && + svc_gr_cand_ring[ri].rel_va == va && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + +/* * Author a GR launch completion release (the QMD-level semaphore the * physical engine writes when the launch completes) in software, * forward-only. Used when a hold resolves without a successful hardware @@ -18605,6 +19024,14 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, { if (!svc_fwd_req.gr) return; + /* + * SESSION 127 C5C0 translator: the launch is complete and its output + * pages were copied back into the VA shadow on this edge - run any + * pending GRCE copy-outs (image -> linear readback buffer) BEFORE + * authoring the client's release word, so the fence can never clear + * on a pre-render image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, ch ? ch->h_object : 0, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); @@ -18742,6 +19169,13 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_push_va, waited); return; } + /* + * SESSION 127 C5C0 translator: this edge authors the guest release + * without a hold - run any pending GRCE copy-outs first, exactly like + * the hold's success edge, so the fence cannot clear on a pre-render + * image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); @@ -20086,8 +20520,34 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) goto note_done; } } - if (svc_resolve_va(vgpu, page, &pa, &ap)) - return; + if (svc_resolve_va(vgpu, page, &pa, &ap)) { + /* + * SESSION 127 C5C0 translator (gated): a client-VAS page. The + * Vulkan + * QMD's program, parameter and descriptor pages live in the + * client's own page tables (vidmem aperture) and the trusted + * sources - pins, learned cache, discovered map - do not + * carry them, which made the fill refuse the launch with + * -ENOENT. Resolve through the registered-VAS walk instead: + * the exact path walk_probe proves (r707w). + */ + u64 wpa = 0; + u32 wap = SVC_APERTURE_SYS; + + if (!gsp_sema_gr_xlate || + svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + return; + pa = wpa; + ap = svc_ap_of(wap); + svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "vas"; + goto note_done; + } svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); /* @@ -21841,7 +22301,12 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, u64 qpa = 0; u32 qap = SVC_APERTURE_SYS; - if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) { + if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap) || + (gsp_sema_gr_xlate && + !svc_walk_vaspaces(vgpu, + svc_gr_cand.qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + &qpa, &qap))) { svc_fwd_req.gr_qmd_pa = qpa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap); @@ -21914,10 +22379,19 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, if (pg->pa) continue; - if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) - return -ENOENT; - if (!svc_gr_page_inline_covers(pg->va)) + if (pg->qmd_word == SVC_GR_QMD_WORD_NONE || + !svc_gr_page_inline_covers(pg->va)) { + if (svc.fwd_gr_enoe_log < 8) { + svc.fwd_gr_enoe_log++; + pr_info("vgsp-work: GR fill ENOENT va=0x%llx qmd_word=%u inline=%u covered=%d n_pages=%u push=0x%llx\n", + pg->va, pg->qmd_word, + svc_gr_cand.n_inline, + svc_gr_page_inline_covers(pg->va), + svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + } return -ENOENT; + } } /* * GRFIX FIX1c: the fill hosts the self-enforcing decision point and @@ -22504,11 +22978,25 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || - svc_gr_cand.h_object != ch->h_object) + svc_gr_cand.h_object != ch->h_object) { + if (svc.fwd_gr_nocand_log < 8) { + svc.fwd_gr_nocand_log++; + pr_info("vgsp-work: GR request without candidate h=0x%x (forward=%d cand_valid=%d cand_h=0x%x offered=%d)\n", + ch->h_object, gsp_phys_gr_forward_enabled(), + svc_gr_cand.valid, svc_gr_cand.h_object, + svc_gr_cand.offered); + } return false; + } svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { + svc.fwd_gr_nordy++; + if ((svc.fwd_gr_nordy & 0x3F) == 1) + pr_info("vgsp-work: GR request refused: backend not ready h=0x%x ready=%d gmem=%d n=%llu\n", + ch->h_object, gsp_phys_backend_gr_ready(), + gsp_phys_guest_mem_enabled(), + svc.fwd_gr_nordy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); @@ -22519,6 +23007,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * request's done/hw_rc are still being read. The capture falls back. */ if (svc_fwd_hold.active) { + svc.fwd_gr_holdbusy++; + if ((svc.fwd_gr_holdbusy & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: hold active h=0x%x (hold ch=0x%x n=%llu)\n", + ch->h_object, svc_fwd_hold.ch_handle, + svc.fwd_gr_holdbusy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; /* @@ -22539,6 +23032,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_inflight++; + if ((svc.fwd_gr_inflight & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: inflight h=0x%x (state=0x%lx n=%llu)\n", + ch->h_object, svc_fwd_state, + svc.fwd_gr_inflight); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; if (!gsp_cuda_bridge_enabled()) { @@ -22565,7 +23063,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; - if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + if (svc.fwd_gr_refused < 8 || + (svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", rc, ch->h_object, svc_gr_cand.push_va); svc_gr_bridge_fallback(vgpu); @@ -23532,8 +24031,786 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, put, get, ch->submissions, waited, reason); } +/* + * --------------------------------------------------------------------- + * SESSION 126 - FENCE FOLD: channel-anchored sweep of a GR-class + * channel's PRE-BUILT ring continuation. + * + * The Vulkan client's fence waits never clear unaided because the + * driver's kick boundary (GP_PUT) stops at the first batch while the ring + * already carries the continuation entries it pre-built for the next + * submits - and the fence's own INC-5 RELEASE group (observed + * [2005c017][0x45ba090][0][payload 1][0][0x02100001]) lives in one of + * those entries, so the pump's GP_PUT-bounded walk never decodes it + * (r715/r716: put=11, entries 12..47 carried content, no dmesg row ever + * mentioned 0x45ba). The host-side fenceword_assist wrote the wait word + * directly; this fold makes the service author it from the driver's own + * push bytes. + * + * Doctrine: unlike the disabled page-content sweep (svc_apply_page_data, + * off since the false-positive corruption incidents), this sweep is + * CHANNEL-ANCHORED: it only walks entries reachable through the ring's + * validated GPFIFO translation, only accepts entries whose bytes pass the + * strict method-stream gate, and only applies the semaphore release + * groups it finds - forward-only, through the standard pending ledger + * (svc_pending_add). No consumption bookkeeping is touched (last_get, + * put_prev, slot_done_epoch and the replay state stay as the pump left + * them), so a later real kick of the same entries decodes them normally; + * repeated sweeps are idempotent (the pending ledger dedups (va,payload) + * and the apply path is forward-only). + * + * Gated by the same live-epoch arm as the counter publish + * (ch->gr_pub_armed) so a residual ring from a previous session cannot + * author releases before the session switch is proven. + * --------------------------------------------------------------------- + */ +unsigned gsp_sema_gr_ahead_max = 64; +module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_ahead, + "GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)"); + + +static u32 svc_gr_ahead_log; +static u64 svc_gr_ahead_entries; +static u64 svc_gr_ahead_rels; +static u64 svc_gr_ahead_skips; +/* SESSION 127: releases withheld because a live candidate owns the word. */ +static u64 svc_gr_ahead_owned; +/* SESSION 127: bounded witness for the sweep-caller's launch request. */ +static u32 svc_gr_ahead_req_log; +/* SESSION 128: inline (LOAD_INLINE_DATA) writes authored by the sweep. */ +static u64 svc_gr_ahead_inline_writes; + +/* + * SESSION 127 C5C0 translator, bounded tail materialization buffer: a + * LOAD_INLINE_DATA run whose payload overruns the entry end is served from + * the next ring entry (the driver splits large inline runs into a methods + * entry plus a dedicated data slot entry). 4 KB covers the observed + * param-bank write (1568 B) in full and the first page of the big uploads. + */ +#define SVC_AHEAD_TAIL_INLINE_MAX 32768 +static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX]; +static u64 svc_ahead_inline_trunc; + +/* + * SESSION 128: one-copy materialization bound for the utility/CE sweep. + * The engine-side copy-in the sweep replays (a TURING_DMA_COPY_A pitch + * copy from a client-owned staging buffer into the vidmem shadow) moves + * the driver's code/constant uploads; the observed SASS/code-heap class is + * tens of KB, so 4 MB is a generous cap that still refuses a bogus + * geometry before it can walk the shadow. + */ +#define SVC_AHEAD_COPY_IN_MAX (4ULL << 20) + +/* + * SESSION 128: materialize one TURING_DMA_COPY_A pitch copy-in that an + * unprocessed copy-class channel carried - src bytes live in the client's + * staging buffer (guest RAM, resolved through the trusted walk like any + * operand page), dst is the vidmem page the driver staged the upload for + * (code heap / constant store). This is the emulated engine write for the + * copy the real CE would perform; it is once-only keyed by (push_va, + * content hash, dst, bytes) so re-scans of a stable ring are free. + */ +static u64 svc_ahead_copy_in_skip; +static void svc_ahead_copy_in(struct nv_vgpu_device *vgpu, u32 ch_h, + u64 push_va, u64 hash, u64 src_va, u64 dst_va, + u32 row, u32 rows, u32 dma) +{ + static u8 cb[65536]; + u64 bytes = (u64)row * (rows ? rows : 1); + u64 done = 0; + u64 spa = 0, dpa = 0; + int sap = SVC_APERTURE_SYS, dap = SVC_APERTURE_SYS; + int rc = 0; + bool src_ok, dst_ok; + + if (!bytes || src_va < SVC_LOW_REL_VA_LO || src_va >= SEMA_VA_HI || + dst_va < SVC_LOW_REL_VA_LO || dst_va >= SEMA_VA_HI) + return; + /* + * Only the upload direction is materialized: the source must be a + * resolved client page in sysmem/guest RAM (the staging buffer the + * client CPU-wrote), and the destination must be vidmem (resolved to + * the FB aperture) or a not-yet-resolved VA whose bytes the VA + * shadow will hold. A resolved SYS destination is a copy-OUT (the + * engine reading vidmem into a client buffer): replaying it here + * would overwrite the client's buffer with shadow garbage before the + * producing launch ran, so it is skipped - the GR channels' copy-out + * machinery owns that direction. + */ + src_ok = !svc_lookup(vgpu, src_va, &spa, &sap) && sap != SVC_APERTURE_FB; + dst_ok = svc_lookup(vgpu, dst_va, &dpa, &dap) || + dap == SVC_APERTURE_FB; + if (!src_ok || !dst_ok) { + svc_ahead_copy_in_skip++; + if (svc_ahead_copy_in_skip <= 8) + pr_info("sema-utilsweep: copy-in skipped (src_ok=%d dst_ok=%d) push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx\n", + src_ok, dst_ok, push_va, src_va, dst_va, + bytes); + return; + } + if (bytes > SVC_AHEAD_COPY_IN_MAX) { + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-utilsweep: copy-in too large push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx cap=0x%llx\n", + push_va, src_va, dst_va, bytes, + SVC_AHEAD_COPY_IN_MAX); + return; + } + if (svc_inline_once_seen(push_va, hash, dst_va, (u32)bytes)) + return; + svc_inline_once_store(push_va, hash, dst_va, (u32)bytes); + while (done < bytes) { + u32 chunk = bytes - done > sizeof(cb) ? + (u32)sizeof(cb) : (u32)(bytes - done); + + rc = svc_read_va(vgpu, src_va + done, cb, chunk); + if (rc) { + rc = -EIO; + break; + } + rc = svc_grce_inline_write(vgpu, dst_va + done, cb, chunk, + ch_h); + if (rc) + break; + done += chunk; + } + svc_gr_ahead_inline_writes++; + pr_info("sema-utilsweep: copy-in src=0x%llx -> dst=0x%llx bytes=0x%llx dma=0x%x done=0x%llx rc=%d h=0x%x\n", + src_va, dst_va, bytes, dma, done, rc, ch_h); +} + +/* One 4-byte word of a method stream that may continue into the tail. */ +static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + u32 off) +{ + u32 v = 0; + + if (off + 4 <= len) + memcpy(&v, buf + off, 4); + else if (tail && off >= len && (u64)(off - len) + 4 <= taillen) + memcpy(&v, tail + (off - len), 4); + return v; +} + +/* + * Strict method-stream parse with a one-entry tail (see the sweep comment). + * A run whose payload overruns this entry's end may be completed from the + * next ring entry; anything else invalid means the page is not a method + * stream (the same false-positive arbiter as svc_stream_full_ok). Returns + * the method count, or 0; *used_tail reports that the tail served a + * trailing data run (nothing after it is parsed). + */ +static u32 svc_ahead_parse(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + bool *used_tail) +{ + u32 off = 0, methods = 0; + + *used_tail = false; + for (;;) { + u32 hdr, op, count; + u64 end; + + if (off + 4 > len) + break; + memcpy(&hdr, buf + off, 4); + if (hdr == 0) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + switch (op) { + case 1: /* INC_METHOD */ + case 3: /* NON_INC_METHOD */ + case 5: /* ONE_INC */ + if (!count || count > 0xfff) + return 0; + end = (u64)off + 4 + (u64)count * 4; + if (end > len) { + if (!tail || (u64)(end - len) > taillen) + return 0; + *used_tail = true; + off = len; + goto done; + } + off = (u32)end; + break; + case 0: /* GRP0_USE_TERT */ + case 2: /* GRP2_USE_TERT */ + case 4: /* IMMD_DATA_METHOD */ + case 7: /* END_PB_SEGMENT */ + off += 4; + break; + default: + return 0; + } + if (++methods > 8192) + return 0; + } +done: + while (off < len && buf[off] == 0) + off++; + return (methods > 0 && (off == len || *used_tail)) ? methods : 0; +} + +/* + * SESSION 127 C5C0 translator sweep pass over one coherent push. Besides the + * original release extraction (svc_gr_ahead_push, below), it recognizes, for + * pre-built continuation entries: + * + * - the C5C0 inline-QMD window (SET_INLINE_QMD_ADDRESS_A/B + the 64-word + * LOAD_INLINE_QMD_DATA run) and feeds it to the standard svc_qmd_accum + * capture, so the physical GR forward (the CUDA QMD path) fires for a + * launch that never reaches the GP_PUT-bounded main walk - observed + * beec0100 idx23: `0x31c = 0x4b85f` + `0x320 x 64`; + * - GRCE / KEPLER_INLINE_TO_MEMORY_B LOAD_INLINE_DATA runs (op 3 at + * 0x1b4, offsets from the 0x180/0x184/0x188/0x18c group), materialized + * into the VA shadow exactly like the main decode does - observed + * beef0100 idx00 (256 B -> 0x423f000, payload in idx01) and idx08 + * (12288 B -> 0x4448000, payload in idx09), and beec0100 idx23 + * (1568 B -> 0x4b86020, payload in-push); + * - TURING_DMA_COPY_A (0xC5B5, subch >= 2) copy operands + * (0x400/0x404 in, 0x408/0x40c out, 0x410/0x414 pitch, 0x418/0x41c + * line length/count, 0x300 LAUNCH_DMA, 0x700..0x708 remap) recorded as + * pending copy-outs for the completion edge (svc_gr_copy_pend_run) - + * observed beec0100 idx24: image 0xdfd8d60000 -> readback 0x050b0000, + * 640x480 rows. + * + * Returns the number of release groups queued (as before). The buffer must + * pass the strict tail-aware parse first; a page that is not a method + * stream is not processed at all. + */ +static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + const u8 *buf, u32 len, const u8 *tail, + u32 taillen, u64 push_va, bool inline_only) +{ + static struct svc_push_state stq; /* service thread only */ + static u32 dw[66]; /* QMD window words */ + u32 off = 0, nrel = 0, methods, i; + u64 off_in = 0, off_out = 0; + u32 have_in = 0, have_out = 0; + u32 pitch_in = 0, pitch_out = 0; + u32 line_len = 0, line_count = 0; + u32 last_dma = 0, last_remap = 0; + u32 cp_subch = ~0u; + bool used_tail = false; + bool forward = gsp_phys_gr_forward_enabled(); + bool xlate = gsp_sema_gr_xlate; + + if (!xlate) { + /* Historical sweep: releases from exactly-covered entries. */ + tail = NULL; + taillen = 0; + } + methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail); + if (!methods) + return 0; + + memset(&stq, 0, sizeof(stq)); + stq.push_va = push_va; + + while (off + 4 <= len) { + u32 hdr, op, addr, count, subch; + u32 w0, w1, w2, w3, w4; + u64 end; + + memcpy(&hdr, buf + off, 4); + if (!hdr) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + addr = (hdr & 0x1fff) << 2; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + if (op == 0 || op == 2 || op == 4 || op == 7) { + off += 4; + continue; + } + if (op != 1 && op != 3 && op != 5) + break; + if (!count || count > 0xfff) + break; + end = (u64)off + 4 + (u64)count * 4; + /* + * Tail-served trailing run: process from the head word, then + * stop (the tail is a data slot, nothing parses after it). + */ + w0 = svc_ahead_w32(buf, len, tail, taillen, off + 4); + w1 = svc_ahead_w32(buf, len, tail, taillen, off + 8); + w2 = svc_ahead_w32(buf, len, tail, taillen, off + 12); + w3 = svc_ahead_w32(buf, len, tail, taillen, off + 16); + w4 = svc_ahead_w32(buf, len, tail, taillen, off + 20); + + /* + * C5C0 inline-QMD window (mirrors the main decode's intercept): + * only accumulate once the run declares itself a QMD window. + */ + if (xlate && forward && !inline_only && subch >= 1 && + addr >= SVC_QMD_ADDR_A && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + (stq.qmd_have || + (addr <= SVC_QMD_ADDR_B && + (u64)addr + (u64)count * 4 > SVC_QMD_ADDR_A))) { + u32 n = count > 66 ? 66 : count; + + for (i = 0; i < n; i++) + dw[i] = svc_ahead_w32(buf, len, tail, taillen, + off + 4 + 4 * i); + svc_qmd_accum(vgpu, ch, &stq, addr, n, + (const u8 *)dw, buf, len, true, false); + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* + * GRCE / INLINE_TO_MEMORY_B LOAD_INLINE_DATA: materialize the + * DMA write (the engine would do this; the emulation authors + * the bytes into the VA shadow so the forwarded launch's + * operand pages - parameter banks - are real). + */ + if (xlate && op == 3 && subch >= 1 && + addr == GRCE_M_LOAD_INLINE_DATA && + have_out && line_len) { + u64 nbytes = (u64)line_len * + (line_count ? line_count : 1); + u32 cap = count * 4; + u32 bytes, k, words; + u64 h = svc_push_hash(buf, len); + + if (nbytes > cap) + nbytes = cap; + bytes = (u32)nbytes; + if (bytes > SVC_AHEAD_TAIL_INLINE_MAX) { + bytes = SVC_AHEAD_TAIL_INLINE_MAX; + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-ahead: inline write truncated push=0x%llx dst=0x%llx want=%llu cap=%u\n", + push_va, off_out, nbytes, + SVC_AHEAD_TAIL_INLINE_MAX); + } + words = (bytes + 3) / 4; + for (k = 0; k < words; k++) + memcpy(svc_ahead_inline_buf + 4 * k, + &(u32){ svc_ahead_w32(buf, len, tail, + taillen, + off + 4 + 4 * k) }, + 4); + if (bytes && !svc_inline_once_seen(push_va, h, off_out, + bytes)) { + svc_inline_once_store(push_va, h, off_out, + bytes); + svc_gr_ahead_inline_writes++; + (void)svc_grce_inline_write(vgpu, off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + /* + * Record the destination and the fresh payload bytes + * for the launch fill's inline-served pages, exactly + * like the main decode's LOAD_INLINE_DATA arm: the + * QMD capture later in this entry collects the ring + * and the fill can serve the parameter bank from it + * even while its shadow page is still empty. + */ + { + struct svc_grce_rec *rec = + &svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX]; + + rec->dst_va = off_out; + rec->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, svc_ahead_inline_buf, + rec->len); + svc_grce_dst_ring_next++; + } + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* TURING_DMA_COPY_A operand runs (subch >= 2). */ + if (xlate && subch >= 2) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + if (addr >= 0x400 && addr <= 0x41c && n) { + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x400: + off_in = (off_in & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_in = 1; + break; + case 0x404: + off_in = (off_in & + ~0xffffffffULL) | + vv[k]; + have_in = 1; + break; + case 0x408: + off_out = (off_out & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x40c: + off_out = (off_out & + ~0xffffffffULL) | + vv[k]; + have_out = 1; + break; + case 0x410: + pitch_in = vv[k]; + break; + case 0x414: + pitch_out = vv[k]; + break; + case 0x418: + line_len = vv[k]; + break; + case 0x41c: + line_count = vv[k]; + break; + default: + break; + } + } + } else if (addr >= 0x700 && addr <= 0x708 && n) { + for (k = 0; k < n; k++) { + if (addr + 4 * k == 0x700) + last_remap = (last_remap & + ~0xffffu) | + (vv[k] & 0xffffu); + else if (addr + 4 * k == 0x708) + last_remap = (last_remap & + 0xffffu) | + ((vv[k] & 0xffffu) + << 16); + } + } else if (addr == 0x300 && count) { + u32 dma = w0; + + /* + * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a + * release-only / semaphore op - no data + * movement, nothing to record). + */ + last_dma = dma; + if ((dma & 0x3) && have_in && have_out && + line_len && line_count && + off_in && off_out) { + if (inline_only) + svc_ahead_copy_in(vgpu, + ch->h_object, push_va, + svc_push_hash(buf, + len), + off_in, off_out, + line_len * 4, + line_count, dma); + else if (forward) { + svc_gr_copy_pend_add(ch->h_object, + push_va, svc_push_hash(buf, + len), + off_in, off_out, + line_len * 4, + line_count, + pitch_out ? pitch_out : + line_len * 4, + pitch_in, dma, + last_remap); + cp_subch = subch; + } + } + } + } else if (xlate && (subch == 1 || subch == 2) && + addr >= 0x180 && addr <= 0x18c && count) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + /* + * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c). + * The mem2mem subchannel (2, the A140/KEPLER_INLINE_ + * TO_MEMORY_B upload lane the utility channels drive) + * carries the same operand methods as the compute + * subchannel (1). + */ + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x180: + line_len = vv[k]; + break; + case 0x184: + line_count = vv[k]; + break; + case 0x188: + off_out = (off_out & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x18c: + off_out = (off_out & + ~0xffffffffULL) | vv[k]; + have_out = 1; + break; + default: + break; + } + } + } + + if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) { + u64 va; + + va = ((u64)(w1 & 0xff) << 32) | + (u64)(w0 & 0xfffffffc); + op = w4 & 0x7; + if ((op == HOST_SEM_OP_RELEASE || + op == HOST_SEM_OP_REDUCTION) && w2 && + va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) { + /* + * A release targeting a live launch candidate's + * own QMD release word is owned by the + * completion edge (post-copy-out); sweeping it + * here would clear the client's fence before + * the engine output landed. + */ + if (xlate && forward && + svc_gr_ahead_owned_rel_va(va, push_va)) { + svc_gr_ahead_owned++; + } else { + svc_pending_add(va, w2, + op == HOST_SEM_OP_REDUCTION ? + 1 : 0); + nrel++; + } + } + } + if (end > len) + break; + off += 4 + count * 4; + } + (void)cp_subch; + return nrel; +} + +static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put) +{ + static u8 abuf[2048]; + static u8 nbuf[2048]; + u32 i, maxent, nrel = 0, swept = 0, nskip = 0; + + if (!put || !ch->gpfifo_entries) + return; + maxent = min_t(u32, gsp_sema_gr_ahead_max, 512); + for (i = 0; i < maxent; i++) { + u32 idx = (put + i) % ch->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen, tail_len = 0; + u64 va; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + break; /* unwritten slot: end of the continuation */ + if (!va || !len || len > SVC_PUSH_MAX) + break; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + break; + rlen = min_t(u32, len, sizeof(abuf)); + if (svc_read_va_push(vgpu, ch, va, abuf, rlen)) { + /* + * Unreadable here (pre-anchor translation) is a + * reason to stop this pass: the continuation is not + * proven coherent, so nothing is authored. + */ + nskip++; + break; + } + /* + * SESSION 127 C5C0 translator: read the next ring entry as + * the tail of a stream whose final inline run overruns this + * entry (the driver splits large LOAD_INLINE_DATA payloads + * into a dedicated data slot entry; observed beef0100 + * idx00->idx01 and idx08->idx09). + */ + { + u32 nidx = (put + i + 1) % ch->gpfifo_entries; + u8 nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 ne0, ne1, nlen; + u64 nva; + + if (!svc_read_va(vgpu, + ch->gpfifo_va + + (u64)nidx * SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent))) { + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + if (nva && nlen < SVC_PUSH_MAX) { + tail_len = min_t(u32, nlen, + sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, + nbuf, tail_len)) + tail_len = 0; + } + } + } + swept++; + nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf, + tail_len, va, false); + } + svc_gr_ahead_entries += swept; + svc_gr_ahead_rels += nrel; + svc_gr_ahead_skips += nskip; + if (swept && (nrel || svc_gr_ahead_log < 64)) { + svc_gr_ahead_log++; + pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u owned=%llu skipped=%u (total rels=%llu)\n", + ch->h_object, put, swept, nrel, svc_gr_ahead_owned, + nskip, svc_gr_ahead_rels); + } +} + +/* + * SESSION 128: utility-channel inline-download materialization. + * + * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D + * composite: observed h=0xbeef0100 with gpfifo=0x4020000) are registered + * with an unresolved USERD (userd_ok=0), so the put-bounded entry loop in + * svc_pump_channels never decodes their pushes and the LOAD_INLINE_DATA + * downloads they carry (the client's code-heap/SASS upload, command-stream + * staging) never land anywhere the emulation can see: the destinations + * stay zero in the vidmem shadow. + * + * This pass scans a bounded window of the channel's GPFIFO directly (the + * gpfifo VA resolves through the trusted walk even when the USERD does + * not) and feeds each entry + its successor (the source-view call carrying + * the payload) through the proven sweep decoder in inline-only mode: the + * GRCE/INLINE_TO_MEMORY operand group (0x180..0x18c on subch 1/2), + * LAUNCH_DMA and the LOAD_INLINE_DATA run are materialized by + * svc_grce_inline_write; releases, QMD windows and copy-pend records are + * skipped so nothing about the client's fence/completion state changes. + * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring + * costs only the reads. Requires sema_gr_xlate; bounded by + * sema_gr_util_scan entries per channel per pass (0 = off). + */ +static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ +/* + * The observed utility-channel pushes are the ~17 KB setup block and the + * two split uploads (256 B command stream + 12 KB code/heap payload); a + * 64 KB window covers every one of them with room to spare while keeping + * the two static buffers modest. + */ +#define SVC_UTIL_BUF (64 * 1024) + static u8 ubuf[SVC_UTIL_BUF]; + static u8 nbuf[SVC_UTIL_BUF]; + static u64 svc_gr_util_entries; + static u64 svc_gr_util_writes; + static u32 svc_gr_util_log; + u32 i, maxent = min_t(u32, gsp_sema_gr_util_scan, 64); + u32 swept = 0, nmade = 0; + + if (!gsp_sema_gr_xlate || !gsp_sema_gr_util_scan || + !ch->gpfifo_va || !ch->gpfifo_entries) + return; + for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) { + u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen, tail_len = 0; + u64 va, before; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + continue; /* unwritten slot */ + if (!va || !len || len > SVC_PUSH_MAX) + continue; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + continue; + rlen = min_t(u32, len, sizeof(ubuf)); + if (svc_read_va_push(vgpu, ch, va, ubuf, rlen)) + continue; + /* + * The payload of a trailing LOAD_INLINE_DATA run rides the + * next GPFIFO entry (the driver's source-view GPFIFO call, + * observed beef0100 idx00->idx01 and idx08->idx09): read it + * as the tail exactly like svc_gr_ahead_sweep does. + */ + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)((i + 1) % ch->gpfifo_entries) * + SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent)) == 0) { + u32 ne0, ne1, nlen; + u64 nva; + + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + if (nva && nlen && nlen < SVC_PUSH_MAX) { + tail_len = min_t(u32, nlen, sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, nbuf, + tail_len)) + tail_len = 0; + } + } + before = svc_gr_ahead_inline_writes; + (void)svc_gr_ahead_push(vgpu, ch, ubuf, rlen, nbuf, + tail_len, va, true); + if (svc_gr_ahead_inline_writes != before) + nmade++; + swept++; } + svc_gr_util_entries += swept; + svc_gr_util_writes += nmade; + if (swept && (nmade || svc_gr_util_log < 24)) { + svc_gr_util_log++; + pr_info("sema-utilsweep: h=0x%x swept=%u writes=%u (total %llu/%llu)\n", + ch->h_object, swept, nmade, svc_gr_util_entries, + svc_gr_util_writes); + } +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { + if (svc_pdb_disc_pending) { + svc_pdb_disc_pending = false; + pr_info("sema-pdb-disc: requested va=0x%llx pa=0x%llx\n", + svc_pdb_disc_va, svc_pdb_disc_pa); + svc_vas_candidates_verify(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + svc_xvas_pdb_discover(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + } /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; /* @@ -23547,6 +24824,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + if (svc_rewind_pending) { + svc_rewind_pending = false; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu) + continue; + if (!svc_rewind_all && c->h_object != svc_rewind_h) + continue; + pr_info("sema-chan: rewind h=0x%x last_get=%u put=%u (slot bitmap + rescue state cleared)\n", + c->h_object, + (unsigned int)c->last_get, + (unsigned int)c->put_prev); + bitmap_zero(c->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + c->last_get = 0; + c->rescue_fail = 0; + c->unread_va = 0; + c->unread_until = 0; + c->defer_since = 0; + } + } + /* + * r707e: every-pass candidate verify with the last-good ring pair. + * Cheap (<=16 parked candidates, one walk each) and idempotent; the + * first pass after the user-VAS candidate parks registers it, so the + * client's very first push reads resolve. + */ + if (svc_lastgood_ring_pa) + svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, + svc_lastgood_ring_pa); /* * r507o stream-order: walk the CE channels BEFORE the GR channels. * The client's stream order places its H2D copies before the kernel @@ -23764,6 +25071,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!rrc) svc_pb_pair(vgpu, ch); + if (!rrc && ring_pa) { + svc_lastgood_ring_va = ch->gpfifo_va; + svc_lastgood_ring_pa = ring_pa; + } + /* + * r707f (FOLD): the moment ANY channel's ring resolves through + * the channel object, verify the parked COPY_SERVER_PDES VAS + * candidates against that content-backed pair. The candidates + * are parked at device-create time, before the client's compute + * channels first submit, so registering here lands the VAS page + * directory BEFORE the submit's entries get their first push + * read - the window the host-triggered path could not win (the + * entry's first read and the ring's first resolution happen in + * the same pass; a pair that arrives later cannot help, because + * a failed GR entry read is never retried with the fresh VAS). + */ + if (!rrc) + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + ring_pa); /* * ISSUE-0002 (r47 diag): run the deferred CE USERD sweep for a @@ -23784,6 +25110,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies + 27 * HZ; } + /* + * SESSION 128: utility/upload channels carry the engine's + * inline code/data downloads (LOAD_INLINE_DATA on the + * mem2mem subchannel) and the copy-class channels (eng=11/12) + * carry the driver's pitch copy-ins (staging -> code heap / + * constant store). With an unresolved USERD the put-bounded + * entry loop below never decodes them (the !userd_ok branch + * continues before it); with a resolved USERD the main + * decode's LOAD_INLINE_DATA/operand arms still only accept + * subchannel 1, so the subchannel-2 downloads are missed + * either way. Scan every channel's GPFIFO directly here and + * materialize the downloads (see svc_gr_util_sweep); the + * inline-only mode plus the once-only key make the overlap + * with the main loop and the GR sweep a no-op. + */ + if (gsp_sema_gr_util_scan && gsp_sema_gr_xlate) + svc_gr_util_sweep(vgpu, ch); + if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get, @@ -24001,6 +25345,50 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->put_prev = put; /* + * SESSION 126 fence fold: sweep the ring continuation beyond + * GP_PUT for semaphore releases (see svc_gr_ahead_sweep). + * GR-class channels only, live-armed epochs only, and only + * with the ring translation proven through the channel + * object (rrc == 0). + */ + if (!ch->engine_type && ch->gr_pub_armed && + gsp_sema_gr_ahead_max && !rrc && ring_pa) { + svc_gr_ahead_sweep(vgpu, ch, put); + /* + * SESSION 127 C5C0 translator: offer a launch the + * sweep captured from the continuation (the Vulkan + * client's inline-QMD push lives beyond GP_PUT, so + * the main walk's per-entry request never sees it). + * svc_gr_request is a no-op without a matching + * unoffered candidate and keeps it on a busy + * verdict, so a bounded per-pass retry here is the + * sweep-captured launch's entire dispatch path. + */ + if (gsp_sema_gr_xlate && + gsp_phys_gr_forward_enabled()) { + if (svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + svc_gr_ahead_req_log < 8) { + svc_gr_ahead_req_log++; + pr_info("sema-ahead: request h=0x%x push=0x%llx offered=%d\n", + ch->h_object, + svc_gr_cand.push_va, + svc_gr_cand.offered); + } + (void)svc_gr_request(vgpu, ch); + } + /* + * Fallback copy-out retirement: when no forward is + * owed (the launch was already consumed or the + * backend is off), run any pending GRCE copies so + * they cannot sit forever (the completion edge is + * the primary path). + */ + if (gsp_sema_gr_xlate && !svc_gr_forward_pending()) + svc_gr_copy_pend_run(vgpu); + } + + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. * Runs before the entry loop so a wedged session epoch is * recovered (and logged) instead of spinning: see @@ -24253,9 +25641,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (pb) { int rescue_rc; - bool unreadable = - svc_read_va_push(vgpu, ch, va, - pb, len) != 0; + int rd_rc; + bool unreadable; + bool zero_nop = false; + bool displaced = false; + bool have_ipin = false; + + svc_read_last_src = 0xff; + svc_read_last_pa = 0; + rd_rc = svc_read_va_push(vgpu, ch, va, + pb, len); + unreadable = rd_rc != 0; + if (gsp_sema_dbg && svc_rdw_log < 6000 && + !ch->engine_type) { + svc_rdw_log++; + pr_info("sema-rdw: h=0x%x va=0x%llx len=%u rc=%d src=%u\n", + ch->h_object, va, len, + rd_rc, + (unsigned)svc_read_last_src); + } bool injected = false; bool allow_ops = (ch->engine_type != 0); @@ -24280,8 +25684,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 ipa; int iap; - injected = (svc_ipin_lookup(va, &ipa, - &iap) == 0); + have_ipin = + (svc_ipin_lookup(va, &ipa, + &iap) == 0); + /* + * r715: the entry's treatment + * follows the source that + * ACTUALLY answered the read, + * not a fresh page-granularity + * ipin lookup. The read falls + * through to the trusted walk + * (src 4) whenever an ipin's + * page fails its first-chunk + * guard, and taking the + * injected branch then skipped + * the only source that could + * answer: the entry was + * consumed without ever + * consulting the registered + * VAS walk (session 124). + */ + injected = have_ipin && + (svc_read_last_src == 0); /* * r496e: an all-zero push on a * content-scanned-ring channel is the @@ -24290,7 +25714,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * translation: accept it and let the * decode find no ops. */ - bool zero_nop = ch->gr_ram_ring && + zero_nop = ch->gr_ram_ring && len >= 4 && !memchr_inv(pb, 0, len); @@ -24323,7 +25747,82 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(fb); } } else { - unreadable = true; + /* + * r707v: a primary-source + * hit (injected pin, window + * guess, lat cache) whose + * content is implausible + * must not shadow the + * registered VAS walk - a + * stale pin otherwise makes + * every push read "resolve" + * to the wrong page, fail + * plausibility and turn the + * entry unreadable forever. + * Retry once with the pins + * bypassed so the trusted + * walk (svc_walk_vaspaces) + * answers; accept only a + * plausible stream. + */ + u8 *vb = kmalloc(len, + GFP_KERNEL); + + if (vb) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool ok = false; + + /* + * r707w: consult ONLY the + * registered VAS walk (the + * exact path walk_probe + * proves) - not + * svc_lookup_trusted, whose + * pin/lat hits would answer + * with the same wrong page + * again and keep the entry + * unreadable. + */ + int wr = svc_walk_vaspaces(vgpu, + va & ~0xfffULL, + &wpa, &wap); + int rr = wr ? wr : svc_read_ap(vgpu, + wpa + (va & 0xfff), + wap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + vb, len); + bool pl = !rr ? svc_method_stream_plausible(vb, len) + : false; + + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk rc=%d pa=0x%llx plaus=%d\n", + ch->h_object, va, + wr ? wr : rr, + wpa + (va & 0xfff), + pl ? 1 : 0); + } + if (!wr && !rr && pl) { + memcpy(pb, vb, len); + svc.pushpg_fallback++; + ok = true; + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk ok pa=0x%llx\n", + ch->h_object, va, + wpa + (va & 0xfff)); + } + } + kfree(vb); + if (!ok) + unreadable = true; + } else { + unreadable = true; + } } /* * Still implausible (or @@ -24339,18 +25838,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u8 *db = kmalloc(len, GFP_KERNEL); - if (db && - svc_push_read_displaced(vgpu, - ch->h_object, va, - db, len)) { - memcpy(pb, db, len); - injected = false; - unreadable = false; - } - kfree(db); - } - } - } + if (db && + svc_push_read_displaced(vgpu, + ch->h_object, va, + db, len)) { + memcpy(pb, db, len); + injected = false; + unreadable = false; + displaced = true; + } + kfree(db); + } + } + } /* * The entry's page was unreadable through @@ -24369,11 +25869,40 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, db, len); injected = false; unreadable = false; + displaced = true; } kfree(db); } /* + * r715 pushdec witness: name the branch + * the entry takes straight after the + * read, before the rescue/hold paths. + * Session 124 had an entry consumed + * between the read (rc=0 src=4) and the + * decode with no row naming why; this + * line makes the branch and the content + * (first 8 bytes) visible on every read. + * Budget: 512 rows per client (reset in + * svc_session_flush). + */ + if (gsp_sema_dbg && svc_pushdec_log < 2048) { + svc_pushdec_log++; + pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + svc_method_stream_plausible(pb, len) ? 1 : 0, + injected ? 1 : 0, + have_ipin ? 1 : 0, + zero_nop ? 1 : 0, + displaced ? 1 : 0, + unreadable ? 1 : 0, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + + /* * Content rescue before any fake-release * fallback (CE channels): the push bytes * are in guest RAM even when the VA->PA @@ -26509,12 +28038,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } - pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", + pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx armed=%d notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->last_rel_va, ch->counter_pa, + ch->gr_pub_armed, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index fbc7e4d..8d63a72 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -110,6 +110,8 @@ int gsp_sema_service_dump_channels(void); int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap); int gsp_sema_service_dump_vidmem(u64 addr, u64 len); int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits); +int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa); +int gsp_sema_service_chan_rewind(u32 h_object); int gsp_sema_service_counter_add(u32 h_object, u64 pa); void gsp_sema_service_doorbell(u32 token); diff --git a/kernel/main.c b/kernel/main.c index dd11d12..7af236d 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2960,6 +2960,61 @@ module_param_cb(sema_map_add, &nv_vgpu_sema_map_add_ops, NULL, 0200); MODULE_PARM_DESC(sema_map_add, "Inject a GPU VA -> guest PA mapping (hex va,hex pa)"); /* + * Debug: retrigger the client-VAS page-directory discovery with a host + * content-verified {ring VA -> ring PA} pair (the ring page's GPFIFO + * entries are the fingerprint): + * echo 0x<va>,0x<pa> > .../parameters/sema_pdb_discover + */ +static int nv_vgpu_sema_pdb_discover_set(const char *val, + const struct kernel_param *kp) +{ + char buf[64]; + char *comma; + u64 va = 0, pa = 0; + + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (!comma) + return -EINVAL; + *comma = '\0'; + if (kstrtoull(buf, 0, &va) || kstrtoull(comma + 1, 0, &pa)) + return -EINVAL; + + return gsp_sema_service_pdb_discover(va, pa); +} + +static const struct kernel_param_ops nv_vgpu_sema_pdb_discover_ops = { + .set = nv_vgpu_sema_pdb_discover_set, +}; + +module_param_cb(sema_pdb_discover, &nv_vgpu_sema_pdb_discover_ops, NULL, 0200); +MODULE_PARM_DESC(sema_pdb_discover, "Retrigger the VAS pdb discovery with a verified ring pair (hex va,hex pa)"); + +/* + * Debug: bounded channel rewind. Clears the consumed-slot bitmap and the + * per-entry rescue state for one channel (hex h_object) or every channel + * ("all"), so the pump re-processes its ring from slot 0 - used after the + * VAS pdb anchor lands so entries skipped before it re-decode (the release + * they carry would otherwise be lost for that put epoch). + */ +static int nv_vgpu_sema_chan_rewind_set(const char *val, + const struct kernel_param *kp) +{ + u32 h = 0xffffffffu; + + if (strncmp(val, "all", 3) && kstrtouint(val, 0, &h)) + return -EINVAL; + return gsp_sema_service_chan_rewind(h); +} + +static const struct kernel_param_ops nv_vgpu_sema_chan_rewind_ops = { + .set = nv_vgpu_sema_chan_rewind_set, +}; + +module_param_cb(sema_chan_rewind, &nv_vgpu_sema_chan_rewind_ops, NULL, 0200); +MODULE_PARM_DESC(sema_chan_rewind, "Rewind a channel's ring (hex h_object or 'all')"); + +/* * Debug: register a channel's GPU completion counter (guest PA of the u32 * tracking counter the GPU would write when a push completes): * echo <h_object>,<pa> > .../parameters/sema_counter diff --git a/tools/guest-diag/fenceword_assist.py b/tools/guest-diag/fenceword_assist.py new file mode 100644 index 0000000..9e207e2 --- /dev/null +++ b/tools/guest-diag/fenceword_assist.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# fenceword_assist.py - keep the Vulkan fence wait word of a live vk_frame120 +# client released (session 125). +# +# SUPERSEDED (SESSION 126, commit 95e7f57e): the service now authors these +# releases itself. The `sema_gr_ahead` fence fold sweeps the channel's +# pre-built ring continuation beyond GP_PUT through the ring's validated +# GPFIFO translation, decodes the INC-5 RELEASE groups it finds (addr 0x5c +# count>=5) and queues them in the standard pending ledger; r717 evidence: +# `sema-ahead: h=0xbeec0100 put=11 swept=10 rels=4` followed 30 ms later by +# `sema-svc: deferred release va=0x45ba090 payload=1 (was cur=0)`, with +# VKFRAME seq/DONE and NO assist running (see +# backups/evidence-20261002-victory/r717-fencefold/). Keep this tool only as +# the A/B fallback for boots that predate the fold (or with +# sema_gr_ahead=0). +# +# The client's fence wait polls a word in a 16 KB /dev/nvidiactl BO mapped +# read-write into the client: the init wait word lives at +0x80 (the t122 +# "BW" chaser writes it once) and the per-submit fence word at +0x90 of the +# SAME BO (GPU VA <base>+0x90, delivered by the submit push's INC-5 semaphore +# group). While the emulation does not apply that group, this host assist +# writes 1 to the +0x90 slot whenever it reads 0 - the direct predecessor of +# the service-side fold (see Handoff SESSION 125/126). +# +# Usage (host, while a t122-driven client runs): +# tools/guest-diag/fenceword_assist.py [seconds] [guest-pid] +import os +import subprocess +import sys +import time + +VMEXEC = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "..", "scripts", "vgpu_vm.py")) +PY = r''' +import struct, sys, time +pids = [p for p in sys.argv[1:] if p] +maps_of = {} +def vma(pid, addr): + for line in open("/proc/%d/maps" % pid): + rng, perms = line.split(None, 2)[:2] + a, b = [int(x, 16) for x in rng.split("-")] + if a <= addr < b: + return line.strip() + return None +found = {} +for pid in pids: + try: + maps = [l for l in open("/proc/%d/maps" % pid) if "nvidia" in l and "rw-s" in l] + except OSError: + continue + for l in maps: + rng, perms, off = l.split(None, 3)[:3] + a, b = [int(x, 16) for x in rng.split("-")] + if b - a != 0x4000: + continue + try: + mem = open("/proc/%d/mem" % pid, "r+b", 0) + mem.seek(a + 0x80); init = mem.read(4) + mem.seek(a + 0x90); fw = mem.read(4) + except Exception: + continue + if init == b"\x01\x00\x00\x00" and fw == b"\x00\x00\x00\x00": + found[pid] = (a + 0x90, mem) + print("found pid=%s word=%#x" % (pid, a + 0x90), flush=True) +end = time.time() + float(sys.argv[1]) +while time.time() < end: + for pid, (w, mem) in list(found.items()): + try: + mem.seek(w) + if mem.read(4) == b"\x00\x00\x00\x00": + mem.seek(w); mem.write(struct.pack("<I", 1)); mem.flush() + print("released pid=%s word=%#x" % (pid, w), flush=True) + except Exception: + del found[pid] + time.sleep(0.5) +print("assist done", flush=True) +''' + + +def main(): + dur = sys.argv[1] if len(sys.argv) > 1 else "300" + pid = sys.argv[2] if len(sys.argv) > 2 else "" + b64 = __import__("base64").b64encode(PY.encode()).decode() + subprocess.run([VMEXEC, "exec", "--timeout", str(int(dur) + 60), + "echo %s | base64 -d > /tmp/fwa.py; sudo python3 /tmp/fwa.py %s %s" + % (b64, dur, pid)]) + + +if __name__ == "__main__": + main() diff --git a/tools/guest-diag/wall2/README-session77.md b/tools/guest-diag/wall2/README-session77.md index 1ae9a62..d830bef 100644 --- a/tools/guest-diag/wall2/README-session77.md +++ b/tools/guest-diag/wall2/README-session77.md @@ -1061,3 +1061,135 @@ service-side durable route: have the SERVICE write the marker into the channel-buffer page once it can resolve it (the memdesc VA = the CE channel's buffer; the service's channel registration fields + the PT walk give the page candidates). + +**39. Sessions 112b-114: THE SCRUB GATE FELL, THE COMPR WALL WAS DECODED AND FLIPPED, AND THE FIRST VULKAN SUBMISSION LANDED (VKFENCES IS THE NEW WALL).** +Context first (sessions 112b/112c, commits `97489187`/`cb7d8ce7`/`752e2b30`/ +`d6ccdaf3`/`8700f65a`): the CeUtils scrubber's finish payload is authored +in-service (bound to the EXACT 4096-entry channel), the client then leaves +`scrubCheckAndWaitForSize` permanently and runs +vkCreateImage -> vkGetImageMemoryRequirements -> vkAllocateMemory; the +remaining `-2` was decomposed to the guest-internal VidHeap `#149`-shape +call (attr COMPR_ANY + 2MB page + owner 0x474C0000) returning 0x51 +NV_ERR_NO_MEMORY; session 113 read that as a 64KB comptag/CBM alloc. This +session REFINED it: the COMPR+2MB class is not always 64KB - the r705 +runs show the *image data* alloc itself (size 0x140000 = 1.25 MB for +640x480x4 + pad, attr 0x1002000, flags 0xc10d) in exactly the same shape. +The "separate 64KB CBM" reading was too narrow; every alloc of the class +{attr&0xF000 == 0x2000, 2MB page, owner 0x474C0000} fails 0x51. + +**A. THE ZERO-TOUCH CREATE RECIPE (NEW, SCRIPTED).** Two live assists are +required per client, both proven again this session: (1) the +instance-phase marker clears at eglcore+0xB33D30 (t106/t99 `BC`, 5 +clears); (2) THE DEVICE-PHASE WAIT-WORD CHASER at eglcore+0xDCEBD9 +(t99 `BW`: word = *(holder+0x58)+*(u32)(obj2+0x60) with +obj2 = *(*(rbx+0x70)+0x58); write 1 when 0). Without (2) the client +reaches `VKDEVICE`'s poll(`/dev/nvidia0`, 10 ms) retry loop and sits +there INDEFINITELY (session-76 state; the a097 engine-monitor worker +keeps polling 0x2080a097 every 1.25 s regardless - its reply words are +still a red herring for the main-thread gate). With (1)+(2) the run is +`VKINSTANCE ok / VKDEVS 2 / VKDEVICE ok family=0` then straight to the +image path. `t99.py`-style attach works too: attaching `BW` to an +already-stuck client writes the word and the client immediately +proceeds (proven on a stuck pid 15797: WORDWROTE x2 -> VKDEVICE ok -> +VKFAIL -2 at the alloc). +NEW TOOLS: `t120.py` (BC+BW+single-shot COMPR->non-COMPR flip), +`t121.py` (BC+BW+full VidHeap trace from #120), `t122.py` (BC+BW+flip +for ANY COMPR+2MB internal alloc, size >= 0x10000), `a8clear.py` (the +no-ptrace count-gate clearer for base+0x2070b7c, reconstructed). + +**B. THE FLIP (THE DECISIVE EXPERIMENT).** With `t122.py`: at the +vidheap ioctl whose params match (attr&0xF000)==0x2000, owner +0x474C0000, size>=0x10000, the script clears attr bits 13:12 +(COMPR_ANY -> NONE) in the client's own params before the kernel sees +them. Result (r705c, reproducible twice on the same boot): +`HIT149 size=0x140000 attr=0x1002000 -> 0x1000000` (image data alloc +SUCCEEDS), a second `HIT149 size=0x140000 attr=0x1022004 -> 0x1020004` +(buffer-class COMPR alloc) also succeeds, and the client runs the WHOLE +pipeline for the first time in project history: +vkCreateImageView -> buffer -> shader module -> descriptor set +layout/pool -> pipeline layout -> compute pipeline -> command pool -> +command buffer -> fence -> vkResetFences -> **vkQueueSubmit OK** -> +`VKFAIL vkWaitForFences(dev, 1, &fence, VK_TRUE, 30000000000ull) -> 2 +(line 459)` - THE FIRST VULKAN SUBMISSION, then the FIRST FENCE TIMEOUT +(30 s, the client's own bound; the run exits rc=1 cleanly). + +**C. THE NEW WALL = THE FIRST FENCE RELEASE.** Host-side evidence in the +submit window: the client's GR channel registers +(`sema-chan: h=0xbeec0100 eng=0 gpfifo=0x4cc0000 ent=32768 +userd=0xfee00 put=11 get=2 ring rc=0 pa=0x1217e3000 ap=1`) - the guest +driver published 11 GPFIFO entries, the emulation's reader consumed 2; +the service still reports the channel-init pushes unread +(`sema-entry: unread h=0xbeec0100 idx=2 va=0x4ca059c len=976 want=0 +exp=0 unread=1`) and `gr_wedge` grew 3 -> 9 over the runs. So the fence's +release is never authored after the submit - this is the session-76/93 +"wall #2" class (push-page readability/completion), now the ONLY gate +between the flipped client and the first `VKFRAME seq/DONE` + PPM. + +**D. DURABLE-FIX LEADS (for the COMPR class, no guest patching).** +(1) memmgrAllocHal_GM107 (570.144) downgrades COMPR_ANY -> NONE when +`pMemorySystemConfig->bDisableCompbitBacking` is set - the guest's +MEMORY_SYSTEM_STATIC_CONFIG comes from the GSP (the MEMSYS_GET_STATIC_ +CONFIG control 0x20800a1c / the GSP static info path); answering it (or +the same knob wherever the guest's config actually comes from) is the +host-side durable equivalent of the flip. 0x20800a1c was NEVER observed +in the RM_CONTROL log, so first find where the guest's config is fed +(control cache? GSP init args?) before coding the handler. +(2) Alternatively locate glcore's COMPR decision (attr construction at +eglcore+0x2c83fb/0x2c8465 and +0x50e038/0x50e894/0x50e8bd per session +113) and answer the capability that drives it. (3) The session-85 +device-local -2 story (2g mdev type) remains a prerequisite context. + +NEXT (ranked): (1) author the fence release for the first submit (the +push-page-5 content-match / pinning machinery; or have the service +complete the fence when it consumes the submit's GPFIFO entries); (2) +then the first VKFRAME seq/DONE + PPM; (3) fold the COMPR flip into a +host-side answer (see D); (4) the mission series (Xorg/GZDOOM, KVMFR, +CUDA 15/15 + soak, test series, video, Windows). + +**40. Session 115: THE FENCE WALL DECOMPOSED - THE CLIENT VAS PAGE DIRECTORY IS UNDISCOVERABLE AND THE PUSHES ARE NONCONTIGUOUS; RING PAGES ARE CONTENT-LOCATABLE AND INJECTABLE, BUT THE RING PAIR ALONE DOES NOT UNLOCK THE PUSH DECODE.** +(1) The fence timeout is upstream of everything else: the sema service cannot +decode the submit's GR pushes (`sema-entry: unread ... va=0x4ca0000 +len=1432` -> `skipping stale entry ... after 185-194 failed rescues`; +`work_gr submitted=0` - no GR forward ever starts). (2) The pump's push +reads resolve ONLY through the client VAS page directory (or pins). The +service has NO pdb for this client: `walk_probe` of the ring and push VAs +prints `discovered pdb=0x0 found=0` and no registered VASes; the r496f +vidmem pdb discovery never ran (it is only called from the RAM ring-scan +latch), and the r496j COPY_SERVER_PDES candidate (0x109b9c000) is garbage +(its content is not a page table). (3) THE RAM RING SCAN MISSES BY +CONSTRUCTION for this client: it sweeps [0, 6 GB) in 12 slices and gives +up (`found no candidate in 12 slice(s)`), but the sweep runs BEFORE the +client's pushes exist and never re-runs - while the ring page content is +provably in RAM (content-located below). (4) CONTENT LOCATION WORKS: the +GR ring page (ring VA 0x4cc0000, first entries {va=0x4ca0000 len=1432, +va=0x4ca0598 len=4}) was found in the guest RAM file by its first two GPFIFO +entries (at GPA 0x12C600000 then 0x12898c000 on later runs - per-run +allocations), and `sema_map_add` injection of the TRUE ring pair is +accepted (`INJECT va=0x4d20000 pa=0x13619d000 : ok`). BUT the pushes do not +become readable from the ring pair alone: the pushbuffer is NONCONTIGUOUS +(the window guess `ring_pa - 0x20000` reads zeros for both the GR and the +CE channels), the pushes are not CPU-mapped in the client process (the +ring GPA is absent from /proc/PID/pagemap of every rw-s nvidia mapping), +and a bottom-up search for the level-3 table (a 4 KB page whose index-2 +big/small entry maps into the ring's 2 MB window) found ZERO candidates in +RAM - the page tables are almost certainly in VIDMEM (r496f's note) or use +a shape the walker does not model. (5) The service's OWN successful read of +one CE init push (`sema-push: h=0xbeec0104 va=0x4d00000 ... rel=0x4b19b10 +pay=1 have_rel=1`, `ops_runs=0`) did NOT apply its release either - +whatever resolved that one read is not reproducible for the GR pushes. +NEW TOOLS: `fence_assist.py` (host: watches a `dmesg -w` capture for the +pump's entry decodes, content-locates the ring pages by their first two +entries in the RAM file, injects the pairs via sema_map_add - proven to +find and inject, but the ring pair alone is insufficient) and +`find_push.py` (guest: tests whether the client CPU-maps the ring GPA). +NEXT (ranked, concrete): (1) find the client VAS page directory: extract the +pdb from the GUEST kernel's RM objects (kcore walk of the VAS/GPU object +for this channel - the GPFIFO ring page content anchors the search), then +register it via the service's VAS table (a new sysfs/debugfs entry point +or the existing candidate path) - this is the "PDB injection" the mission +names; (2) or scan VIDMEM for the pdb with the ring pair as the verifier +(extend `svc_xvas_pdb_discover`'s trigger: call it from the ring-injection +path, not only from the RAM scan latch); (3) make the RAM ring scan re-arm +per channel with the pushes present (its 12-slice give-up is the reason +the auto-discovery never latches on this client); (4) then the submit +decodes, the release lands and `vkWaitForFences` returns -> VKFRAME. diff --git a/tools/guest-diag/wall2/a8clear.py b/tools/guest-diag/wall2/a8clear.py new file mode 100644 index 0000000..261312a --- /dev/null +++ b/tools/guest-diag/wall2/a8clear.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# a8clear.py - no-ptrace A8/NvOgl count clearer: finds each live vk_frame's +# libnvidia-eglcore base (the off==0 mapping) and writes 0 to the u32 at +# base+0x2070b7c whenever it is nonzero (the count gate t91 cleared via gdb). +import os, subprocess, sys, time +DUR = float(sys.argv[1]) if len(sys.argv) > 1 else 600 +def pids(): + try: + out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout + return [int(x) for x in out.split()] + except Exception: + return [] +log = set() +end = time.time() + DUR +while time.time() < end: + for pid in pids(): + base = None + try: + for line in open("/proc/%d/maps" % pid): + p = line.split() + if len(p) < 6 or "libnvidia-eglcore" not in p[-1]: + continue + try: + off = int(p[2], 16) + except Exception: + continue + if off == 0: + base = int(p[0].split("-")[0], 16) + break + except Exception: + continue + if not base: + continue + w = base + 0x2070b7c + try: + fd = os.open("/proc/%d/mem" % pid, os.O_RDWR) + os.lseek(fd, w, 0) + v = os.read(fd, 4) + if len(v) == 4: + n = int.from_bytes(v, "little") + if n: + os.lseek(fd, w, 0) + os.write(fd, b"\x00\x00\x00\x00") + k = (pid, n) + if len(log) < 40: + log.add(k) + print("A8CLEAR pid=%d count=%u -> 0" % (pid, n), flush=True) + os.close(fd) + except Exception: + pass + time.sleep(0.5) +print("A8CLEAR-DONE", flush=True) diff --git a/tools/guest-diag/wall2/fasttick.py b/tools/guest-diag/wall2/fasttick.py new file mode 100644 index 0000000..413d3ba --- /dev/null +++ b/tools/guest-diag/wall2/fasttick.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# fasttick.py - watch the service log for the GR0 channel's ring resolution +# (sema-chan: h=0xbeec0100 ... ring rc=0 pa=0x...) and immediately hand the +# verified {ring VA -> ring PA} pair to the service (sema_pdb_discover), so +# the parked VAS pdb candidate can be verified before/at age-out. +import re, subprocess, sys, time + +PARAM = "/sys/module/vgsp/parameters/sema_pdb_discover" +RING_VA = 0x4cc0000 + +def main(): + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 120 + end = time.time() + dur + fired = set() + seen_pdbs = set() + print("# fasttick start (%.0fs)" % dur, flush=True) + while time.time() < end: + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + tail = out.splitlines()[-120:] + for ln in tail: + m = re.search(r"sema-chan: h=0xbeec0100 eng=\d+ cid=\d+ gpfifo=0x([0-9a-f]+) ent=\d+ userd=0x[0-9a-f]+ as=\d+ put=\d+ get=\d+ ring rc=0 pa=0x([0-9a-f]+)", ln) + if m: + ring_va = int(m.group(1), 16) + ring_pa = int(m.group(2), 16) + key = (ring_va, ring_pa) + if key not in fired: + fired.add(key) + cmd = "echo 0x%x,0x%x > %s" % (ring_va, ring_pa, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print("[%s] TRIGGER va=0x%x pa=0x%x rc=%d" % ( + time.strftime("%H:%M:%S"), ring_va, ring_pa, r.returncode), flush=True) + m2 = re.search(r"page directory (?:registered UNVERIFIED after \d+ ms parked \()?pdb=0x([0-9a-f]+) entries=(\d+)", ln) + if not m2: + m2 = re.search(r"VA space 0x[0-9a-f]+ page directory 0x([0-9a-f]+) entries=(\d+)", ln) + if m2: + key = int(m2.group(1), 16) + if key not in seen_pdbs: + seen_pdbs.add(key) + print("[%s] PDB candidate 0x%x entries=%s" % ( + time.strftime("%H:%M:%S"), key, m2.group(2)), flush=True) + time.sleep(0.3) + print("# fasttick end", flush=True) + +main() diff --git a/tools/guest-diag/wall2/fence_assist.py b/tools/guest-diag/wall2/fence_assist.py new file mode 100644 index 0000000..4ea0b0c --- /dev/null +++ b/tools/guest-diag/wall2/fence_assist.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# fence_assist.py - content-locate the client's ring pages in guest RAM from the +# service's own GPFIFO entry decodes (dmesg) and inject the TRUE VA->PA pairs +# via sema_map_add, so the pump's push reads (and its pushbuffer pairing) have +# a correct anchor. +import mmap, re, struct, subprocess, sys, threading, time + +F = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_map_add" + +def gpa_of(fo): + return fo if fo < 0x80000000 else fo - 0x80000000 + 0x100000000 + +def inject(va, pa, ln=0x1000, flag=1): + cmd = "echo 0x%x,0x%x,0x%x,%d > %s" % (va & ~0xfff, pa & ~0xfff, ln, flag, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print("INJECT va=0x%x pa=0x%x : %s" % (va & ~0xfff, pa & ~0xfff, + "ok" if r.returncode == 0 else "FAIL"), flush=True) + +def search(mm, pats, results): + CH = 8 << 20 + size = len(mm) + off = 0 + while off < size: + d = mm[off:off+CH] + for (name, pat) in pats.items(): + if name in results: + continue + i = d.find(pat) + if i >= 0: + results[name] = gpa_of(off + i) + off += CH - 32 + +def main(): + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 120 + f = open(F, "rb") + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + end = time.time() + dur + pats = {} # name -> pattern bytes + ring_va = {} # name -> ring VA + seen = set() # raw entry lines processed + done = set() # injected names + print("# fence_assist start (%.0fs)" % dur, flush=True) + while time.time() < end: + try: + with open("/tmp/opencode/dmsg_w.txt", "r", errors="ignore") as df: + tail = df.read().splitlines()[-6000:] + out = "\n".join(tail) + except OSError: + out = "" + tail = [] + tail = [ln for ln in tail if "sema-chan:" in ln or "sema-push" in ln] + cur = {} + for ln in tail: + m = re.search(r"sema-chan: entry\[(\d+)\] h=0x([0-9a-f]+) eng=\d+ raw=([0-9a-f]+) va=0x([0-9a-f]+)", ln) + if m: + idx, h, raw, va = int(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4), 16) + cur.setdefault(h, {})[idx] = raw + if idx == 0: + ring_va[h] = va # entry[0] va is the push VA (ring_va - 0x20000) + for h, ents in cur.items(): + key = (h, tuple(sorted(ents.items()))) + if key in seen or len(ents) < 2: + continue + seen.add(key) + name = h + pat = b"".join(struct.pack("<Q", ents[i]) for i in sorted(ents)[:2]) + pats[name] = pat + print("# channel h=0x%s entries=%s pattern ready" % (h, sorted(ents)), flush=True) + if pats: + results = {} + threads = [] + # split the map among 4 threads + LO = 0x100000000; HI = 0x180000000 + size = len(mm) + step = size // 4 + class Sl(shim := object): + def __init__(self, a, b): + self.a, self.b = a, b + def worker(a, b): + CH = 8 << 20 + off = a + while off < b: + if off < 0x80000000 and b <= 0x80000000: pass + d = mm[off:off+CH] + for (nm, p) in list(pats.items()): + if nm in results: continue + i = d.find(p) + if i >= 0: + results[nm] = gpa_of(off + i) + off += CH - 32 + for k in range(4): + a = 0x80000000 + k * (0x80000000 // 4) + b = 0x80000000 + (k + 1) * (0x80000000 // 4) + t = threading.Thread(target=worker, args=(a, b)) + t.start(); threads.append(t) + for t in threads: t.join() + for nm, gpa in results.items(): + if nm in done: continue + done.add(nm) + # ring page content starts with entry[0]; the ring VA is the + # channel's gpfifo VA = we reconstruct: entry0 va is the push, + # ring = first "gpfifo=0x..." dmesg line for this h + m = re.search(r"sema-chan: h=0x%s eng=\d+ cid=\d+ gpfifo=0x([0-9a-f]+)" % nm, out) + rva = int(m.group(1), 16) if m else None + if rva: + print("RING FOUND h=0x%s gpfifo_va=0x%x -> gpa=0x%x" % (nm, rva, gpa), flush=True) + inject(rva, gpa, 0x1000) + time.sleep(0.6) + print("# fence_assist done, injected %d" % len(done), flush=True) + +main() diff --git a/tools/guest-diag/wall2/find_push.py b/tools/guest-diag/wall2/find_push.py new file mode 100644 index 0000000..4bf8ccc --- /dev/null +++ b/tools/guest-diag/wall2/find_push.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# find_push.py - locate the client's CPU mapping that owns the GR ring page +# (GPA 0x12C600000 as content-located on the host) and print the CPU VA<-> +# GPU VA correspondence for the low GPU VAs (0x4ca0000 etc). +import os, struct, sys, subprocess, mmap, time + +RING_GPA = 0x12C600000 +PUSH_GVA = 0x4ca0000 +RING_GVA = 0x4cc0000 + +def pids(): + out = subprocess.run(["pgrep", "-x", "vk_frame"], capture_output=True).stdout + return [int(x) for x in out.split()] + +def read_pagemap(pid, va): + try: + with open("/proc/%d/pagemap" % pid, "rb") as f: + f.seek((va // 4096) * 8) + d = f.read(8) + if len(d) < 8: + return None + e = int.from_bytes(d, "little") + if not (e >> 63) & 1: + return None + return (e & ((1 << 55) - 1)) << 12 + except OSError: + return None + +def main(): + for pid in pids(): + print("pid=%d" % pid) + try: + maps = open("/proc/%d/maps" % pid).read().splitlines() + except OSError: + continue + for line in maps: + p = line.split() + if len(p) < 6 or "nvidia" not in p[-1]: + continue + lo, hi = [int(x, 16) for x in p[0].split("-")] + if hi - lo > 0x1000000: + continue + # scan pages; look for RING_GPA + for va in range(lo, hi, 0x10000): + for off in range(0, 0x10000, 0x1000): + pa = read_pagemap(pid, va + off) + if pa == RING_GPA: + print("FOUND ring page: cpu_va=0x%x (map 0x%x-0x%x %s)" % (va+off, lo, hi, p[-1])) + delta_cpu = (va + off) - RING_GVA + push_cpu = PUSH_GVA + delta_cpu + print(" delta_cpu = 0x%x" % delta_cpu) + for gva in (PUSH_GVA, PUSH_GVA+0x1000, 0x4ca059c, 0x4ca0980, 0x4ca5870, 0x4903000): + cva = gva + delta_cpu + pa2 = read_pagemap(pid, cva & ~0xfff) + print(" gva=0x%x -> cpu=0x%x -> pa=%s" % (gva, cva, hex(pa2) if pa2 else None)) + return + print(" ring GPA 0x%x not found in this pid's mappings" % RING_GPA) + +main() diff --git a/tools/guest-diag/wall2/kc_dump_fn.py b/tools/guest-diag/wall2/kc_dump_fn.py new file mode 100644 index 0000000..e481d24 --- /dev/null +++ b/tools/guest-diag/wall2/kc_dump_fn.py @@ -0,0 +1,21 @@ +import struct, sys +VA = int(sys.argv[1], 16) +N = int(sys.argv[2], 16) if len(sys.argv) > 2 else 0x800 +f = open("/proc/kcore","rb") +hdr = f.read(64) +e_phoff = struct.unpack_from("<Q", hdr, 0x20)[0] +e_phentsize = struct.unpack_from("<H", hdr, 0x36)[0] +e_phnum = struct.unpack_from("<H", hdr, 0x38)[0] +segs = [] +for i in range(e_phnum): + f.seek(e_phoff + i*e_phentsize) + ph = f.read(e_phentsize) + p_type, p_flags, p_offset, p_vaddr, p_paddr, p_filesz, p_memsz = struct.unpack_from("<IIQQQQQ", ph, 0) + if p_type == 1: + segs.append((p_vaddr, p_memsz, p_offset)) +for v, sz, off in segs: + if v <= VA < v+sz: + f.seek(off + (VA - v)) + d = f.read(min(N, v+sz-VA)) + sys.stdout.write(d.hex()) + break diff --git a/tools/guest-diag/wall2/pdbtick.py b/tools/guest-diag/wall2/pdbtick.py new file mode 100644 index 0000000..d65b79d --- /dev/null +++ b/tools/guest-diag/wall2/pdbtick.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# pdbtick.py - content-locate the GR0 ring page (deterministic first three +# GPFIFO entries) in the guest RAM file, then hand the verified +# {ring VA -> ring PA} pair to the service (sema_pdb_discover), which +# verifies the parked COPY_SERVER_PDES candidates and falls back to the +# vidmem walk-root sweep. Runs while the client is alive. +import mmap, struct, subprocess, sys, threading, time + +F = "/dev/shm/vgpu-guest-ram" +PARAM = "/sys/module/vgsp/parameters/sema_pdb_discover" +PAT = struct.pack("<QQQ", 0x0005980004ca0000, 0x8000040004ca0598, + 0x0003d00004ca059c) +RING_VA = 0x4cc0000 + +def gpa_of(x): + return x if x < 0x80000000 else x - 0x80000000 + 0x100000000 + +def main(): + fobj = open(F, "rb") + mm = mmap.mmap(fobj.fileno(), 0, access=mmap.ACCESS_READ) + size = len(mm) + nthr = 8 + step = size // nthr + res = [] + lock = threading.Lock() + def worker(a, b): + CH = 8 << 20 + off = a + while off < b: + d = mm[off:off + CH] + i = d.find(PAT) + while i >= 0: + with lock: + res.append(off + i) + i = d.find(PAT, i + 1) + off += CH - 64 + ts = [] + for k in range(nthr): + a = k * step + b = (k + 1) * step if k < nthr - 1 else size + t = threading.Thread(target=worker, args=(a, b)); t.start(); ts.append(t) + for t in ts: t.join() + if not res: + print("RING NOT FOUND") + return None + hits = [gpa_of(x) for x in res] + print("ring candidates:", [hex(h) for h in hits], flush=True) + for gpa in hits: + cmd = "echo 0x%x,0x%x > %s" % (RING_VA, gpa, PARAM) + r = subprocess.run(["sudo", "sh", "-c", cmd], capture_output=True) + print("sema_pdb_discover 0x%x,0x%x rc=%d" % (RING_VA, gpa, r.returncode), flush=True) + return hits + + +def loop(): + import time as _t + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 120 + end = _t.time() + dur + fired = set() + while _t.time() < end: + hits = main() + if hits: + new = [h for h in hits if h not in fired] + if new: + fired.update(new) + _t.sleep(1.0) + print("# pdbtick end", flush=True) + +loop() diff --git a/tools/guest-diag/wall2/runbook.py b/tools/guest-diag/wall2/runbook.py new file mode 100644 index 0000000..256d255 --- /dev/null +++ b/tools/guest-diag/wall2/runbook.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# runbook.py - watch for the GR0 channel ring registration, hand the pair to +# sema_pdb_discover, then (after the pdb anchor) issue bounded +# sema_chan_rewind calls so entries skipped before the anchor re-decode. +import re, subprocess, sys, time + +PDB = "/sys/module/vgsp/parameters/sema_pdb_discover" +REW = "/sys/module/vgsp/parameters/sema_chan_rewind" + +def dmesg_tail(n=150): + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + return out.splitlines()[-n:] + +def main(): + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 180 + end = time.time() + dur + fired = set() + rewinds = {} + print("# runbook start (%.0fs)" % dur, flush=True) + while time.time() < end: + for ln in dmesg_tail(): + m = re.search(r"sema-chan: h=0x([0-9a-f]+) eng=\d+ cid=\d+ gpfifo=0x([0-9a-f]+) ent=\d+ userd=0x[0-9a-f]+ as=\d+ put=\d+ get=\d+ ring rc=0 pa=0x([0-9a-f]+)", ln) + if not m: + continue + h = int(m.group(1), 16) + gv = int(m.group(2), 16) + rp = int(m.group(3), 16) + key = (h, gv, rp) + if key in fired: + # after the anchor has had time to land, rewind this channel + if h == 0xbeec0100: + t0 = rewinds.setdefault(key, time.time()) + n = int((time.time() - t0) // 8) + done = rewinds.get(("done", key), 0) + if n > done: + subprocess.run(["sudo", "sh", "-c", + "echo all > %s" % REW], capture_output=True) + rewinds[("done", key)] = n + print("[%s] REWIND all (h=0x%x, n=%d)" % ( + time.strftime("%H:%M:%S"), h, n), flush=True) + continue + fired.add(key) + r = subprocess.run(["sudo", "sh", "-c", + "echo 0x%x,0x%x > %s" % (gv, rp, PDB)], + capture_output=True) + print("[%s] TRIGGER h=0x%x va=0x%x pa=0x%x rc=%d" % ( + time.strftime("%H:%M:%S"), h, gv, rp, r.returncode), flush=True) + # report decodes/releases + out = subprocess.run(["sudo", "dmesg"], capture_output=True).stdout.decode(errors="ignore") + for ln in out.splitlines()[-8:]: + if ("sema-push: h=0xbeec0100" in ln and "have_rel=1" in ln) or \ + "registered after verified" in ln or "rewind" in ln or \ + ("sema-push: h=0xbeec0100" in ln and "len=1432" in ln): + print("LOG %s" % ln.strip()[-170:], flush=True) + time.sleep(0.4) + print("# runbook end", flush=True) + +main() diff --git a/tools/guest-diag/wall2/t112c-gdbstub.py b/tools/guest-diag/wall2/t112c-gdbstub.py new file mode 100644 index 0000000..27d0b9d --- /dev/null +++ b/tools/guest-diag/wall2/t112c-gdbstub.py @@ -0,0 +1,42 @@ +import gdb + +gr = gdb.execute + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +state = {"hits": {}} + +class HB(gdb.Breakpoint): + def __init__(self, pa, tag, quiet_after=8): + super().__init__("*0x%x" % pa, type=gdb.BP_BREAKPOINT, internal=False) + self.tag = tag + self.quiet_after = quiet_after + self.silent = True + def stop(self): + n = state["hits"].get(self.tag, 0) + 1 + state["hits"][self.tag] = n + try: + rdi = int(gdb.parse_and_eval("$rdi")) + rsi = int(gdb.parse_and_eval("$rsi")) + rdx = int(gdb.parse_and_eval("$rdx")) + rcx = int(gdb.parse_and_eval("$rcx")) + r8 = int(gdb.parse_and_eval("$r8")) + r9 = int(gdb.parse_and_eval("$r9")) + if n <= self.quiet_after or self.tag in ("PRED", "COMPR"): + print("HIT %s #%d rdi=0x%x rsi=0x%x rdx=0x%x rcx=0x%x r8=0x%x r9=0x%x" % + (self.tag, n, rdi, rsi, rdx, rcx, r8, r9), flush=True) + except Exception as e: + print("err", e, flush=True) + return False + +HB(0xffffffffc0c403b0, "VIDH") +HB(0xffffffffc0d667a0, "PMA", 4) +HB(0xffffffffc0d6ae30, "PRED") +HB(0xffffffffc0d57d60, "COMPR", 16) +HB(0xffffffffc0d607f0, "SCRUB", 4) +print("HB-SET", flush=True) +gr("continue") +print("T112B-DONE %s" % state["hits"], flush=True) diff --git a/tools/guest-diag/wall2/t120.py b/tools/guest-diag/wall2/t120.py new file mode 100644 index 0000000..68a1b8b --- /dev/null +++ b/tools/guest-diag/wall2/t120.py @@ -0,0 +1,148 @@ +import gdb +gr = gdb.execute +state = {"n": 0, "any": 0, "pend": None, "base": None, "chits": 0, "cleared": 0, + "whits": 0, "wrote": 0} + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +def ioc(d, t, nr, size): + return (d << 30) | (size << 16) | (t << 8) | nr + +VIDHEAP = ioc(3, 0x46, 0x4a, 0xb8) + +def glbase(): + try: + pid = gdb.selected_inferior().pid + for line in open("/proc/%d/maps" % pid): + parts = line.split() + if "libnvidia-eglcore" in line and int(parts[2], 16) == 0: + return int(parts[0].split("-")[0], 16) + except Exception: + pass + return None + +class BC(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i * 8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + return False + +class BW(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val), flush=True) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + except Exception: + pass + return False + +class IOC(gdb.Breakpoint): + def __init__(self): + super().__init__("ioctl", internal=False) + self.silent = True + def stop(self): + if state["base"] is None: + b = glbase() + if b: + state["base"] = b + print("GLBASE 0x%x" % b, flush=True) + try: + BC(b + 0xB33D30) + BW(b + 0xDCEBD9) + print("BC-BW-SET", flush=True) + except Exception as e: + print("bpset-fail %s" % e, flush=True) + try: + req = int(gdb.parse_and_eval("$rsi")) & 0xFFFFFFFF + arg = int(gdb.parse_and_eval("$rdx")) + except Exception: + return False + if state["pend"] is not None: + try: + st = u32(state["pend"] + 0x14) + at = u32(state["pend"] + 0x38) + off = u64(state["pend"] + 0x68) + print("PREV149-OUT st=0x%x attr=0x%x off=0x%x" % (st, at, off), flush=True) + except Exception as e: + print("PREV149-STATUS unreadable %s" % e, flush=True) + state["pend"] = None + if req != VIDHEAP: + return False + state["any"] += 1 + try: + fn = u32(arg + 0x08) + size = u64(arg + 0x58) + attr = u32(arg + 0x38) + owner = u32(arg + 0x28) + flags = u32(arg + 0x34) + except Exception: + return False + if fn == 2 and size == 0x10000 and (attr & 0xf000) == 0x2000 and owner == 0x474C0000: + state["n"] += 1 + print("HIT149 n=%d attr=0x%x flags=0x%x owner=0x%x align=0x%x" % ( + state["n"], attr, flags, owner, u64(arg + 0x60)), flush=True) + if state["n"] == 1: + gr("bt 12") + newattr = attr & 0xFFFFCFFF + gr("set {unsigned int}0x%x = 0x%x" % (arg + 0x38, newattr)) + print("FLIP149 attr 0x%x -> 0x%x" % (attr, newattr), flush=True) + state["pend"] = arg + return False + +try: + IOC() + print("BPS-SET", flush=True) +except Exception as e: + print("bpset-fail %s" % e, flush=True) + +gr("set pagination off") +gr("set confirm off") +try: + gr("run") +except Exception as e: + print("runend %s" % e, flush=True) +print("T120-DONE n=%d any=%d chits=%d cleared=%d whits=%d wrote=%d" % ( + state["n"], state["any"], state["chits"], state["cleared"], + state["whits"], state["wrote"]), flush=True) +gr("quit") diff --git a/tools/guest-diag/wall2/t121.py b/tools/guest-diag/wall2/t121.py new file mode 100644 index 0000000..1ae25ed --- /dev/null +++ b/tools/guest-diag/wall2/t121.py @@ -0,0 +1,151 @@ +import gdb +gr = gdb.execute +state = {"n": 0, "any": 0, "pend": None, "base": None, "chits": 0, "cleared": 0, + "whits": 0, "wrote": 0} + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +def ioc(d, t, nr, size): + return (d << 30) | (size << 16) | (t << 8) | nr + +VIDHEAP = ioc(3, 0x46, 0x4a, 0xb8) + +def glbase(): + try: + pid = gdb.selected_inferior().pid + for line in open("/proc/%d/maps" % pid): + parts = line.split() + if "libnvidia-eglcore" in line and int(parts[2], 16) == 0: + return int(parts[0].split("-")[0], 16) + except Exception: + pass + return None + +class BC(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i * 8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + return False + +class BW(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val), flush=True) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + except Exception: + pass + return False + +class IOC(gdb.Breakpoint): + def __init__(self): + super().__init__("ioctl", internal=False) + self.silent = True + def stop(self): + if state["base"] is None: + b = glbase() + if b: + state["base"] = b + print("GLBASE 0x%x" % b, flush=True) + try: + BC(b + 0xB33D30) + BW(b + 0xDCEBD9) + print("BC-BW-SET", flush=True) + except Exception as e: + print("bpset-fail %s" % e, flush=True) + try: + req = int(gdb.parse_and_eval("$rsi")) & 0xFFFFFFFF + arg = int(gdb.parse_and_eval("$rdx")) + except Exception: + return False + if state["pend"] is not None: + try: + st = u32(state["pend"] + 0x14) + at = u32(state["pend"] + 0x38) + off = u64(state["pend"] + 0x68) + print("PREV149-OUT st=0x%x attr=0x%x off=0x%x" % (st, at, off), flush=True) + except Exception as e: + print("PREV149-STATUS unreadable %s" % e, flush=True) + state["pend"] = None + if req != VIDHEAP: + return False + state["any"] += 1 + try: + fn = u32(arg + 0x08) + size = u64(arg + 0x58) + attr = u32(arg + 0x38) + owner = u32(arg + 0x28) + flags = u32(arg + 0x34) + except Exception: + return False + if fn == 2: + state["n"] += 1 + if state["n"] >= 120: + print("VH#%d size=0x%x align=0x%x attr=0x%x attr2=0x%x flags=0x%x owner=0x%x range=0x%x-0x%x" % ( + state["n"], size, u64(arg + 0x60), attr, u32(arg + 0x90), flags, owner, + u64(arg + 0x80), u64(arg + 0x88)), flush=True) + if size == 0x10000 and (attr & 0xf000) == 0x2000 and owner == 0x474C0000: + print("HIT149 attr=0x%x flags=0x%x" % (attr, flags), flush=True) + gr("bt 12") + newattr = attr & 0xFFFFCFFF + gr("set {unsigned int}0x%x = 0x%x" % (arg + 0x38, newattr)) + print("FLIP149 attr 0x%x -> 0x%x" % (attr, newattr), flush=True) + state["pend"] = arg + return False + +try: + IOC() + print("BPS-SET", flush=True) +except Exception as e: + print("bpset-fail %s" % e, flush=True) + +gr("set pagination off") +gr("set confirm off") +try: + gr("run") +except Exception as e: + print("runend %s" % e, flush=True) +print("T120-DONE n=%d any=%d chits=%d cleared=%d whits=%d wrote=%d" % ( + state["n"], state["any"], state["chits"], state["cleared"], + state["whits"], state["wrote"]), flush=True) +gr("quit") diff --git a/tools/guest-diag/wall2/t122.py b/tools/guest-diag/wall2/t122.py new file mode 100644 index 0000000..8c9c6fd --- /dev/null +++ b/tools/guest-diag/wall2/t122.py @@ -0,0 +1,151 @@ +import gdb +gr = gdb.execute +state = {"n": 0, "any": 0, "pend": None, "base": None, "chits": 0, "cleared": 0, + "whits": 0, "wrote": 0} + +def u32(a): + return int(gdb.parse_and_eval("*(unsigned int*)0x%x" % a)) +def u64(a): + return int(gdb.parse_and_eval("*(unsigned long*)0x%x" % a)) + +def ioc(d, t, nr, size): + return (d << 30) | (size << 16) | (t << 8) | nr + +VIDHEAP = ioc(3, 0x46, 0x4a, 0xb8) + +def glbase(): + try: + pid = gdb.selected_inferior().pid + for line in open("/proc/%d/maps" % pid): + parts = line.split() + if "libnvidia-eglcore" in line and int(parts[2], 16) == 0: + return int(parts[0].split("-")[0], 16) + except Exception: + pass + return None + +class BC(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["chits"] += 1 + try: + obj = int(gdb.parse_and_eval("$rdi")) + n = int(gdb.parse_and_eval("$rsi")) + cnt = u32(obj + 0x22efc) + for i in range(0, min(cnt, 32)): + p = u64(obj + 0x1428 + i * 8) + if not p: + continue + tgt = p + n * 0x10 + try: + v16 = int(gdb.parse_and_eval("*(unsigned short*)0x%x" % (tgt + 0xe))) + if v16 == 0xffff: + gr("set {unsigned short}0x%x = 0" % (tgt + 0xe)) + gr("set {unsigned int}0x%x = 0" % tgt) + gr("set {unsigned short}0x%x = 0" % (tgt + 0xc)) + state["cleared"] += 1 + print("CLEAR#%d obj=0x%x n=0x%x tgt=0x%x" % (state["cleared"], obj, n, tgt), flush=True) + except Exception: + pass + except Exception: + pass + return False + +class BW(gdb.Breakpoint): + def __init__(self, addr): + super().__init__("*0x%x" % addr, internal=False) + self.silent = True + def stop(self): + state["whits"] += 1 + try: + rbx = int(gdb.parse_and_eval("$rbx")) + sub = u64(rbx + 0x70) + obj2 = u64(sub + 0x58) + off = u32(obj2 + 0x60) + holder = u64(obj2 + 0x58) + base2 = u64(holder + 0x58) + word = base2 + off + val = u32(word) + if state["whits"] <= 6 or val == 0: + print("WORD#%d addr=0x%x val=0x%x" % (state["whits"], word, val), flush=True) + if val == 0: + gr("set {unsigned int}0x%x = 1" % word) + state["wrote"] += 1 + except Exception: + pass + return False + +class IOC(gdb.Breakpoint): + def __init__(self): + super().__init__("ioctl", internal=False) + self.silent = True + def stop(self): + if state["base"] is None: + b = glbase() + if b: + state["base"] = b + print("GLBASE 0x%x" % b, flush=True) + try: + BC(b + 0xB33D30) + BW(b + 0xDCEBD9) + print("BC-BW-SET", flush=True) + except Exception as e: + print("bpset-fail %s" % e, flush=True) + try: + req = int(gdb.parse_and_eval("$rsi")) & 0xFFFFFFFF + arg = int(gdb.parse_and_eval("$rdx")) + except Exception: + return False + if state["pend"] is not None: + try: + st = u32(state["pend"] + 0x14) + at = u32(state["pend"] + 0x38) + off = u64(state["pend"] + 0x68) + print("PREV149-OUT st=0x%x attr=0x%x off=0x%x" % (st, at, off), flush=True) + except Exception as e: + print("PREV149-STATUS unreadable %s" % e, flush=True) + state["pend"] = None + if req != VIDHEAP: + return False + state["any"] += 1 + try: + fn = u32(arg + 0x08) + size = u64(arg + 0x58) + attr = u32(arg + 0x38) + owner = u32(arg + 0x28) + flags = u32(arg + 0x34) + except Exception: + return False + if fn == 2: + state["n"] += 1 + if state["n"] >= 120: + print("VH#%d size=0x%x align=0x%x attr=0x%x attr2=0x%x flags=0x%x owner=0x%x range=0x%x-0x%x" % ( + state["n"], size, u64(arg + 0x60), attr, u32(arg + 0x90), flags, owner, + u64(arg + 0x80), u64(arg + 0x88)), flush=True) + if size >= 0x10000 and (attr & 0xf000) == 0x2000 and owner == 0x474C0000: + print("HIT149 size=0x%x attr=0x%x flags=0x%x" % (size, attr, flags), flush=True) + gr("bt 12") + newattr = attr & 0xFFFFCFFF + gr("set {unsigned int}0x%x = 0x%x" % (arg + 0x38, newattr)) + print("FLIP149 attr 0x%x -> 0x%x" % (attr, newattr), flush=True) + state["pend"] = arg + return False + +try: + IOC() + print("BPS-SET", flush=True) +except Exception as e: + print("bpset-fail %s" % e, flush=True) + +gr("set pagination off") +gr("set confirm off") +try: + gr("run") +except Exception as e: + print("runend %s" % e, flush=True) +print("T120-DONE n=%d any=%d chits=%d cleared=%d whits=%d wrote=%d" % ( + state["n"], state["any"], state["chits"], state["cleared"], + state["whits"], state["wrote"]), flush=True) +gr("quit") diff --git a/tools/vk/vk_frame.c b/tools/vk/vk_frame.c index 8b864ff..62548be 100644 --- a/tools/vk/vk_frame.c +++ b/tools/vk/vk_frame.c @@ -27,6 +27,8 @@ #define FRAME_OFFSET 0x100000UL #define HDR_MAGIC 0x4b564d46u +static VkImageTiling g_tiling = VK_IMAGE_TILING_OPTIMAL; + struct frame_hdr { uint32_t magic; uint32_t version; @@ -59,6 +61,11 @@ static uint64_t now_ns(void) int main(int argc, char **argv) { + if (argc > 7 && atoi(argv[7]) == 1) { + g_tiling = VK_IMAGE_TILING_LINEAR; + printf("TILING linear\n"); + } + int frames = argc > 1 ? atoi(argv[1]) : 60; uint32_t W = argc > 2 ? (uint32_t)atoi(argv[2]) : 640; uint32_t H = argc > 3 ? (uint32_t)atoi(argv[3]) : 480; @@ -179,7 +186,7 @@ int main(int argc, char **argv) ci.mipLevels = 1; ci.arrayLayers = 1; ci.samples = VK_SAMPLE_COUNT_1_BIT; - ci.tiling = VK_IMAGE_TILING_OPTIMAL; + ci.tiling = g_tiling; ci.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; ci.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; @@ -244,7 +251,25 @@ int main(int argc, char **argv) break; } } - VKC(vkAllocateMemory(dev, &ma, NULL, &buf_mem)); + { + int rc_ = vkAllocateMemory(dev, &ma, NULL, &buf_mem); + if (rc_ != VK_SUCCESS) { + VkPhysicalDeviceMemoryProperties mp2; + uint32_t k2; + vkGetPhysicalDeviceMemoryProperties(phys, &mp2); + for (k2 = 0; k2 < mp2.memoryTypeCount && rc_ != VK_SUCCESS; k2++) { + if (!(mr.memoryTypeBits & (1u << k2))) + continue; + ma.memoryTypeIndex = k2; + rc_ = vkAllocateMemory(dev, &ma, NULL, &buf_mem); + printf("BUF retry type=%u rc=%d\n", k2, rc_); + } + } + if (rc_ != VK_SUCCESS) { + printf("VKFAIL vkAllocateMemory(buf) -> %d (line %d)\n", rc_, __LINE__); + return 1; + } + } VKC(vkBindBufferMemory(dev, buf, buf_mem, 0)); } /* shader + pipeline */ |